From 4b18c2062ba377d41afa1126fe1510882ccdc1d7 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 07:07:22 +0800 Subject: [PATCH] fix(agent99): require durable queue receipt --- agent99-sre-alert-inbox.ps1 | 49 ++++++- agent99-sre-alert-relay.ps1 | 116 ++++++++++++++++- apps/api/src/api/v1/webhooks.py | 7 +- .../agent99_controlled_dispatch_ledger.py | 38 +++++- apps/api/src/services/agent99_sre_bridge.py | 122 ++++++++++++++++-- .../backup_restore_signal_automation.py | 21 ++- .../src/services/platform_operator_service.py | 9 ++ apps/api/src/services/telegram_gateway.py | 15 +++ .../test_agent99_controlled_dispatch_p1.py | 4 +- ...est_agent99_dispatch_receipt_projection.py | 4 +- apps/api/tests/test_agent99_sre_bridge.py | 98 ++++++++++---- .../test_awooop_operator_timeline_labels.py | 16 ++- ...est_backup_restore_alertmanager_ingress.py | 6 +- .../test_backup_restore_signal_automation.py | 6 +- .../test_controlled_alert_target_router.py | 4 +- ...t_agent99_recovery_coordinator_contract.py | 20 +++ 16 files changed, 476 insertions(+), 59 deletions(-) diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 5b2c05c3b..85936a403 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -17,11 +17,12 @@ $IgnoredDir = Join-Path $AlertRoot "ignored" $QueueDir = Join-Path $AgentRoot "queue" $RequestDir = Join-Path $AgentRoot "requests" $StateDir = Join-Path $AgentRoot "state" +$RelayReceiptDir = Join-Path $EvidenceDir "relay-dispatch-receipts" $DedupeStatePath = Join-Path $StateDir "sre-alert-single-flight.json" $InboxLockPath = Join-Path $StateDir "sre-alert-inbox.lock" $DedupeWindowSeconds = 300 -New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null +New-Item -ItemType Directory -Force -Path $EvidenceDir, $RelayReceiptDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage") $remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed") @@ -55,6 +56,45 @@ function Convert-AgentSafeFileToken { return $safe } +function Write-AgentRelayQueueReceipt { + param( + [object]$Alert, + [string]$AlertId, + [string]$Status, + [bool]$QueueAccepted = $false, + [string]$QueueId = "", + [string]$Reason = "", + [string]$DedupeDecision = "" + ) + + $receiptId = [string](Get-AgentField $Alert "relayReceiptId" "") + if ($receiptId -notmatch "^agent99-relay-[0-9a-f]{32}$") { return } + + $awoooi = Get-AgentField $Alert "awoooi" $null + $identity = Get-AgentField $awoooi "agent99DispatchIdentity" $null + $routing = Get-AgentField $Alert "routing" $null + $payload = [pscustomobject]@{ + schemaVersion = "agent99_sre_queue_receipt_v1" + receiptId = $receiptId + alertId = $AlertId + status = $Status + queueAccepted = $QueueAccepted + queueId = $QueueId + reason = $Reason + dedupeDecision = $DedupeDecision + automationRunId = [string](Get-AgentField $identity "run_id" (Get-AgentField $routing "runId" "")) + traceId = [string](Get-AgentField $identity "trace_id" (Get-AgentField $routing "traceId" "")) + workItemId = [string](Get-AgentField $identity "work_item_id" (Get-AgentField $routing "workItemId" "")) + executionGeneration = [string](Get-AgentField $identity "execution_generation" (Get-AgentField $routing "executionGeneration" "")) + generatedAt = (Get-Date -Format o) + storesRawPayload = $false + } + $path = Join-Path $RelayReceiptDir "agent99-sre-queue-receipt-$receiptId.json" + $temporaryPath = "$path.tmp-$PID" + $payload | ConvertTo-Json -Depth 6 | Set-Content -Path $temporaryPath -Encoding UTF8 + Move-Item -Force $temporaryPath $path +} + function Get-AgentAlertText { param([object]$Alert) $labels = @() @@ -675,6 +715,7 @@ foreach ($file in $files) { file = $ignoredPath reason = "not_actionable" } + Write-AgentRelayQueueReceipt $alert $alertId "rejected" $false "" "not_actionable" "not_actionable" continue } @@ -690,6 +731,7 @@ foreach ($file in $files) { reason = "mode_not_allowlisted" mode = $modeName } + Write-AgentRelayQueueReceipt $alert $alertId "rejected" $false "" "mode_not_allowlisted" "mode_not_allowlisted" continue } @@ -732,6 +774,7 @@ foreach ($file in $files) { originalSeverity = [string]$previous.severity } $duplicateSuppressedCount += 1 + Write-AgentRelayQueueReceipt $alert $alertId "duplicate_suppressed" $false ([string]$previous.queueId) "duplicate_single_flight" "duplicate_single_flight_suppressed" continue } $dedupeDecision = "severity_escalation_dispatch" @@ -780,6 +823,7 @@ foreach ($file in $files) { mode = $modeName runtimeWritePerformed = $false } + Write-AgentRelayQueueReceipt $alert $alertId "rejected" $false "" "controlled_dispatch_identity_missing" "identity_rejected" continue } @@ -888,7 +932,10 @@ foreach ($file in $files) { } Save-AgentAlertDedupeState $dedupeEntries + Write-AgentRelayQueueReceipt $alert $alertId "queued" $true $queueId "queue_item_persisted" $dedupeDecision + $queued += [pscustomobject]@{ + queueId = $queueId alertId = $alertId source = $source severity = $severity diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index b35d910eb..1e3df9010 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -16,12 +16,13 @@ $LogDir = Join-Path $AgentRoot "logs" $BinDir = Join-Path $AgentRoot "bin" $QueueDir = Join-Path $AgentRoot "queue" $ProcessedDir = Join-Path $QueueDir "processed" +$RelayReceiptDir = Join-Path $EvidenceDir "relay-dispatch-receipts" $SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" $SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit" $AcceptedIgnoredDir = Join-Path $AlertRoot "ignored" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" -New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null +New-Item -ItemType Directory -Force -Path $EvidenceDir, $RelayReceiptDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null function Convert-AgentSafeFileToken { param([string]$Value) @@ -274,7 +275,7 @@ function Start-AgentSreAlertInbox { return [pscustomobject]@{ started = $false; reason = "sre_alert_inbox_script_missing" } } $ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe" - $stamp = Get-Date -Format "yyyyMMdd-HHmmss" + $stamp = "$(Get-Date -Format 'yyyyMMdd-HHmmss')-$([guid]::NewGuid().ToString('N').Substring(0, 8))" $stdout = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.out" $stderr = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.err" $args = @( @@ -293,6 +294,71 @@ function Start-AgentSreAlertInbox { } } +function Wait-AgentRelayQueueReceipt { + param( + [string]$ReceiptId, + [object]$Alert, + [int]$TimeoutMilliseconds = 1800 + ) + $path = Join-Path $RelayReceiptDir "agent99-sre-queue-receipt-$ReceiptId.json" + $deadline = [DateTime]::UtcNow.AddMilliseconds($TimeoutMilliseconds) + do { + if (Test-Path $path) { + try { + $receipt = Get-Content $path -Raw | ConvertFrom-Json + $alertId = [string](Get-AgentField $Alert "id" "") + if ( + [string](Get-AgentField $receipt "schemaVersion" "") -eq "agent99_sre_queue_receipt_v1" -and + [string](Get-AgentField $receipt "receiptId" "") -eq $ReceiptId -and + [string](Get-AgentField $receipt "alertId" "") -eq $alertId + ) { + $awoooi = Get-AgentField $Alert "awoooi" $null + $expectedIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null + $routing = Get-AgentField $Alert "routing" $null + $expectedRunId = [string](Get-AgentField $expectedIdentity "run_id" (Get-AgentField $routing "runId" "")) + $expectedTraceId = [string](Get-AgentField $expectedIdentity "trace_id" (Get-AgentField $routing "traceId" "")) + $expectedWorkItemId = [string](Get-AgentField $expectedIdentity "work_item_id" (Get-AgentField $routing "workItemId" "")) + $expectedGeneration = [string](Get-AgentField $expectedIdentity "execution_generation" (Get-AgentField $routing "executionGeneration" "")) + $identityMatched = [bool]( + (-not $expectedRunId -or [string](Get-AgentField $receipt "automationRunId" "") -eq $expectedRunId) -and + (-not $expectedTraceId -or [string](Get-AgentField $receipt "traceId" "") -eq $expectedTraceId) -and + (-not $expectedWorkItemId -or [string](Get-AgentField $receipt "workItemId" "") -eq $expectedWorkItemId) -and + (-not $expectedGeneration -or [string](Get-AgentField $receipt "executionGeneration" "") -eq $expectedGeneration) + ) + return [pscustomobject]@{ + found = $true + queueAccepted = [bool](Get-AgentField $receipt "queueAccepted" $false) + status = [string](Get-AgentField $receipt "status" "unknown") + queueId = [string](Get-AgentField $receipt "queueId" "") + reason = [string](Get-AgentField $receipt "reason" "") + dedupeDecision = [string](Get-AgentField $receipt "dedupeDecision" "") + dispatchIdentityMatched = $identityMatched + automationRunId = [string](Get-AgentField $receipt "automationRunId" "") + traceId = [string](Get-AgentField $receipt "traceId" "") + workItemId = [string](Get-AgentField $receipt "workItemId" "") + executionGeneration = [string](Get-AgentField $receipt "executionGeneration" "") + } + } + } catch { + } + } + Start-Sleep -Milliseconds 50 + } while ([DateTime]::UtcNow -lt $deadline) + return [pscustomobject]@{ + found = $false + queueAccepted = $false + status = "receipt_pending" + queueId = "" + reason = "queue_receipt_not_observed_before_deadline" + dedupeDecision = "" + dispatchIdentityMatched = $false + automationRunId = "" + traceId = "" + workItemId = "" + executionGeneration = "" + } +} + function Get-AgentSha256Text { param([string]$Value) $sha = [Security.Cryptography.SHA256]::Create() @@ -1036,6 +1102,11 @@ try { $alert = $requestPayload $alertId = [string](Get-AgentField $alert "id" ("relay-" + [guid]::NewGuid().ToString("N"))) + if (-not $alert.PSObject.Properties["id"]) { + $alert | Add-Member -MemberType NoteProperty -Name "id" -Value $alertId + } + $relayReceiptId = "agent99-relay-" + [guid]::NewGuid().ToString("N") + $alert | Add-Member -MemberType NoteProperty -Name "relayReceiptId" -Value $relayReceiptId -Force $safeId = Convert-AgentSafeFileToken $alertId $fileStamp = Get-Date -Format "yyyyMMdd-HHmmss" $tmpPath = Join-Path $IncomingDir ".$safeId-$fileStamp-$([guid]::NewGuid().ToString("N")).tmp" @@ -1044,21 +1115,54 @@ try { Move-Item -Force $tmpPath $finalPath $trigger = Start-AgentSreAlertInbox + $queueReceipt = if ($trigger.started) { + Wait-AgentRelayQueueReceipt $relayReceiptId $alert + } else { + [pscustomobject]@{ + found = $false + queueAccepted = $false + status = "inbox_not_started" + queueId = "" + reason = [string](Get-AgentField $trigger "reason" "inbox_not_started") + dedupeDecision = "" + dispatchIdentityMatched = $false + automationRunId = "" + traceId = "" + workItemId = "" + executionGeneration = "" + } + } + $queueAccepted = [bool]($queueReceipt.queueAccepted -and $queueReceipt.dispatchIdentityMatched) Send-AgentRelayResponse $context 202 @{ ok = $true alertId = $alertId - incomingFile = $finalPath inboxTriggered = [bool]$trigger.started - inboxProcessId = $trigger.processId + queueAccepted = $queueAccepted + queueReceiptFound = [bool]$queueReceipt.found + queueReceiptId = $relayReceiptId + queueReceiptStatus = [string]$queueReceipt.status + queueId = [string]$queueReceipt.queueId + queueReason = [string]$queueReceipt.reason + dedupeDecision = [string]$queueReceipt.dedupeDecision + dispatchIdentityMatched = [bool]$queueReceipt.dispatchIdentityMatched + automationRunId = [string]$queueReceipt.automationRunId + traceId = [string]$queueReceipt.traceId + workItemId = [string]$queueReceipt.workItemId + executionGeneration = [string]$queueReceipt.executionGeneration + storesRawPayload = $false } Write-AgentRelayEvent @{ event = "relay_accepted" ok = $true remote = $remote alertId = $alertId - incomingFile = $finalPath inboxTriggered = [bool]$trigger.started - inboxProcessId = $trigger.processId + queueAccepted = $queueAccepted + queueReceiptFound = [bool]$queueReceipt.found + queueReceiptId = $relayReceiptId + queueReceiptStatus = [string]$queueReceipt.status + dispatchIdentityMatched = [bool]$queueReceipt.dispatchIdentityMatched + storesRawPayload = $false } } catch { try { diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index e766b502f..b910886e1 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -492,7 +492,12 @@ async def _try_auto_repair_background( ) accepted = bool(dispatch_receipt.get("accepted") is True) inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True) - dispatch_promoted = bool(accepted and inbox_triggered) + dispatch_promoted = bool( + accepted + and inbox_triggered + and dispatch_receipt.get("queue_accepted") is True + and dispatch_receipt.get("dispatch_identity_matched") is True + ) receipt_persisted = bool( correlated_receipt.get("receipt_persisted") is True or ( diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index 16c7f7dae..af334421b 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -311,7 +311,16 @@ def build_agent99_dispatch_receipt_envelope( ) -> dict[str, Any]: accepted = bool(dispatch_receipt.get("accepted") is True) inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True) - dispatch_promoted = bool(accepted and inbox_triggered) + queue_accepted = bool(dispatch_receipt.get("queue_accepted") is True) + dispatch_identity_matched = bool( + dispatch_receipt.get("dispatch_identity_matched") is True + ) + dispatch_promoted = bool( + accepted + and inbox_triggered + and queue_accepted + and dispatch_identity_matched + ) delivery_certainty = str( dispatch_receipt.get("delivery_certainty") or ( @@ -355,7 +364,7 @@ def build_agent99_dispatch_receipt_envelope( "dispatch_receipt": { "schema_version": str( dispatch_receipt.get("schema_version") - or "agent99_sre_dispatch_receipt_v1" + or "agent99_sre_dispatch_receipt_v2" ), "status": str(dispatch_receipt.get("status") or "unknown"), "transport": str(dispatch_receipt.get("transport") or "unknown"), @@ -364,6 +373,24 @@ def build_agent99_dispatch_receipt_envelope( "http_status": int(dispatch_receipt.get("http_status") or 0), "accepted": accepted, "inbox_triggered": inbox_triggered, + "transport_accepted": bool( + dispatch_receipt.get("transport_accepted") is True + ), + "queue_receipt_found": bool( + dispatch_receipt.get("queue_receipt_found") is True + ), + "queue_accepted": queue_accepted, + "queue_receipt_status": str( + dispatch_receipt.get("queue_receipt_status") or "unknown" + ), + "queue_receipt_id": str( + dispatch_receipt.get("queue_receipt_id") or "" + ), + "queue_id": str(dispatch_receipt.get("queue_id") or ""), + "dispatch_identity_matched": dispatch_identity_matched, + "alert_id_matched": bool( + dispatch_receipt.get("alert_id_matched") is True + ), "delivery_certainty": delivery_certainty, "stores_raw_response": False, }, @@ -841,7 +868,12 @@ class PostgresAgent99DispatchLedger: ) accepted = bool(dispatch_receipt.get("accepted") is True) inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True) - dispatch_promoted = bool(accepted and inbox_triggered) + dispatch_promoted = bool( + accepted + and inbox_triggered + and dispatch_receipt.get("queue_accepted") is True + and dispatch_receipt.get("dispatch_identity_matched") is True + ) retry_safe = bool( isinstance(envelope.get("retry_policy"), dict) and envelope["retry_policy"].get("safe_to_retry") is True diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index a0ffeda4c..f94f19c6a 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -856,15 +856,77 @@ def dispatch_agent99_sre_alert_with_receipt( except json.JSONDecodeError: response_payload = {} http_status = int(raw_receipt.get("status") or 0) - accepted = bool(200 <= http_status < 300 and response_payload.get("ok") is True) + transport_accepted = bool( + 200 <= http_status < 300 and response_payload.get("ok") is True + ) inbox_triggered = bool(response_payload.get("inboxTriggered") is True) + queue_receipt_found = bool( + response_payload.get("queueReceiptFound") is True + ) + queue_accepted = bool(response_payload.get("queueAccepted") is True) + response_identity_matched = bool( + response_payload.get("dispatchIdentityMatched") is True + ) + dispatch_identity = ( + awoooi.get("agent99DispatchIdentity") + if isinstance(awoooi.get("agent99DispatchIdentity"), dict) + else {} + ) + identity_pairs = ( + ("run_id", "automationRunId"), + ("trace_id", "traceId"), + ("work_item_id", "workItemId"), + ("execution_generation", "executionGeneration"), + ) + dispatch_identity_matched = bool( + response_identity_matched + and all( + not _safe_text(dispatch_identity.get(source_key), max_length=240) + or _safe_text( + response_payload.get(response_key), max_length=240 + ) + == _safe_text( + dispatch_identity.get(source_key), max_length=240 + ) + for source_key, response_key in identity_pairs + ) + ) + alert_id_matched = bool( + _safe_text(response_payload.get("alertId"), max_length=160) + == alert_id + ) + queue_receipt_status = _safe_text( + response_payload.get("queueReceiptStatus"), max_length=80 + ) + queue_receipt_id = _safe_text( + response_payload.get("queueReceiptId"), max_length=120 + ) + queue_id = _safe_text(response_payload.get("queueId"), max_length=120) + accepted = bool( + transport_accepted + and inbox_triggered + and queue_receipt_found + and queue_accepted + and dispatch_identity_matched + and alert_id_matched + ) + explicitly_rejected = bool( + transport_accepted + and queue_receipt_found + and queue_receipt_status == "rejected" + ) return { - "schema_version": "agent99_sre_dispatch_receipt_v1", + "schema_version": "agent99_sre_dispatch_receipt_v2", "status": ( - "accepted_inbox_triggered" - if accepted and inbox_triggered - else "accepted_inbox_pending" + "accepted_queue_persisted" if accepted + else "transport_accepted_queue_rejected" + if explicitly_rejected + else "transport_accepted_duplicate_reconcile" + if transport_accepted + and queue_receipt_status == "duplicate_suppressed" + else "transport_accepted_queue_receipt_pending" + if transport_accepted else "relay_response_not_accepted" ), "transport": "relay", @@ -874,14 +936,24 @@ def dispatch_agent99_sre_alert_with_receipt( "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, "http_status": http_status, + "transport_accepted": transport_accepted, "accepted": accepted, "inbox_triggered": inbox_triggered, + "queue_receipt_found": queue_receipt_found, + "queue_accepted": queue_accepted, + "queue_receipt_status": queue_receipt_status, + "queue_receipt_id": queue_receipt_id, + "queue_id": queue_id, + "dispatch_identity_matched": dispatch_identity_matched, + "alert_id_matched": alert_id_matched, "delivery_certainty": ( "delivered" - if accepted and inbox_triggered - else "unknown" if accepted else "not_delivered" + if explicitly_rejected + else "unknown" + if transport_accepted + else "not_delivered" ), "stores_raw_response": False, } @@ -890,7 +962,7 @@ def dispatch_agent99_sre_alert_with_receipt( written = write_agent99_sre_alert(payload) if written is not None: return { - "schema_version": "agent99_sre_dispatch_receipt_v1", + "schema_version": "agent99_sre_dispatch_receipt_v2", "status": "file_written", "transport": "file", "alert_id": alert_id, @@ -898,8 +970,16 @@ def dispatch_agent99_sre_alert_with_receipt( "target_resource": target_resource, "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, - "accepted": True, + "transport_accepted": True, + "accepted": False, "inbox_triggered": False, + "queue_receipt_found": False, + "queue_accepted": False, + "queue_receipt_status": "receipt_pending", + "queue_receipt_id": "", + "queue_id": "", + "dispatch_identity_matched": False, + "alert_id_matched": False, # A file write is not proof that the Agent99 inbox consumed it. "delivery_certainty": "unknown", "stores_raw_response": False, @@ -912,7 +992,7 @@ def dispatch_agent99_sre_alert_with_receipt( kind=payload.get("kind"), ) return { - "schema_version": "agent99_sre_dispatch_receipt_v1", + "schema_version": "agent99_sre_dispatch_receipt_v2", "status": "bridge_disabled", "transport": "disabled", "alert_id": alert_id, @@ -920,8 +1000,16 @@ def dispatch_agent99_sre_alert_with_receipt( "target_resource": target_resource, "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, + "transport_accepted": False, "accepted": False, "inbox_triggered": False, + "queue_receipt_found": False, + "queue_accepted": False, + "queue_receipt_status": "not_dispatched", + "queue_receipt_id": "", + "queue_id": "", + "dispatch_identity_matched": False, + "alert_id_matched": False, "delivery_certainty": "not_delivered", "stores_raw_response": False, } @@ -1091,7 +1179,12 @@ async def bridge_alertmanager_to_agent99( inbox_triggered = bool( dispatch_receipt.get("inbox_triggered") is True ) - dispatch_promoted = bool(accepted and inbox_triggered) + dispatch_promoted = bool( + accepted + and inbox_triggered + and dispatch_receipt.get("queue_accepted") is True + and dispatch_receipt.get("dispatch_identity_matched") is True + ) reconcile_only = bool( retry_policy.get("reconcile_only") is True or reservation.get("status") == "idempotent_running" @@ -1199,7 +1292,12 @@ async def bridge_alertmanager_to_agent99( ) accepted = bool(dispatch_receipt.get("accepted") is True) inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True) - dispatch_promoted = bool(accepted and inbox_triggered) + dispatch_promoted = bool( + accepted + and inbox_triggered + and dispatch_receipt.get("queue_accepted") is True + and dispatch_receipt.get("dispatch_identity_matched") is True + ) delivery_certainty = str( dispatch_receipt.get("delivery_certainty") or "unknown" ) diff --git a/apps/api/src/services/backup_restore_signal_automation.py b/apps/api/src/services/backup_restore_signal_automation.py index bcd0523f9..cc844095d 100644 --- a/apps/api/src/services/backup_restore_signal_automation.py +++ b/apps/api/src/services/backup_restore_signal_automation.py @@ -196,6 +196,8 @@ def _normalized_agent99_dispatch_receipt( "dispatched": False, "accepted": False, "inbox_triggered": False, + "queue_accepted": False, + "dispatch_identity_matched": False, "backupcheck_route": False, "receipt_persisted": False, "identity_complete": False, @@ -224,6 +226,10 @@ def _normalized_agent99_dispatch_receipt( receipt.get("inbox_triggered") is True or receipt.get("inboxTriggered") is True ) + queue_accepted = receipt.get("queue_accepted") is True + dispatch_identity_matched = ( + receipt.get("dispatch_identity_matched") is True + ) kind = str(receipt.get("kind") or "") suggested_mode = str( receipt.get("suggested_mode") or receipt.get("suggestedMode") or "" @@ -325,6 +331,8 @@ def _normalized_agent99_dispatch_receipt( and identity_binding_valid and accepted and inbox_triggered + and queue_accepted + and dispatch_identity_matched and backupcheck_route ) if not valid_schema: @@ -345,8 +353,13 @@ def _normalized_agent99_dispatch_receipt( elif not identity_binding_valid: status = "current_ledger_identity_mismatch_fail_closed" next_action = "repair_agent99_run_trace_work_item_binding" - elif not accepted or not inbox_triggered: - status = "dispatch_not_accepted_or_inbox_not_triggered" + elif ( + not accepted + or not inbox_triggered + or not queue_accepted + or not dispatch_identity_matched + ): + status = "dispatch_queue_receipt_not_verified" next_action = "retry_idempotent_agent99_backupcheck_dispatch" elif not backupcheck_route: status = "dispatch_route_mismatch_fail_closed" @@ -361,7 +374,7 @@ def _normalized_agent99_dispatch_receipt( status = "verifier_failed_no_runtime_closure" next_action = "collect_failed_verifier_evidence_then_retry_bounded_readback" else: - status = "accepted_inbox_triggered" + status = "accepted_queue_persisted" next_action = "ingest_agent99_backupcheck_outcome_and_independent_dr_verifier" return { @@ -372,6 +385,8 @@ def _normalized_agent99_dispatch_receipt( "dispatched": dispatched, "accepted": accepted, "inbox_triggered": inbox_triggered, + "queue_accepted": queue_accepted, + "dispatch_identity_matched": dispatch_identity_matched, "backupcheck_route": backupcheck_route, "receipt_persisted": receipt_persisted, "identity_complete": identity_complete, diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index f80c0e5e0..e891da881 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -2485,6 +2485,8 @@ def _agent99_current_dispatch_projection( "suggested_mode": suggested_mode, "accepted": False, "inbox_triggered": False, + "queue_accepted": False, + "dispatch_identity_matched": False, "incident_id": incident_id or None, "run_id": run_id or None, "trace_id": trace_id or None, @@ -2512,6 +2514,13 @@ def _agent99_current_dispatch_projection( and dispatch.get("accepted") is True ), "inbox_triggered": bool(dispatch.get("inbox_triggered") is True), + "queue_accepted": bool(dispatch.get("queue_accepted") is True), + "queue_receipt_id": str(dispatch.get("queue_receipt_id") or ""), + "queue_id": str(dispatch.get("queue_id") or ""), + "dispatch_identity_matched": bool( + dispatch.get("dispatch_identity_matched") is True + ), + "alert_id_matched": bool(dispatch.get("alert_id_matched") is True), "incident_id": incident_id, "run_id": run_id, "trace_id": trace_id, diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 539aaa88e..292409548 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -2812,6 +2812,8 @@ async def _mirror_ai_automation_alert_card_to_agent99( "suggested_mode": str(payload.get("suggestedMode") or "Status"), "accepted": False, "inbox_triggered": False, + "queue_accepted": False, + "dispatch_identity_matched": False, "dispatch_performed": False, "runtime_closure_verified": False, } @@ -2854,6 +2856,19 @@ async def _mirror_ai_automation_alert_card_to_agent99( "suggested_mode": str(payload.get("suggestedMode") or "Status"), "accepted": bool(dispatch_receipt.get("accepted") is True), "inbox_triggered": bool(dispatch_receipt.get("inbox_triggered") is True), + "queue_accepted": bool( + dispatch_receipt.get("queue_accepted") is True + ), + "queue_receipt_id": str( + dispatch_receipt.get("queue_receipt_id") or "" + ), + "queue_id": str(dispatch_receipt.get("queue_id") or ""), + "dispatch_identity_matched": bool( + dispatch_receipt.get("dispatch_identity_matched") is True + ), + "alert_id_matched": bool( + dispatch_receipt.get("alert_id_matched") is True + ), "incident_id": incident_id, "commit_sha": str(awoooi.get("commitSha") or ""), "run_id": str(identity.get("run_id") or ""), diff --git a/apps/api/tests/test_agent99_controlled_dispatch_p1.py b/apps/api/tests/test_agent99_controlled_dispatch_p1.py index eeea4cd7d..ca3e979b8 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_p1.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_p1.py @@ -802,10 +802,12 @@ async def test_safe_failed_generation_advances_bounded_and_keeps_one_flight_key( monkeypatch.setattr( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda _payload: { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "delivery_certainty": "delivered", }, ) diff --git a/apps/api/tests/test_agent99_dispatch_receipt_projection.py b/apps/api/tests/test_agent99_dispatch_receipt_projection.py index d7b292828..870dc27aa 100644 --- a/apps/api/tests/test_agent99_dispatch_receipt_projection.py +++ b/apps/api/tests/test_agent99_dispatch_receipt_projection.py @@ -21,10 +21,12 @@ async def test_telegram_projects_correlated_receipt_without_dispatch( "idempotency_key": "agent99-controlled:stable-key", }, "dispatch_receipt": { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, "receipt_persisted": True, "runtime_execution_authorized": False, diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index 0f6d461ce..7a9ad5c6d 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -333,7 +333,7 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: assert seen["headers"]["X-agent99-relay-token"] == "relay-token" -def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -> None: +def test_dispatch_receipt_requires_queue_receipt_without_raw_body(monkeypatch) -> None: payload = build_agent99_sre_alert( alert_id="receipt-smoke", alertname="ColdStartGateBlocked", @@ -356,24 +356,64 @@ def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) - receipt = dispatch_agent99_sre_alert_with_receipt(payload) - assert receipt == { - "schema_version": "agent99_sre_dispatch_receipt_v1", - "status": "accepted_inbox_triggered", - "transport": "relay", - "alert_id": "awoooi-alertmanager-receipt-smoke", - "kind": "host_recovery", - "target_resource": "cold-start-gate", - "suggested_mode": "Recover", - "controlled_apply_requested": True, - "http_status": 202, - "accepted": True, - "inbox_triggered": True, - "delivery_certainty": "delivered", - "stores_raw_response": False, - } + assert receipt["schema_version"] == "agent99_sre_dispatch_receipt_v2" + assert receipt["status"] == "transport_accepted_queue_receipt_pending" + assert receipt["transport_accepted"] is True + assert receipt["accepted"] is False + assert receipt["inbox_triggered"] is True + assert receipt["queue_receipt_found"] is False + assert receipt["queue_accepted"] is False + assert receipt["dispatch_identity_matched"] is False + assert receipt["delivery_certainty"] == "unknown" + assert receipt["stores_raw_response"] is False assert "private-path" not in str(receipt) +def test_dispatch_receipt_promotes_only_persisted_identity_bound_queue_item( + monkeypatch, +) -> None: + payload = build_agent99_sre_alert( + alert_id="receipt-queued", + alertname="ColdStartGateBlocked", + severity="critical", + namespace="awoooi-prod", + target_resource="cold-start-gate", + message="cold-start gate blocked", + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL", + "http://192.168.0.99:8787/agent99/sre-alert/", + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.post_agent99_sre_alert", + lambda _payload: { + "status": 202, + "body": json.dumps( + { + "ok": True, + "alertId": "awoooi-alertmanager-receipt-queued", + "inboxTriggered": True, + "queueReceiptFound": True, + "queueAccepted": True, + "queueReceiptStatus": "queued", + "queueReceiptId": "agent99-relay-" + "a" * 32, + "queueId": "sre-alert-receipt-queued", + "dispatchIdentityMatched": True, + } + ), + }, + ) + + receipt = dispatch_agent99_sre_alert_with_receipt(payload) + + assert receipt["status"] == "accepted_queue_persisted" + assert receipt["accepted"] is True + assert receipt["queue_accepted"] is True + assert receipt["dispatch_identity_matched"] is True + assert receipt["alert_id_matched"] is True + assert receipt["delivery_certainty"] == "delivered" + + def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None: key = agent99_sre_single_flight_key("private/source/fingerprint") @@ -395,10 +435,12 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, ) @@ -431,10 +473,12 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, ) @@ -708,12 +752,14 @@ async def test_mutating_dispatch_is_deduplicated_by_postgres_identity( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, ) common = { @@ -832,13 +878,15 @@ async def test_backupcheck_read_only_dispatch_has_durable_same_run_receipt( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "schema_version": "agent99_sre_dispatch_receipt_v1", - "status": "accepted_inbox_triggered", + "schema_version": "agent99_sre_dispatch_receipt_v2", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "delivery_certainty": "delivered", }, ) @@ -938,12 +986,14 @@ async def test_accepted_transport_with_receipt_write_failure_is_not_replayed( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "delivery_certainty": "delivered", }, ) @@ -1007,12 +1057,14 @@ async def test_safe_not_delivered_failure_advances_generation_before_dispatch( "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", lambda payload: dispatched.append(payload) or { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "delivery_certainty": "delivered", }, ) diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index cd2d185d6..33f6a1bc9 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -1307,13 +1307,15 @@ def test_backup_item_consumes_agent99_dispatch_receipt_without_faking_outcome() }, "agent99_dispatch_receipt": { "schema_version": "telegram_agent99_dispatch_receipt_projection_v1", - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": "backup-alert-1", "kind": "backup_health", "suggested_mode": "BackupCheck", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "receipt_persisted": True, "runtime_closure_verified": False, "run_id": "agent99-run-1", @@ -1387,13 +1389,15 @@ def test_backup_item_consumes_real_bridge_and_ledger_identity() -> None: current_receipt = build_agent99_dispatch_receipt_envelope( identity=identity, dispatch_receipt={ - "schema_version": "agent99_sre_dispatch_receipt_v1", - "status": "accepted", + "schema_version": "agent99_sre_dispatch_receipt_v2", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(alert["id"]), "kind": "backup_health", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, ) public_identity = identity.public_dict() @@ -1417,6 +1421,8 @@ def test_backup_item_consumes_real_bridge_and_ledger_identity() -> None: "suggested_mode": "BackupCheck", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "incident_id": "INC-BACKUP-REAL", "run_id": public_identity["run_id"], "trace_id": public_identity["trace_id"], @@ -1495,6 +1501,8 @@ def test_backup_item_advances_from_current_durable_ledger_terminal() -> None: "suggested_mode": "BackupCheck", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "incident_id": "INC-BACKUP", **stage_identity, "receipt_persisted": True, @@ -1514,6 +1522,8 @@ def test_backup_item_advances_from_current_durable_ledger_terminal() -> None: "kind": "backup_health", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, "dispatch_accepted": True, "post_verifier_passed": True, diff --git a/apps/api/tests/test_backup_restore_alertmanager_ingress.py b/apps/api/tests/test_backup_restore_alertmanager_ingress.py index 9b12ecc0b..d63468c08 100644 --- a/apps/api/tests/test_backup_restore_alertmanager_ingress.py +++ b/apps/api/tests/test_backup_restore_alertmanager_ingress.py @@ -774,13 +774,15 @@ async def test_backup_bridge_postgres_identity_deduplicates_transport( lambda payload: ( transported.append(payload) or { - "schema_version": "agent99_sre_dispatch_receipt_v1", - "status": "accepted_inbox_triggered", + "schema_version": "agent99_sre_dispatch_receipt_v2", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": str(payload["id"]), "kind": str(payload["kind"]), "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "delivery_certainty": "delivered", } ), diff --git a/apps/api/tests/test_backup_restore_signal_automation.py b/apps/api/tests/test_backup_restore_signal_automation.py index a5656c1f4..a6f3447fa 100644 --- a/apps/api/tests/test_backup_restore_signal_automation.py +++ b/apps/api/tests/test_backup_restore_signal_automation.py @@ -154,13 +154,15 @@ def test_backup_restore_work_item_clusters_recurrence_but_receipts_stay_unique() def test_backup_dispatch_requires_accepted_triggered_backupcheck_receipt() -> None: receipt = { "schema_version": "telegram_agent99_dispatch_receipt_projection_v1", - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "transport": "relay", "alert_id": "awoooi-backup-alert", "kind": "backup_health", "suggested_mode": "BackupCheck", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, "receipt_persisted": True, "runtime_closure_verified": False, "run_id": "agent99-run-123", @@ -178,7 +180,7 @@ def test_backup_dispatch_requires_accepted_triggered_backupcheck_receipt() -> No ) dispatch = contract["candidate"]["agent99_dispatch"] - assert dispatch["status"] == "accepted_inbox_triggered" + assert dispatch["status"] == "accepted_queue_persisted" assert dispatch["dispatched"] is True assert dispatch["receipt_persisted"] is True assert dispatch["identity_complete"] is True diff --git a/apps/api/tests/test_controlled_alert_target_router.py b/apps/api/tests/test_controlled_alert_target_router.py index 3496425a0..9de0cd5b9 100644 --- a/apps/api/tests/test_controlled_alert_target_router.py +++ b/apps/api/tests/test_controlled_alert_target_router.py @@ -179,9 +179,11 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start( "idempotency_key": "agent99-controlled:stable-key", }, "dispatchReceipt": { - "status": "accepted_inbox_triggered", + "status": "accepted_queue_persisted", "accepted": True, "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, }, "correlatedReceipt": { "status": "dispatch_accepted_verifier_pending", diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index b668a3df8..dc6f54d6f 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -209,6 +209,26 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> assert forbidden not in preflight +def test_agent99_relay_requires_identity_bound_queue_receipt_before_promotion() -> None: + relay = read("agent99-sre-alert-relay.ps1") + inbox = read("agent99-sre-alert-inbox.ps1") + + assert 'schemaVersion = "agent99_sre_queue_receipt_v1"' in inbox + assert 'queueAccepted = $QueueAccepted' in inbox + assert 'storesRawPayload = $false' in inbox + assert 'Write-AgentRelayQueueReceipt $alert $alertId "queued" $true' in inbox + assert 'Write-AgentRelayQueueReceipt $alert $alertId "duplicate_suppressed" $false' in inbox + assert 'Wait-AgentRelayQueueReceipt $relayReceiptId $alert' in relay + assert '$queueReceipt.queueAccepted -and $queueReceipt.dispatchIdentityMatched' in relay + assert 'queueReceiptFound = [bool]$queueReceipt.found' in relay + assert 'dispatchIdentityMatched = [bool]$queueReceipt.dispatchIdentityMatched' in relay + response_block = relay.split("Send-AgentRelayResponse $context 202 @{", 1)[1] + response_block = response_block.split("Write-AgentRelayEvent", 1)[0] + assert "incomingFile" not in response_block + assert "inboxProcessId" not in response_block + assert "storesRawPayload = $false" in response_block + + def test_recover_receipt_readback_requires_verifier_and_notification() -> None: readback = read("scripts/reboot-recovery/agent99-recover-receipt-readback.ps1")