diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml
index d0b9dfb52..8e644c009 100644
--- a/.gitea/workflows/cd.yaml
+++ b/.gitea/workflows/cd.yaml
@@ -2877,17 +2877,36 @@ jobs:
import socket
import sys
- reachable = []
+ connected = []
+ refused = []
+ permitted = []
for host in sys.argv[1:]:
try:
connection = socket.create_connection((host, 22), timeout=1.0)
+ except ConnectionRefusedError:
+ # A TCP refusal proves that the packet crossed the egress
+ # NetworkPolicy boundary and the target replied. Keep host
+ # service readiness separate from this security-boundary
+ # verifier; a stopped sshd must not make a healthy rollout
+ # look like a policy failure.
+ refused.append(host)
+ permitted.append(host)
except OSError:
continue
else:
connection.close()
- reachable.append(host)
- print("broker_ssh_reachable=" + ",".join(reachable))
- raise SystemExit(0 if len(reachable) == len(sys.argv) - 1 else 1)
+ connected.append(host)
+ permitted.append(host)
+ print("broker_ssh_connected=" + ",".join(connected))
+ print(
+ "broker_ssh_refused_but_egress_permitted="
+ + ",".join(refused)
+ )
+ raise SystemExit(
+ 0
+ if connected and len(permitted) == len(sys.argv) - 1
+ else 1
+ )
PY
}
verify_ssh_denied awoooi-api api || {
diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1
index fedc1d480..22edfae5e 100644
--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -6,6 +6,7 @@ param(
[switch]$SelfTestTelegramDelivery,
[switch]$SelfTestSensorGate,
[switch]$SelfTestOutcomeContract,
+ [switch]$SelfTestBackupReadbackContract,
[switch]$SuppressAlerts,
[string]$SuppressReason = "",
[string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json",
@@ -53,6 +54,71 @@ function Convert-AgentBool {
return [bool]($text -match "^(?i:true|1|yes|y|on)$")
}
+function Convert-AgentPublicReceiptLeaf {
+ param([object]$Value)
+ $text = ([string]$Value).Trim()
+ if (-not $text) { return "missing" }
+ $leaf = [regex]::Replace($text, "^.*[\\/]", "")
+ $safe = [regex]::Replace($leaf, "[^A-Za-z0-9._:@+%-]", "-").Trim(".-")
+ if (-not $safe) { return "missing" }
+ if ($safe.Length -gt 180) { return $safe.Substring(0, 180) }
+ return $safe
+}
+
+function Send-AgentOutcomeWriteback {
+ param(
+ [object]$Result,
+ [string]$PostVerifierEvidenceRef,
+ [string]$SourceEventEvidenceRef
+ )
+ $configured = if ($Config.PSObject.Properties["outcomeWriteback"]) { $Config.outcomeWriteback } else { $null }
+ $enabled = if ($configured -and $configured.PSObject.Properties["enabled"]) { Convert-AgentBool $configured.enabled } else { $true }
+ if (-not $enabled) {
+ return [pscustomobject]@{ ok = $false; status = "outcome_writeback_disabled"; attempts = 0 }
+ }
+ $url = if ($configured -and $configured.PSObject.Properties["url"] -and $configured.url) { [string]$configured.url } else { "https://awoooi.wooo.work/api/v1/webhooks/agent99/outcomes" }
+ $tokenEnv = if ($configured -and $configured.PSObject.Properties["tokenEnv"] -and $configured.tokenEnv) { [string]$configured.tokenEnv } else { "AGENT99_SRE_RELAY_TOKEN" }
+ $token = [Environment]::GetEnvironmentVariable($tokenEnv)
+ if (-not $token) {
+ return [pscustomobject]@{ ok = $false; status = "outcome_writeback_token_missing"; attempts = 0 }
+ }
+ if (-not $Result.identity -or -not $Result.outcome) {
+ return [pscustomobject]@{ ok = $false; status = "outcome_writeback_identity_or_outcome_missing"; attempts = 0 }
+ }
+ $body = [pscustomobject]@{
+ identity = $Result.identity
+ outcome_receipt = [pscustomobject]@{
+ identity = $Result.identity
+ mode = $Result.mode
+ controlledApply = [bool]$Result.controlledApply
+ transportOk = [bool]$Result.transportOk
+ outcome = $Result.outcome
+ }
+ evidence_refs = [pscustomobject]@{
+ agent99_outcome_receipt_id = "agent99-outcome:$($Result.id)"
+ post_verifier_evidence_ref = $PostVerifierEvidenceRef
+ source_event_evidence_ref = $SourceEventEvidenceRef
+ }
+ learning_receipt_refs = [pscustomobject]@{}
+ } | ConvertTo-Json -Depth 12 -Compress
+ $lastStatus = "outcome_writeback_failed"
+ for ($attempt = 1; $attempt -le 3; $attempt++) {
+ try {
+ $response = Invoke-RestMethod -Method Post -Uri $url -Headers @{ "X-Agent99-Relay-Token" = $token } -ContentType "application/json" -Body $body -TimeoutSec 20
+ return [pscustomobject]@{
+ ok = [bool]($response -and $response.accepted -eq $true)
+ status = if ($response -and $response.status) { [string]$response.status } else { "outcome_writeback_acknowledged" }
+ attempts = $attempt
+ runtimeClosureVerified = [bool]($response -and $response.runtime_closure_verified -eq $true)
+ }
+ } catch {
+ $lastStatus = "outcome_writeback_http_error"
+ if ($attempt -lt 3) { Start-Sleep -Seconds 2 }
+ }
+ }
+ [pscustomobject]@{ ok = $false; status = $lastStatus; attempts = 3; runtimeClosureVerified = $false }
+}
+
function Get-HostSshUser {
param([string]$TargetHost)
if ($Config.sshUsers -and $Config.sshUsers.PSObject.Properties[$TargetHost]) {
@@ -3292,6 +3358,151 @@ function Get-AgentBackupHealthConfig {
targets = if ($backupHealth -and $backupHealth.targets) { @($backupHealth.targets) } else { $targets }
fileChecks = if ($backupHealth -and $backupHealth.fileChecks) { @($backupHealth.fileChecks) } else { $fileChecks }
cronPatterns = if ($backupHealth -and $backupHealth.cronPatterns) { @($backupHealth.cronPatterns) } else { $cronPatterns }
+ protectionMetricsPath = if ($backupHealth -and $backupHealth.protectionMetricsPath) { [string]$backupHealth.protectionMetricsPath } else { "/home/wooo/node_exporter_textfiles/backup_health.prom" }
+ protectionReadTimeoutSeconds = if ($backupHealth -and $backupHealth.protectionReadTimeoutSeconds) { [int]$backupHealth.protectionReadTimeoutSeconds } else { 45 }
+ protectionMaxAgeMinutes = if ($backupHealth -and $backupHealth.protectionMaxAgeMinutes) { [double]$backupHealth.protectionMaxAgeMinutes } else { 60 }
+ }
+}
+
+function Get-AgentBackupMetricValues {
+ param([string]$Output, [string]$MetricName)
+ $values = @()
+ $pattern = "(?m)^" + [regex]::Escape($MetricName) + "(?:\{[^}]*\})?\s+(-?[0-9]+(?:\.[0-9]+)?)\s*$"
+ foreach ($match in [regex]::Matches([string]$Output, $pattern)) {
+ $parsed = 0.0
+ if ([double]::TryParse($match.Groups[1].Value, [Globalization.NumberStyles]::Float, [Globalization.CultureInfo]::InvariantCulture, [ref]$parsed)) {
+ $values += $parsed
+ }
+ }
+ @($values)
+}
+
+function Convert-AgentBackupProtectionReadback {
+ param(
+ [object]$Readback,
+ [double]$MaxAgeMinutes = 60,
+ [double]$NowUnix = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds()
+ )
+
+ $readback = $Readback
+ $mtimeMatch = [regex]::Match([string]$readback.output, "__PROTECTION_MTIME__=([0-9]+)")
+ $mtime = if ($mtimeMatch.Success) { $mtimeMatch.Groups[1].Value } else { "" }
+ $evidenceAgeMinutes = if ($mtime) { ($NowUnix - [double]$mtime) / 60 } else { $null }
+ $evidenceFresh = [bool]($null -ne $evidenceAgeMinutes -and $evidenceAgeMinutes -ge -5 -and $evidenceAgeMinutes -le [math]::Max(1, $MaxAgeMinutes))
+ $configured = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_configured")
+ $offsiteFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_fresh")
+ $remoteVerify = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_remote_verify_ok")
+ $escrowFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_credential_escrow_fresh")
+ $escrowMissingValues = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_dr_credential_escrow_missing_count")
+ $escrowMissing = if ($escrowMissingValues.Count -gt 0 -and @($escrowMissingValues | Where-Object { $_ -lt 0 }).Count -eq 0) { [int](($escrowMissingValues | Measure-Object -Sum).Sum) } else { -1 }
+ $offsiteOk = [bool](
+ $readback.ok -and $mtime -and $evidenceFresh -and
+ $configured.Count -gt 0 -and @($configured | Where-Object { $_ -ge 1 }).Count -eq $configured.Count -and
+ $offsiteFresh.Count -gt 0 -and @($offsiteFresh | Where-Object { $_ -ge 1 }).Count -eq $offsiteFresh.Count -and
+ $remoteVerify.Count -gt 0 -and @($remoteVerify | Where-Object { $_ -ge 1 }).Count -eq $remoteVerify.Count
+ )
+ $escrowOk = [bool](
+ $readback.ok -and $mtime -and $evidenceFresh -and $escrowMissing -eq 0 -and
+ $escrowFresh.Count -gt 0 -and @($escrowFresh | Where-Object { $_ -ge 1 }).Count -eq $escrowFresh.Count
+ )
+ [pscustomobject]@{
+ offsiteVerify = [pscustomobject]@{
+ ok = $offsiteOk
+ receiptId = if ($offsiteOk) { "offsite-verify:$mtime" } else { $null }
+ reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } else { "offsite_verify_not_green" }
+ configuredMetricCount = $configured.Count
+ freshMetricCount = $offsiteFresh.Count
+ remoteVerifyMetricCount = $remoteVerify.Count
+ evidenceFresh = $evidenceFresh
+ evidenceAgeMinutes = $evidenceAgeMinutes
+ maxAgeMinutes = $MaxAgeMinutes
+ }
+ escrow = [pscustomobject]@{
+ ok = $escrowOk
+ missingCount = $escrowMissing
+ receiptId = if ($escrowOk) { "escrow-status:$mtime:missing-0" } else { $null }
+ reason = if ($escrowOk) { "non_secret_metrics_verified" } elseif ($escrowMissing -lt 0) { "escrow_missing_metric_absent" } else { "escrow_evidence_incomplete" }
+ freshMetricCount = $escrowFresh.Count
+ evidenceFresh = $evidenceFresh
+ evidenceAgeMinutes = $evidenceAgeMinutes
+ maxAgeMinutes = $MaxAgeMinutes
+ }
+ evidenceMtime = $mtime
+ route = $readback.route
+ exitCode = $readback.exitCode
+ readOnly = $true
+ backupRunPerformed = $false
+ restoreRunPerformed = $false
+ markerWritePerformed = $false
+ }
+}
+
+function Test-AgentBackupProtectionReceipts {
+ param([object]$BackupConfig)
+
+ $hostIp = [string]$BackupConfig.host
+ $path = [string]$BackupConfig.protectionMetricsPath
+ $timeoutSeconds = [math]::Min(120, [math]::Max(5, [int]$BackupConfig.protectionReadTimeoutSeconds))
+ $maxAgeMinutes = [math]::Min(1440, [math]::Max(1, [double]$BackupConfig.protectionMaxAgeMinutes))
+ if (-not $path -or $path -notmatch "^/[A-Za-z0-9_.\/-]+$" -or $path -match "\.\.") {
+ return [pscustomobject]@{
+ offsiteVerify = [pscustomobject]@{ ok = $false; receiptId = $null; reason = "invalid_protection_metrics_path" }
+ escrow = [pscustomobject]@{ ok = $false; missingCount = -1; receiptId = $null; reason = "invalid_protection_metrics_path" }
+ readOnly = $true
+ backupRunPerformed = $false
+ restoreRunPerformed = $false
+ markerWritePerformed = $false
+ }
+ }
+
+ $quotedPath = Quote-ShSingle $path
+ $metricPattern = "awoooi_backup_(offsite_configured|offsite_fresh|offsite_remote_verify_ok|credential_escrow_fresh|dr_credential_escrow_missing_count)"
+ $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi"
+ $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1
+ Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes
+}
+
+function Invoke-AgentBackupReadbackContractSelfTest {
+ $fixture = [pscustomobject]@{
+ ok = $true
+ route = "selftest-read-only"
+ exitCode = 0
+ output = @"
+__PROTECTION_MTIME__=1783785600
+awoooi_backup_offsite_configured{host="110",provider="rclone"} 1
+awoooi_backup_offsite_fresh{host="110",provider="rclone"} 1
+awoooi_backup_offsite_remote_verify_ok{host="110",provider="rclone"} 1
+awoooi_backup_credential_escrow_fresh{host="110",item="redacted-id"} 1
+awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0
+"@
+ }
+ $result = Convert-AgentBackupProtectionReadback $fixture 60 1783785660
+ $staleResult = Convert-AgentBackupProtectionReadback $fixture 60 1783792800
+ $partialFixture = [pscustomobject]@{
+ ok = $true
+ route = "selftest-read-only"
+ exitCode = 0
+ output = $fixture.output -replace 'awoooi_backup_offsite_remote_verify_ok\{host="110",provider="rclone"\} 1', 'awoooi_backup_offsite_remote_verify_ok{host="110",provider="rclone"} 0'
+ }
+ $partialResult = Convert-AgentBackupProtectionReadback $partialFixture 60 1783785660
+ $ok = [bool](
+ $result.offsiteVerify.ok -and
+ $result.offsiteVerify.receiptId -eq "offsite-verify:1783785600" -and
+ $result.escrow.ok -and
+ $result.escrow.missingCount -eq 0 -and
+ $result.escrow.receiptId -eq "escrow-status:1783785600:missing-0" -and
+ $result.readOnly -and
+ -not $staleResult.offsiteVerify.ok -and
+ -not $staleResult.escrow.ok -and
+ -not $partialResult.offsiteVerify.ok -and
+ -not $result.backupRunPerformed -and
+ -not $result.restoreRunPerformed -and
+ -not $result.markerWritePerformed
+ )
+ [pscustomobject]@{
+ schemaVersion = "agent99_backup_readback_contract_selftest_v1"
+ ok = $ok
+ result = $result
}
}
@@ -3549,6 +3760,7 @@ function Test-AgentBackupHealth {
}
$cron = Test-AgentBackupCronPatterns @($backupConfig.cronPatterns) $backupConfig.host
+ $protection = Test-AgentBackupProtectionReceipts $backupConfig
$targetCritical = @($targets | Where-Object { $_.severity -eq "critical" }).Count
$targetWarning = @($targets | Where-Object { $_.severity -eq "warning" }).Count
@@ -3556,7 +3768,8 @@ function Test-AgentBackupHealth {
$fileWarning = @($files | Where-Object { $_.severity -eq "warning" }).Count
$cronCritical = @($cron | Where-Object { $_.severity -eq "critical" }).Count
$cronWarning = @($cron | Where-Object { $_.severity -eq "warning" }).Count
- $critical = $targetCritical + $fileCritical + $cronCritical
+ $protectionCritical = @(@($protection.offsiteVerify, $protection.escrow) | Where-Object { -not $_.ok }).Count
+ $critical = $targetCritical + $fileCritical + $cronCritical + $protectionCritical
$warning = $targetWarning + $fileWarning + $cronWarning
$severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" }
@@ -3567,6 +3780,7 @@ function Test-AgentBackupHealth {
fileWarning = $fileWarning
cronCritical = $cronCritical
cronWarning = $cronWarning
+ protectionCritical = $protectionCritical
targetCount = @($targets).Count
fileCheckCount = @($files).Count
cronPatternCount = @($cron).Count
@@ -3584,6 +3798,9 @@ function Test-AgentBackupHealth {
targets = $targets
files = $files
cron = $cron
+ offsiteVerify = $protection.offsiteVerify
+ escrow = $protection.escrow
+ protectionReadback = $protection
summary = $summary
}
}
@@ -3672,6 +3889,7 @@ function Get-AgentOutcomeContract {
$modeVerifierPassed = $false
$successState = "resolved"
$failureState = "degraded"
+ $modeEvidenceRefs = @{}
if ($data) {
switch ($ModeName) {
@@ -3770,9 +3988,27 @@ function Get-AgentOutcomeContract {
$modeVerifierPassed = $selfOk
}
"BackupCheck" {
- $backupOk = [bool]($data.backupHealth -and $data.backupHealth.severity -eq "ok")
- $checks += New-AgentOutcomeCheck "backup_health_ok" $backupOk
- $modeVerifierPassed = $backupOk
+ $backup = $data.backupHealth
+ $backupStatusRow = @($backup.files | Where-Object { $_.name -eq "backup_status_log" -and $_.ok } | Select-Object -First 1)
+ $requiredTargets = @($backup.targets | Where-Object { $_.required })
+ $requiredFiles = @($backup.files | Where-Object { $_.required })
+ $backupStatusOk = [bool]($backup -and $backup.severity -eq "ok" -and $backupStatusRow.Count -gt 0)
+ $freshnessOk = [bool]($requiredTargets.Count -gt 0 -and @($requiredTargets | Where-Object { -not $_.ok -or $null -eq $_.ageMinutes -or $_.ageMinutes -gt $_.maxAgeMinutes }).Count -eq 0 -and @($requiredFiles | Where-Object { -not $_.ok -or $null -eq $_.ageMinutes -or $_.ageMinutes -gt $_.maxAgeMinutes }).Count -eq 0)
+ $offsiteOk = [bool]($backup -and $backup.PSObject.Properties["offsiteVerify"] -and $backup.offsiteVerify -and $backup.offsiteVerify.ok -eq $true -and $backup.offsiteVerify.receiptId)
+ $escrowOk = [bool]($backup -and $backup.PSObject.Properties["escrow"] -and $backup.escrow -and $backup.escrow.ok -eq $true -and [int]$backup.escrow.missingCount -eq 0 -and $backup.escrow.receiptId)
+ $restoreRow = @($backup.files | Where-Object { $_.name -eq "restore_drill_status" -and $_.ok } | Select-Object -First 1)
+ $restoreOk = [bool]($restoreRow.Count -gt 0 -and $restoreRow[0].lastWriteTime)
+ $checks += New-AgentOutcomeCheck "backup_status_evidence" $backupStatusOk
+ $checks += New-AgentOutcomeCheck "backup_freshness_evidence" $freshnessOk
+ $checks += New-AgentOutcomeCheck "backup_offsite_verify_evidence" $offsiteOk
+ $checks += New-AgentOutcomeCheck "backup_escrow_evidence" $escrowOk
+ $checks += New-AgentOutcomeCheck "backup_restore_drill_evidence" $restoreOk
+ if ($backupStatusOk) { $modeEvidenceRefs["backup_status_evidence_ref"] = "backup-status:$([string]$backupStatusRow[0].lastWriteTime)" }
+ if ($freshnessOk) { $modeEvidenceRefs["freshness_evidence_ref"] = "backup-freshness:targets-$($requiredTargets.Count):files-$($requiredFiles.Count)" }
+ if ($offsiteOk) { $modeEvidenceRefs["offsite_verify_evidence_ref"] = [string]$backup.offsiteVerify.receiptId }
+ if ($escrowOk) { $modeEvidenceRefs["escrow_evidence_ref"] = [string]$backup.escrow.receiptId }
+ if ($restoreOk) { $modeEvidenceRefs["restore_drill_evidence_ref"] = "restore-drill:$([string]$restoreRow[0].lastWriteTime)" }
+ $modeVerifierPassed = [bool]($backupStatusOk -and $freshnessOk -and $offsiteOk -and $escrowOk -and $restoreOk)
}
"ProviderFreshness" {
$provider = $data.providerFreshness
@@ -3819,6 +4055,10 @@ function Get-AgentOutcomeContract {
if ($SourceEventResolutionRequired) {
$checks += New-AgentOutcomeCheck "source_event_resolved" $sourceEventResolvedBool $true "policy=$SourceEventResolutionPolicy evidence=$effectiveSourceEventEvidence"
}
+ if ($ModeName -eq "BackupCheck" -and $sourceEventResolvedBool -and $effectiveSourceEventEvidence) {
+ $sourceEvidenceLeaf = Split-Path -Leaf ([string]$effectiveSourceEventEvidence)
+ $modeEvidenceRefs["source_resolution_receipt_ref"] = "source-resolution:$sourceEvidenceLeaf"
+ }
$state = if (-not $transportOk) {
"failed"
} elseif (-not $basePassed) {
@@ -3850,6 +4090,7 @@ function Get-AgentOutcomeContract {
sourceEventEvidence = if ($effectiveSourceEventEvidence) { $effectiveSourceEventEvidence } else { $null }
failedChecks = $failedChecks
checks = $checks
+ evidenceRefs = $modeEvidenceRefs
verifiedAt = (Get-Date -Format o)
}
}
@@ -4672,6 +4913,20 @@ function Invoke-AgentQueuedCommands {
$sourceEventResolutionPolicy = if ($command.PSObject.Properties["sourceEventResolutionPolicy"]) { [string]$command.sourceEventResolutionPolicy } else { "external_source_receipt_required" }
$sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null }
$sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" }
+ $automationRunId = if ($command.PSObject.Properties["automationRunId"]) { [string]$command.automationRunId } else { "" }
+ $traceId = if ($command.PSObject.Properties["traceId"]) { [string]$command.traceId } else { "" }
+ $workItemId = if ($command.PSObject.Properties["workItemId"]) { [string]$command.workItemId } else { "" }
+ $idempotencyKey = if ($command.PSObject.Properties["idempotencyKey"]) { [string]$command.idempotencyKey } else { "" }
+ $executionGeneration = if ($command.PSObject.Properties["executionGeneration"]) { [string]$command.executionGeneration } else { "1" }
+ $incidentId = if ($command.PSObject.Properties["incidentId"]) { [string]$command.incidentId } else { "" }
+ $approvalId = if ($command.PSObject.Properties["approvalId"]) { [string]$command.approvalId } else { "" }
+ $approvalIdentityPresent = [bool]$command.PSObject.Properties["approvalId"]
+ $projectId = if ($command.PSObject.Properties["projectId"]) { [string]$command.projectId } else { "" }
+ $sourceFingerprint = if ($command.PSObject.Properties["sourceFingerprint"]) { [string]$command.sourceFingerprint } else { "" }
+ $dispatchRouteId = if ($command.PSObject.Properties["dispatchRouteId"]) { [string]$command.dispatchRouteId } else { "" }
+ $singleFlightKey = if ($command.PSObject.Properties["singleFlightKey"]) { [string]$command.singleFlightKey } else { "" }
+ $lockOwner = if ($command.PSObject.Properties["lockOwner"]) { [string]$command.lockOwner } else { "" }
+ $canonicalDigest = if ($command.PSObject.Properties["canonicalDigest"]) { [string]$command.canonicalDigest } else { "" }
$replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" }
$replyMessageId = if ($command.PSObject.Properties["replyMessageId"]) { [string]$command.replyMessageId } else { "" }
$suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false }
@@ -4683,6 +4938,24 @@ function Invoke-AgentQueuedCommands {
$results += $result
continue
}
+ $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey -or $lockOwner -or $canonicalDigest)
+ $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $approvalIdentityPresent -or -not $executionGeneration)
+ if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) {
+ $target = Join-Path $failedDir ("failed-" + $file.Name)
+ Move-Item -Force $file.FullName $target
+ $result = [pscustomobject]@{
+ id = $id
+ mode = $modeName
+ source = $commandSource
+ ok = $false
+ reason = "controlled_dispatch_identity_missing"
+ runtimeWritePerformed = $false
+ file = $target
+ }
+ Record-AgentEvent "operator_command_rejected" "warning" "id=$id mode=$modeName reason=controlled_dispatch_identity_missing" $result -Alert
+ $results += $result
+ continue
+ }
$runningPath = Join-Path $queueDir ("running-" + $file.Name)
Move-Item -Force $file.FullName $runningPath
@@ -4712,6 +4985,31 @@ function Invoke-AgentQueuedCommands {
-3
}
$outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence -SourceEventResolutionPolicy $sourceEventResolutionPolicy
+ $outcome | Add-Member -MemberType NoteProperty -Name automationRunId -Value $automationRunId -Force
+ $outcome | Add-Member -MemberType NoteProperty -Name traceId -Value $traceId -Force
+ $outcome | Add-Member -MemberType NoteProperty -Name workItemId -Value $workItemId -Force
+ $outcome | Add-Member -MemberType NoteProperty -Name executionGeneration -Value $executionGeneration -Force
+ $outcome | Add-Member -MemberType NoteProperty -Name incidentId -Value $incidentId -Force
+ $dispatchIdentity = $null
+ if ($hasDispatchIdentity) {
+ $dispatchIdentity = [pscustomobject]@{
+ schema_version = "agent99_controlled_dispatch_identity_v1"
+ project_id = $projectId
+ incident_id = $incidentId
+ source_fingerprint = $sourceFingerprint
+ route_id = $dispatchRouteId
+ execution_generation = $executionGeneration
+ approval_id = $approvalId
+ run_id = $automationRunId
+ trace_id = $traceId
+ work_item_id = $workItemId
+ idempotency_key = $idempotencyKey
+ single_flight_key = $singleFlightKey
+ lock_owner = $lockOwner
+ canonical_digest = $canonicalDigest
+ }
+ $outcome | Add-Member -MemberType NoteProperty -Name identity -Value $dispatchIdentity -Force
+ }
$result = [pscustomobject]@{
id = $id
mode = $modeName
@@ -4733,6 +5031,14 @@ function Invoke-AgentQueuedCommands {
verifierPassed = [bool]$outcome.verifierPassed
sourceEventResolved = [bool]$outcome.sourceEventResolved
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
+ automationRunId = $automationRunId
+ traceId = $traceId
+ workItemId = $workItemId
+ idempotencyKey = $idempotencyKey
+ executionGeneration = $executionGeneration
+ incidentId = $incidentId
+ approvalId = $approvalId
+ identity = $dispatchIdentity
outcome = $outcome
exitCode = $exitCode
durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1)
@@ -4742,6 +5048,19 @@ function Invoke-AgentQueuedCommands {
}
$problem = Update-AgentProblemKnowledge $result
$result | Add-Member -MemberType NoteProperty -Name problem -Value $problem -Force
+ $donePath = Join-Path $processedDir ("processed-$id.json")
+ $result | ConvertTo-Json -Depth 8 | Set-Content -Path $donePath -Encoding UTF8
+ $sourceEvidenceValue = if ($sourceEventEvidence) { $sourceEventEvidence } elseif ($latest.path) { [string]$latest.path } else { "agent99-outcome-$id" }
+ $sourceEvidenceRef = "agent99-source:$(Convert-AgentPublicReceiptLeaf $sourceEvidenceValue)"
+ $postVerifierEvidenceValue = if ($latest.path) { [string]$latest.path } else { $donePath }
+ $postVerifierEvidenceRef = "agent99-verifier:$(Convert-AgentPublicReceiptLeaf $postVerifierEvidenceValue)"
+ $outcomeWriteback = if ($hasDispatchIdentity) {
+ Send-AgentOutcomeWriteback $result $postVerifierEvidenceRef $sourceEvidenceRef
+ } else {
+ [pscustomobject]@{ ok = $false; status = "no_durable_dispatch_identity"; attempts = 0; runtimeClosureVerified = $false }
+ }
+ $result | Add-Member -MemberType NoteProperty -Name outcomeWriteback -Value $outcomeWriteback -Force
+ $result | ConvertTo-Json -Depth 10 | Set-Content -Path $donePath -Encoding UTF8
Remove-Item $runningPath -Force -ErrorAction SilentlyContinue | Out-Null
$recordData = $result | Select-Object *
if ($replyChatId) {
@@ -5406,6 +5725,13 @@ if ($SelfTestOutcomeContract) {
Invoke-AgentOutcomeContractSelfTest
}
+if ($SelfTestBackupReadbackContract) {
+ $backupReadbackSelfTest = Invoke-AgentBackupReadbackContractSelfTest
+ $backupReadbackSelfTest | ConvertTo-Json -Depth 10
+ if (-not $backupReadbackSelfTest.ok) { exit 1 }
+ exit 0
+}
+
Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath"
Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert
diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1
index 584ed271a..5b2c05c3b 100644
--- a/agent99-sre-alert-inbox.ps1
+++ b/agent99-sre-alert-inbox.ps1
@@ -84,7 +84,7 @@ function Test-AgentAlertActionable {
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
return $true
}
- if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
+ if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|cold[-_ ]start|reboot[-_ ]auto[-_ ]recovery|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
return $true
}
return $false
@@ -180,11 +180,15 @@ function Resolve-AgentAlertMode {
function Get-AgentAlertCorrelationKey {
param([object]$Alert)
+ $awoooi = Get-AgentField $Alert "awoooi" $null
+ $dispatchIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null
+ $stableSingleFlight = [string](Get-AgentField $dispatchIdentity "single_flight_key" "")
+ if ($stableSingleFlight) { return "controlled:$($stableSingleFlight.ToLowerInvariant())" }
+
$routing = Get-AgentField $Alert "routing" $null
$correlationKey = [string](Get-AgentField $routing "correlationKey" "")
if ($correlationKey) { return "route:$($correlationKey.ToLowerInvariant())" }
- $awoooi = Get-AgentField $Alert "awoooi" $null
$fingerprint = [string](Get-AgentField $awoooi "fingerprint" "")
if (-not $fingerprint) { $fingerprint = [string](Get-AgentField $Alert "fingerprint" "") }
if ($fingerprint) { return "fingerprint:$($fingerprint.ToLowerInvariant())" }
@@ -747,6 +751,37 @@ foreach ($file in $files) {
$resolution = Get-AgentField $alert "resolution" $null
$sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required")
$sourceEventResolutionRequired = Convert-AgentBool (Get-AgentField $resolution "required" $true)
+ $routing = Get-AgentField $alert "routing" $null
+ $awoooiMetadata = Get-AgentField $alert "awoooi" $null
+ $dispatchIdentity = Get-AgentField $awoooiMetadata "agent99DispatchIdentity" $null
+ $automationRunId = [string](Get-AgentField $dispatchIdentity "run_id" (Get-AgentField $routing "runId" ""))
+ $traceId = [string](Get-AgentField $dispatchIdentity "trace_id" (Get-AgentField $routing "traceId" ""))
+ $workItemId = [string](Get-AgentField $dispatchIdentity "work_item_id" (Get-AgentField $routing "workItemId" ""))
+ $idempotencyKey = [string](Get-AgentField $dispatchIdentity "idempotency_key" (Get-AgentField $routing "idempotencyKey" ""))
+ $executionGeneration = [string](Get-AgentField $dispatchIdentity "execution_generation" (Get-AgentField $routing "executionGeneration" "1"))
+ $incidentId = [string](Get-AgentField $dispatchIdentity "incident_id" (Get-AgentField $awoooiMetadata "incidentId" ""))
+ $approvalId = [string](Get-AgentField $dispatchIdentity "approval_id" (Get-AgentField $awoooiMetadata "approvalId" ""))
+ $projectId = [string](Get-AgentField $dispatchIdentity "project_id" "")
+ $sourceFingerprint = [string](Get-AgentField $dispatchIdentity "source_fingerprint" "")
+ $dispatchRouteId = [string](Get-AgentField $dispatchIdentity "route_id" "")
+ $singleFlightKey = [string](Get-AgentField $dispatchIdentity "single_flight_key" "")
+ $lockOwner = [string](Get-AgentField $dispatchIdentity "lock_owner" "")
+ $canonicalDigest = [string](Get-AgentField $dispatchIdentity "canonical_digest" "")
+ $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey)
+ $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $executionGeneration)
+ if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) {
+ $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
+ Move-Item -Force $runningPath $failedPath
+ $failed += [pscustomobject]@{
+ alertId = $alertId
+ file = $failedPath
+ ok = $false
+ reason = "controlled_dispatch_identity_missing"
+ mode = $modeName
+ runtimeWritePerformed = $false
+ }
+ continue
+ }
$requestPath = Join-Path $RequestDir "$queueId.json"
$queuePath = Join-Path $QueueDir "$queueId.json"
@@ -774,6 +809,19 @@ foreach ($file in $files) {
suppressTelegram = $suppressTelegram
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
sourceEventResolutionRequired = $sourceEventResolutionRequired
+ automationRunId = $automationRunId
+ traceId = $traceId
+ workItemId = $workItemId
+ idempotencyKey = $idempotencyKey
+ executionGeneration = $executionGeneration
+ incidentId = $incidentId
+ approvalId = $approvalId
+ projectId = $projectId
+ sourceFingerprint = $sourceFingerprint
+ dispatchRouteId = $dispatchRouteId
+ singleFlightKey = $singleFlightKey
+ lockOwner = $lockOwner
+ canonicalDigest = $canonicalDigest
createdAt = (Get-Date -Format o)
} | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8
@@ -802,6 +850,19 @@ foreach ($file in $files) {
suppressTelegram = $suppressTelegram
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
sourceEventResolutionRequired = $sourceEventResolutionRequired
+ automationRunId = $automationRunId
+ traceId = $traceId
+ workItemId = $workItemId
+ idempotencyKey = $idempotencyKey
+ executionGeneration = $executionGeneration
+ incidentId = $incidentId
+ approvalId = $approvalId
+ projectId = $projectId
+ sourceFingerprint = $sourceFingerprint
+ dispatchRouteId = $dispatchRouteId
+ singleFlightKey = $singleFlightKey
+ lockOwner = $lockOwner
+ canonicalDigest = $canonicalDigest
createdAt = (Get-Date -Format o)
} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8
@@ -843,6 +904,13 @@ foreach ($file in $files) {
suppressTelegram = $suppressTelegram
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
sourceEventResolutionRequired = $sourceEventResolutionRequired
+ automationRunId = $automationRunId
+ traceId = $traceId
+ workItemId = $workItemId
+ idempotencyKey = $idempotencyKey
+ executionGeneration = $executionGeneration
+ incidentId = $incidentId
+ approvalId = $approvalId
requestPath = $requestPath
queuePath = $queuePath
alertPath = $processedPath
diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1
index 1e5d4cc40..ac2180376 100644
--- a/agent99-sre-alert-relay.ps1
+++ b/agent99-sre-alert-relay.ps1
@@ -14,9 +14,10 @@ $AlertRoot = Join-Path $AgentRoot "alerts"
$IncomingDir = Join-Path $AlertRoot "incoming"
$LogDir = Join-Path $AgentRoot "logs"
$BinDir = Join-Path $AgentRoot "bin"
+$ProcessedDir = Join-Path (Join-Path $AgentRoot "queue") "processed"
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
-New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir | Out-Null
+New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $ProcessedDir | Out-Null
function Convert-AgentSafeFileToken {
param([string]$Value)
@@ -47,6 +48,115 @@ function Get-AgentEnvValue {
return ""
}
+function Test-AgentPublicReceiptRef {
+ param([string]$Value)
+ if (-not $Value -or $Value.Length -gt 256) { return $false }
+ if ($Value -match "(?i)(authorization|bearer|password|passwd|token|secret|api[_-]?key|private[_-]?key|cookie|session)(\s|:|=)|-----BEGIN.*PRIVATE KEY-----|(^|[/\.])\.env($|[/\.])") { return $false }
+ if ($Value -match "\.\.|://|[\\?&#=\s]") { return $false }
+ return [bool]($Value -match "^[A-Za-z0-9][A-Za-z0-9._:/@+%-]{0,255}$")
+}
+
+function Get-AgentCanonicalOutcomeIdentity {
+ param([object]$Result, [object]$Outcome)
+ $outer = Get-AgentField $Result "identity" $null
+ $nested = Get-AgentField $Outcome "identity" $null
+ $required = @(
+ "schema_version", "project_id", "incident_id", "source_fingerprint",
+ "route_id", "execution_generation", "approval_id", "run_id", "trace_id",
+ "work_item_id", "idempotency_key", "single_flight_key", "lock_owner",
+ "canonical_digest"
+ )
+ if (-not $outer -or -not $nested) { return $null }
+ foreach ($name in $required) {
+ $outerValue = [string](Get-AgentField $outer $name "")
+ $nestedValue = [string](Get-AgentField $nested $name "")
+ if ($name -ne "approval_id" -and (-not $outerValue -or -not $nestedValue)) {
+ return $null
+ }
+ if ($outerValue -ne $nestedValue) { return $null }
+ }
+ return $outer
+}
+
+function Get-AgentOutcomeReadback {
+ param([string]$RunId)
+
+ $parsedRunId = [guid]::Empty
+ if (-not [guid]::TryParse($RunId, [ref]$parsedRunId)) {
+ return @{ ok = $false; found = $false; error = "invalid_run_id" }
+ }
+
+ foreach ($file in @(Get-ChildItem -Path $ProcessedDir -Filter "processed-*.json" -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) {
+ try {
+ $result = Get-Content $file.FullName -Raw | ConvertFrom-Json
+ if ([string](Get-AgentField $result "automationRunId" "") -ne $RunId) { continue }
+ $outcome = Get-AgentField $result "outcome" $null
+ $canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $result $outcome
+ if (-not $canonicalIdentity) { continue }
+ $safeChecks = @()
+ foreach ($check in @((Get-AgentField $outcome "checks" @()) | Select-Object -First 32)) {
+ $safeChecks += @{
+ name = [string](Get-AgentField $check "name" "")
+ required = [bool](Get-AgentField $check "required" $false)
+ passed = [bool](Get-AgentField $check "passed" $false)
+ }
+ }
+ $safeEvidenceRefs = @{}
+ $outcomeEvidenceRefs = Get-AgentField $outcome "evidenceRefs" $null
+ foreach ($name in @(
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref"
+ )) {
+ $value = [string](Get-AgentField $outcomeEvidenceRefs $name "")
+ if (Test-AgentPublicReceiptRef $value) { $safeEvidenceRefs[$name] = $value }
+ }
+ return @{
+ ok = $true
+ found = $true
+ schemaVersion = "agent99_outcome_readback_v1"
+ automationRunId = $RunId
+ traceId = [string](Get-AgentField $result "traceId" "")
+ workItemId = [string](Get-AgentField $result "workItemId" "")
+ idempotencyKey = [string](Get-AgentField $result "idempotencyKey" "")
+ executionGeneration = [string](Get-AgentField $result "executionGeneration" "1")
+ incidentId = [string](Get-AgentField $result "incidentId" "")
+ approvalId = [string](Get-AgentField $result "approvalId" "")
+ identity = $canonicalIdentity
+ controlledApply = [bool](Get-AgentField $result "controlledApply" $false)
+ mode = [string](Get-AgentField $result "mode" "")
+ outcome = @{
+ schemaVersion = [string](Get-AgentField $outcome "schemaVersion" "")
+ state = [string](Get-AgentField $outcome "state" "unknown")
+ resolved = [bool](Get-AgentField $outcome "resolved" $false)
+ transportOk = [bool](Get-AgentField $outcome "transportOk" $false)
+ verifierName = [string](Get-AgentField $outcome "verifierName" "")
+ verifierPassed = [bool](Get-AgentField $outcome "verifierPassed" $false)
+ sourceEventResolved = [bool](Get-AgentField $outcome "sourceEventResolved" $false)
+ identity = $canonicalIdentity
+ failedChecks = @((Get-AgentField $outcome "failedChecks" @()) | Select-Object -First 32)
+ checks = $safeChecks
+ evidenceRefs = $safeEvidenceRefs
+ verifiedAt = [string](Get-AgentField $outcome "verifiedAt" "")
+ }
+ storesRawEvidence = $false
+ }
+ } catch {
+ continue
+ }
+ }
+ return @{
+ ok = $true
+ found = $false
+ schemaVersion = "agent99_outcome_readback_v1"
+ automationRunId = $RunId
+ storesRawEvidence = $false
+ }
+}
+
function Write-AgentRelayEvent {
param([hashtable]$Event)
$path = Join-Path $LogDir "agent99-sre-alert-relay.jsonl"
@@ -136,12 +246,6 @@ try {
$context = $listener.GetContext()
$remote = [string]$context.Request.RemoteEndPoint.Address
try {
- if ($context.Request.HttpMethod -ne "POST") {
- Send-AgentRelayResponse $context 405 @{ ok = $false; error = "method_not_allowed" }
- Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "method_not_allowed" }
- continue
- }
-
if (-not (Test-AgentRelayRemoteAllowed $context)) {
Send-AgentRelayResponse $context 403 @{ ok = $false; error = "remote_not_allowed" }
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "remote_not_allowed" }
@@ -158,6 +262,34 @@ try {
}
}
+ if ($context.Request.HttpMethod -eq "GET" -and -not $expectedToken) {
+ Send-AgentRelayResponse $context 503 @{ ok = $false; found = $false; error = "outcome_readback_auth_not_configured" }
+ Write-AgentRelayEvent @{ event = "outcome_readback_rejected"; ok = $false; remote = $remote; reason = "auth_not_configured" }
+ continue
+ }
+
+ if ($context.Request.HttpMethod -eq "GET") {
+ $runId = [string]$context.Request.QueryString["run_id"]
+ $readback = Get-AgentOutcomeReadback $runId
+ $statusCode = if ($readback.ok) { 200 } else { 400 }
+ Send-AgentRelayResponse $context $statusCode $readback
+ Write-AgentRelayEvent @{
+ event = "outcome_readback"
+ ok = [bool]$readback.ok
+ found = [bool]$readback.found
+ remote = $remote
+ runId = if ($readback.ok) { $runId } else { "invalid" }
+ storesRawEvidence = $false
+ }
+ continue
+ }
+
+ if ($context.Request.HttpMethod -ne "POST") {
+ Send-AgentRelayResponse $context 405 @{ ok = $false; error = "method_not_allowed" }
+ Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "method_not_allowed" }
+ continue
+ }
+
if ($context.Request.ContentLength64 -gt $MaxBodyBytes) {
Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" }
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large" }
diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json
index 29240f9b0..5099fb06b 100644
--- a/agent99.config.99.example.json
+++ b/agent99.config.99.example.json
@@ -144,6 +144,11 @@
"batchLimit": 5,
"retryAfterMinutes": 5
},
+ "outcomeWriteback": {
+ "enabled": true,
+ "url": "https://awoooi.wooo.work/api/v1/webhooks/agent99/outcomes",
+ "tokenEnv": "AGENT99_SRE_RELAY_TOKEN"
+ },
"telegram": {
"enabled": true,
"botTokenEnv": "AGENT99_TELEGRAM_BOT_TOKEN",
@@ -325,6 +330,9 @@
"backupHealth": {
"host": "192.168.0.110",
"snapshotReadTimeoutSeconds": 90,
+ "protectionMetricsPath": "/home/wooo/node_exporter_textfiles/backup_health.prom",
+ "protectionReadTimeoutSeconds": 45,
+ "protectionMaxAgeMinutes": 60,
"targets": [],
"fileChecks": [],
"cronPatterns": []
diff --git a/agent99.config.example.json b/agent99.config.example.json
index 505e545f1..083f9eed2 100644
--- a/agent99.config.example.json
+++ b/agent99.config.example.json
@@ -136,6 +136,11 @@
"batchLimit": 5,
"retryAfterMinutes": 5
},
+ "outcomeWriteback": {
+ "enabled": true,
+ "url": "https://awoooi.wooo.work/api/v1/webhooks/agent99/outcomes",
+ "tokenEnv": "AGENT99_SRE_RELAY_TOKEN"
+ },
"telegram": {
"enabled": true,
"botTokenEnv": "AGENT99_TELEGRAM_BOT_TOKEN",
@@ -260,6 +265,9 @@
},
"backupHealth": {
"host": "192.168.0.110",
+ "protectionMetricsPath": "/home/wooo/node_exporter_textfiles/backup_health.prom",
+ "protectionReadTimeoutSeconds": 45,
+ "protectionMaxAgeMinutes": 60,
"targets": [],
"fileChecks": [],
"cronPatterns": []
diff --git a/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11.sql b/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11.sql
new file mode 100644
index 000000000..1928874c5
--- /dev/null
+++ b/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11.sql
@@ -0,0 +1,14 @@
+-- Durable retry not-before for Agent99 controlled dispatch claims.
+-- Prevents the 15-second reconciler tick from consuming every attempt and
+-- generation while a Redis flight lease or transport cooldown is active.
+
+BEGIN;
+
+ALTER TABLE awooop_run_state
+ ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMP NULL;
+
+CREATE INDEX IF NOT EXISTS idx_run_state_retry_due
+ ON awooop_run_state (project_id, next_attempt_at)
+ WHERE state = 'pending' AND next_attempt_at IS NOT NULL;
+
+COMMIT;
diff --git a/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11_down.sql b/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11_down.sql
new file mode 100644
index 000000000..7079fb15a
--- /dev/null
+++ b/apps/api/migrations/agent99_dispatch_next_attempt_at_2026-07-11_down.sql
@@ -0,0 +1,6 @@
+BEGIN;
+
+DROP INDEX IF EXISTS idx_run_state_retry_due;
+ALTER TABLE awooop_run_state DROP COLUMN IF EXISTS next_attempt_at;
+
+COMMIT;
diff --git a/apps/api/src/api/v1/auto_repair.py b/apps/api/src/api/v1/auto_repair.py
index a75b016a4..cb808aa05 100644
--- a/apps/api/src/api/v1/auto_repair.py
+++ b/apps/api/src/api/v1/auto_repair.py
@@ -17,6 +17,7 @@ from fastapi import APIRouter, HTTPException, Query
from pydantic import BaseModel, Field
from src.core.csrf import CSRFToken # Phase 20: CSRF Protection
+from src.models.incident import IncidentReadDegradation
from src.services.auto_repair_service import (
get_auto_repair_service,
)
@@ -248,6 +249,11 @@ class RepairHistoryResponse(BaseModel):
count: int
items: list[RepairHistoryItem]
+ degraded: bool = False
+ degraded_count: int = 0
+ degraded_records: list[IncidentReadDegradation] = Field(default_factory=list)
+ source_of_truth: str = "postgresql"
+ redis_fallback_used: bool = False
@router.get("/history", response_model=RepairHistoryResponse)
@@ -267,9 +273,20 @@ async def get_repair_history(
"""
try:
incident_service = get_incident_service()
- all_incidents = await incident_service.get_active_incidents(
- project_id=project_id,
+ detailed_reader = getattr(
+ incident_service,
+ "get_active_incidents_readback",
+ None,
)
+ if callable(detailed_reader):
+ readback = await detailed_reader(project_id=project_id)
+ all_incidents = readback.incidents
+ degraded_records = readback.degraded_records
+ else:
+ all_incidents = await incident_service.get_active_incidents(
+ project_id=project_id,
+ )
+ degraded_records = []
items: list[RepairHistoryItem] = []
for incident in all_incidents:
@@ -315,7 +332,13 @@ async def get_repair_history(
# 最多回傳 limit 筆，newest first (updated_at 已是活躍事件，先按 ID 截斷)
items = items[:limit]
- return RepairHistoryResponse(count=len(items), items=items)
+ return RepairHistoryResponse(
+ count=len(items),
+ items=items,
+ degraded=bool(degraded_records),
+ degraded_count=len(degraded_records),
+ degraded_records=degraded_records,
+ )
except IncidentDurableReadError as exc:
raise HTTPException(
diff --git a/apps/api/src/api/v1/incidents.py b/apps/api/src/api/v1/incidents.py
index e1c9c8252..4323242ec 100644
--- a/apps/api/src/api/v1/incidents.py
+++ b/apps/api/src/api/v1/incidents.py
@@ -25,7 +25,12 @@ from pydantic import BaseModel, Field
from src.core.logging import get_logger
from src.models.approval import ApprovalRequestResponse
-from src.models.incident import Incident, IncidentStatus, Severity
+from src.models.incident import (
+ Incident,
+ IncidentReadDegradation,
+ IncidentStatus,
+ Severity,
+)
# Phase 22 P1: 移除 Repository 直接存取 (2026-03-31)
# Phase 16 R3.3b (2026-03-25 台北時區): Repository 層整合 - 已移至 Service 層
@@ -86,6 +91,11 @@ class IncidentListResponse(BaseModel):
"""事件清單回應"""
count: int
incidents: list[IncidentResponse]
+ degraded: bool = False
+ degraded_count: int = 0
+ degraded_records: list[IncidentReadDegradation] = Field(default_factory=list)
+ source_of_truth: str = "postgresql"
+ redis_fallback_used: bool = False
class ProposalGenerateResponse(BaseModel):
@@ -189,9 +199,20 @@ async def list_incidents(
try:
# 透過 Service 取得活躍事件
- incidents = await incident_service.get_active_incidents(
- project_id=project_id,
+ detailed_reader = getattr(
+ incident_service,
+ "get_active_incidents_readback",
+ None,
)
+ if callable(detailed_reader):
+ readback = await detailed_reader(project_id=project_id)
+ incidents = readback.incidents
+ degraded_records = readback.degraded_records
+ else:
+ incidents = await incident_service.get_active_incidents(
+ project_id=project_id,
+ )
+ degraded_records = []
# 按時間排序 (最新優先)
# 2026-03-26 修復: 處理 timezone-aware 與 naive datetime 混合問題
@@ -277,6 +298,9 @@ async def list_incidents(
return IncidentListResponse(
count=len(incidents),
incidents=responses,
+ degraded=bool(degraded_records),
+ degraded_count=len(degraded_records),
+ degraded_records=degraded_records,
)
except IncidentDurableReadError as e:
diff --git a/apps/api/src/api/v1/platform/operator_runs.py b/apps/api/src/api/v1/platform/operator_runs.py
index 606f4c010..764d1f1e9 100644
--- a/apps/api/src/api/v1/platform/operator_runs.py
+++ b/apps/api/src/api/v1/platform/operator_runs.py
@@ -133,6 +133,14 @@ class AiAlertCardDeliveryItem(BaseModel):
lane: str
target: str
gates: list[str]
+ declared_gates: list[str] = Field(default_factory=list)
+ declared_runtime_write_gate_count: int = 0
+ declared_runtime_write_allowed: bool = False
+ declared_runtime_write_gate_state: str = "unknown"
+ declared_candidate_only: bool = False
+ declared_controlled_playbook_queue: bool = False
+ declared_learning_writeback_status: str = "not_evaluated"
+ effective_policy_projection: dict[str, Any] | None = None
runtime_write_gate_count: int
runtime_write_allowed: bool
candidate_only: bool
@@ -143,6 +151,7 @@ class AiAlertCardDeliveryItem(BaseModel):
learning_writeback_status: str = "not_evaluated"
learning_writeback_refs: dict[str, Any] = Field(default_factory=dict)
learning_registry_bindings: list[dict[str, Any]] = Field(default_factory=list)
+ backup_restore_automation: dict[str, Any] | None = None
source_refs: dict[str, Any]
run_state: str | None = None
agent_id: str | None = None
@@ -167,8 +176,13 @@ class AiAlertCardDeliverySummary(BaseModel):
pending_total: int
shadow_total: int
delivery_receipt_required_total: int
+ backup_restore_total: int = 0
+ declared_runtime_write_gate_open_count: int = 0
+ backup_runtime_write_gate_excluded_total: int = 0
runtime_write_gate_open_count: int
runtime_write_allowed: bool
+ declared_learning_writeback_ready_total: int = 0
+ backup_learning_writeback_excluded_total: int = 0
learning_writeback_ready_total: int = 0
learning_writeback_missing_total: int = 0
km_writeback_ref_ready_total: int = 0
@@ -181,6 +195,10 @@ class AiAlertCardDeliverySummary(BaseModel):
learning_registry_ready_target_count: int = 0
learning_registry_missing_target_count: int = 0
learning_registry_next_action: str = "none"
+ backup_restore_readback_required: bool = False
+ no_false_green_enforced: bool = False
+ backup_restore_next_action: str = "none"
+ effective_policy_projection: dict[str, Any] = Field(default_factory=dict)
latest_sent_at: datetime | None = None
latest_queued_at: datetime | None = None
production_write_count: int = 0
diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py
index bc5268995..8d266141e 100644
--- a/apps/api/src/api/v1/webhooks.py
+++ b/apps/api/src/api/v1/webhooks.py
@@ -43,7 +43,13 @@ from src.models.approval import (
DryRunCheck,
RiskLevel,
)
+from src.models.incident import SignalSource
from src.models.webhook import AlertPayload, AlertResponse
+from src.services.agent99_sre_bridge import (
+ bridge_alertmanager_to_agent99,
+ resolve_agent99_durable_route,
+)
+from src.services.agent99_outcome_ingestion import ingest_agent99_outcome_receipt
from src.services.alert_analyzer_service import AlertAnalyzer
from src.services.alert_approval_guard import guard_alert_approval_action
from src.services.alert_grouping_service import get_alert_grouping_service
@@ -55,11 +61,16 @@ from src.services.alertmanager_llm_guard import (
from src.services.approval_db import get_approval_service
from src.services.auto_approve import get_auto_approve_policy
from src.services.auto_repair_service import AutoRepairService
-from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
+from src.services.backup_restore_alertmanager_ingress import (
+ process_backup_restore_alertmanager_signal,
+)
from src.services.channel_hub import (
record_alertmanager_event,
record_grouped_alert_event,
)
+from src.services.controlled_alert_target_router import (
+ build_controlled_recovery_handoff,
+)
from src.services.converged_alert_recurrence_notifier import (
notify_converged_alert_recurrence,
)
@@ -96,6 +107,59 @@ router = APIRouter(prefix="/webhooks", tags=["Webhooks"])
logger = get_logger("awoooi.webhooks")
+class Agent99OutcomeWebhookPayload(BaseModel):
+ """Public-safe Agent99 outcome; learning refs are compatibility-only."""
+
+ identity: dict[str, str]
+ outcome_receipt: dict[str, Any]
+ evidence_refs: dict[str, str]
+ learning_receipt_refs: dict[str, str] = Field(
+ default_factory=dict,
+ description=(
+ "Untrusted compatibility input. Ignored for terminal closure; "
+ "the reconciler reads durable server-side step-3 checkpoints."
+ ),
+ )
+
+
+@router.post("/agent99/outcomes", summary="Ingest a fenced Agent99 outcome receipt")
+async def ingest_agent99_outcome_webhook(
+ payload: Agent99OutcomeWebhookPayload,
+ x_agent99_relay_token: str | None = Header(
+ default=None,
+ alias="X-Agent99-Relay-Token",
+ ),
+) -> dict[str, Any]:
+ """Persist a fenced outcome/verifier; the reconciler owns closure."""
+
+ configured_token = str(settings.AGENT99_SRE_ALERT_RELAY_TOKEN or "")
+ if (
+ not configured_token
+ or not x_agent99_relay_token
+ or not hmac.compare_digest(configured_token, x_agent99_relay_token)
+ ):
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="agent99_outcome_authentication_failed",
+ )
+ try:
+ receipt = await ingest_agent99_outcome_receipt(payload.model_dump())
+ except ValueError as exc:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail=str(exc),
+ ) from exc
+ if receipt.get("accepted") is not True:
+ failure_status = str(receipt.get("status") or "outcome_rejected")
+ response_status = (
+ status.HTTP_503_SERVICE_UNAVAILABLE
+ if "persistence" in failure_status
+ else status.HTTP_409_CONFLICT
+ )
+ raise HTTPException(status_code=response_status, detail=failure_status)
+ return receipt
+
+
# =============================================================================
# Incident-Approval 同步 (feedback_incident_approval_sync.md 鐵律)
# =============================================================================
@@ -238,6 +302,38 @@ def _controlled_alert_execution_priority(risk_level: RiskLevel | str) -> int:
)
+def _resolve_controlled_source_fingerprint(
+ *,
+ source_fingerprint: str,
+ source_alertname: str,
+ alert_type: str,
+ target_resource: str,
+ namespace: str,
+ source_message: str,
+ source_labels: dict[str, Any] | None,
+) -> str:
+ """Preserve the canonical webhook identity for normalized alert routes."""
+
+ fingerprint = str(source_fingerprint or "").strip()
+ if fingerprint:
+ return fingerprint
+
+ labels = dict(source_labels or {})
+ labels["alertname"] = str(
+ source_alertname or labels.get("alertname") or alert_type
+ ).strip()
+ normalized_alert = AlertPayload(
+ alert_type="custom",
+ severity="warning",
+ source="alert_webhook_controlled_router",
+ target_resource=target_resource,
+ namespace=namespace,
+ message=source_message or target_resource,
+ labels=labels,
+ )
+ return AlertAnalyzer.generate_fingerprint(normalized_alert)
+
+
async def _try_auto_repair_background(
incident_id: str,
approval_id: str,
@@ -245,41 +341,195 @@ async def _try_auto_repair_background(
target_resource: str,
namespace: str,
risk_level: str,
+ source_alert_id: str = "",
+ source_alertname: str = "",
+ source_severity: str = "",
+ source_message: str = "",
+ source_labels: dict[str, Any] | None = None,
+ source_annotations: dict[str, Any] | None = None,
+ source_fingerprint: str = "",
+ alert_category: str = "",
+ notification_type: str = "",
+ source_url: str | None = None,
) -> dict:
"""Queue an alert decision for the single-writer executor."""
- from src.repositories.alert_operation_log_repository import (
- get_alert_operation_log_repository,
- )
from src.jobs.awooop_ansible_candidate_backfill_job import (
enqueue_ai_decision_ansible_candidate,
)
+ from src.repositories.alert_operation_log_repository import (
+ get_alert_operation_log_repository,
+ )
op_log = get_alert_operation_log_repository()
execution_priority = _controlled_alert_execution_priority(risk_level)
- incident = await get_incident_service().get_from_working_memory(incident_id)
- if incident is None:
+ resolved_source_fingerprint = _resolve_controlled_source_fingerprint(
+ source_fingerprint=source_fingerprint,
+ source_alertname=source_alertname,
+ alert_type=alert_type,
+ target_resource=target_resource,
+ namespace=namespace,
+ source_message=source_message,
+ source_labels=source_labels,
+ )
+ handoff = build_controlled_recovery_handoff(
+ alertname=alert_type,
+ target_resource=target_resource,
+ namespace=namespace,
+ incident_id=incident_id,
+ )
+ durable_agent99_route = resolve_agent99_durable_route(
+ alertname=source_alertname or alert_type,
+ severity=source_severity or risk_level,
+ target_resource=target_resource,
+ namespace=namespace,
+ message=source_message or target_resource,
+ labels=source_labels or {},
+ annotations=source_annotations or {},
+ )
+ if handoff is None and durable_agent99_route is not None:
+ # Domain-specific Agent99 routes (including read-only BackupCheck) are
+ # durable single-owner lanes too; they must never fall through to the
+ # generic Ansible keyword catalog.
handoff = {
"schema_version": "ai_decision_controlled_executor_handoff_v1",
- "status": "incident_not_found",
+ "status": "agent99_dispatch_reservation_pending",
"queued": False,
"side_effect_performed": False,
- "single_writer_executor": "awoooi-ansible-executor-broker",
- "active_blockers": ["incident_not_found"],
+ "single_writer_executor": "Agent99",
+ "route_id": durable_agent99_route["route_id"],
+ "agent99_kind": durable_agent99_route["kind"],
+ "agent99_mode": durable_agent99_route["suggested_mode"],
+ "active_blockers": ["agent99_dispatch_receipt_pending"],
+ "safe_next_action": "reserve_dispatch_verify_and_writeback_same_run",
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "needs_human": False,
}
+ if handoff is None:
+ incident = await get_incident_service().get_from_working_memory(incident_id)
+ if incident is None:
+ handoff = {
+ "schema_version": "ai_decision_controlled_executor_handoff_v1",
+ "status": "incident_not_found",
+ "queued": False,
+ "side_effect_performed": False,
+ "single_writer_executor": "awoooi-ansible-executor-broker",
+ "active_blockers": ["incident_not_found"],
+ }
+ else:
+ handoff = await enqueue_ai_decision_ansible_candidate(
+ incident=incident,
+ proposal_data={
+ "source": "alert_webhook_controlled_router",
+ "risk_level": risk_level,
+ "action": target_resource,
+ "alert_type": alert_type,
+ "namespace": namespace,
+ "approval_id": approval_id,
+ "execution_priority": execution_priority,
+ },
+ project_id=str(getattr(incident, "project_id", None) or "awoooi"),
+ )
else:
- handoff = await enqueue_ai_decision_ansible_candidate(
- incident=incident,
- proposal_data={
- "source": "alert_webhook_controlled_router",
- "risk_level": risk_level,
- "action": target_resource,
- "alert_type": alert_type,
- "namespace": namespace,
- "approval_id": approval_id,
- "execution_priority": execution_priority,
- },
- project_id=str(getattr(incident, "project_id", None) or "awoooi"),
+ # Agent99 is the only dispatch owner for host control-plane recovery.
+ # The generic Ansible catalog must not manufacture candidates from
+ # incidental words such as 110, backup, Docker, or container.
+ route_id = str(handoff.get("route_id") or "")
+ dispatch_result = await bridge_alertmanager_to_agent99(
+ alert_id=source_alert_id,
+ alertname=source_alertname or alert_type,
+ severity=source_severity or risk_level,
+ namespace=namespace,
+ target_resource=target_resource,
+ message=source_message or target_resource,
+ labels=source_labels or {},
+ annotations=source_annotations or {},
+ fingerprint=resolved_source_fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=source_url,
+ project_id="awoooi",
+ incident_id=incident_id,
+ approval_id=approval_id,
+ route_id=route_id,
+ work_item_id=f"agent99-dispatch:awoooi:{incident_id}:{route_id}",
+ )
+ identity = (
+ dispatch_result.get("identity")
+ if isinstance(dispatch_result.get("identity"), dict)
+ else {}
+ )
+ correlated_receipt = (
+ dispatch_result.get("correlatedReceipt")
+ if isinstance(dispatch_result.get("correlatedReceipt"), dict)
+ else {}
+ )
+ dispatch_receipt = (
+ dispatch_result.get("dispatchReceipt")
+ if isinstance(dispatch_result.get("dispatchReceipt"), dict)
+ else {}
+ )
+ accepted = bool(dispatch_receipt.get("accepted") is True)
+ inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True)
+ dispatch_promoted = bool(accepted and inbox_triggered)
+ receipt_persisted = bool(
+ correlated_receipt.get("receipt_persisted") is True
+ or (
+ dispatch_result.get("status") == "deduplicated"
+ and correlated_receipt
+ )
+ )
+ handoff.update({
+ "status": (
+ "agent99_dispatch_accepted_verifier_pending"
+ if dispatch_promoted and receipt_persisted
+ else "agent99_dispatch_failed_or_receipt_missing"
+ ),
+ "queued": dispatch_promoted and receipt_persisted,
+ "side_effect_performed": bool(
+ dispatch_result.get("dispatchPerformed") is True
+ ),
+ "automation_run_id": identity.get("run_id"),
+ "trace_id": identity.get("trace_id"),
+ "work_item_id": identity.get("work_item_id"),
+ "idempotency_key": identity.get("idempotency_key"),
+ "dispatch_receipt": dispatch_receipt or None,
+ "correlated_receipt": correlated_receipt or None,
+ "receipt_persisted": receipt_persisted,
+ "dispatch_accepted": accepted and receipt_persisted,
+ "inbox_triggered": inbox_triggered and receipt_persisted,
+ "dispatch_promoted": dispatch_promoted and receipt_persisted,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "active_blockers": (
+ [
+ "post_apply_verifier_missing",
+ "km_playbook_writeback_missing",
+ ]
+ if dispatch_promoted and receipt_persisted
+ else [
+ str(
+ dispatch_result.get("reason")
+ or "agent99_dispatch_or_receipt_failed"
+ )
+ ]
+ ),
+ "safe_next_action": (
+ "run_independent_cold_start_verifier_then_km_playbook_writeback"
+ if dispatch_promoted and receipt_persisted
+ else "retry_new_generation_after_dispatch_transport_repair"
+ ),
+ })
+ logger.info(
+ "controlled_alert_domain_route_selected",
+ incident_id=incident_id,
+ target_resource=target_resource,
+ executor=handoff.get("single_writer_executor"),
+ route_id=handoff.get("route_id"),
+ automation_run_id=handoff.get("automation_run_id"),
+ receipt_persisted=handoff.get("receipt_persisted"),
+ runtime_closure_verified=False,
)
handoff["execution_priority"] = execution_priority
@@ -292,6 +542,8 @@ async def _try_auto_repair_background(
action_detail=(
"controlled_check_mode_queued"
if queued
+ else "controlled_domain_route_pending_receipt"
+ if handoff.get("single_writer_executor") == "Agent99"
else "controlled_queue_blocked"
),
success=queued,
@@ -304,13 +556,22 @@ async def _try_auto_repair_background(
"schema_version": handoff.get("schema_version"),
"status": handoff.get("status"),
"automation_run_id": handoff.get("automation_run_id"),
+ "trace_id": handoff.get("trace_id"),
+ "work_item_id": handoff.get("work_item_id"),
+ "idempotency_key": handoff.get("idempotency_key"),
"single_writer_executor": handoff.get("single_writer_executor"),
"execution_priority": execution_priority,
- "side_effect_performed": False,
+ "side_effect_performed": bool(handoff.get("side_effect_performed")),
+ "dispatch_receipt": handoff.get("dispatch_receipt"),
+ "correlated_receipt": handoff.get("correlated_receipt"),
+ "runtime_closure_verified": False,
"safe_next_action": (
- "awooop_ansible_check_mode_worker_claims_candidate"
- if queued
- else "repair_candidate_backfill_or_playbook_coverage_gap"
+ handoff.get("safe_next_action")
+ or (
+ "awooop_ansible_check_mode_worker_claims_candidate"
+ if queued
+ else "repair_candidate_backfill_or_playbook_coverage_gap"
+ )
),
},
)
@@ -321,7 +582,7 @@ async def _try_auto_repair_background(
queued=queued,
status=handoff.get("status"),
automation_run_id=handoff.get("automation_run_id"),
- side_effect_performed=False,
+ side_effect_performed=bool(handoff.get("side_effect_performed")),
)
return handoff
@@ -1064,9 +1325,12 @@ class SignalPayload(BaseModel):
- AlertPayload: 同步處理，含 LLM 分析
"""
- source: str = Field(
+ source: SignalSource = Field(
...,
- description="訊號來源 (prometheus, grafana, k8s-events, signoz)",
+ description=(
+ "受控訊號來源（例如 prometheus, signoz, journal, "
+ "node-exporter, sensor-probe）"
+ ),
)
alert_name: str = Field(
@@ -1313,6 +1577,15 @@ async def receive_alert(
target_resource=alert.target_resource,
namespace=alert.namespace,
risk_level=updated_approval.risk_level.value,
+ source_alert_id=alert_id,
+ source_alertname=str(
+ (alert.labels or {}).get("alertname")
+ or alert.alert_type
+ ),
+ source_severity=alert.severity,
+ source_message=alert.message,
+ source_labels=alert.labels or {},
+ source_fingerprint=fingerprint,
)
# =================================================================
# 2026-03-27 ogt: 收斂告警不重複發送 Telegram，只更新 hit_count
@@ -1448,6 +1721,7 @@ async def receive_alert(
alert_namespace=alert.namespace,
alertname=_alertname_cs1,
alert_category=get_incident_type(_alertname_cs1),
+ target_resource=alert.target_resource,
)
_matched_playbook_id_cs1 = await resolve_playbook_id_for_alert(
alertname=_alertname_cs1,
@@ -1486,7 +1760,7 @@ async def receive_alert(
metadata=_approval_metadata_cs1,
matched_playbook_id=_matched_playbook_id_cs1,
)
- suggested_action = analysis_result.kubectl_command
+ suggested_action = _guarded_action_cs1.action
else:
# LLM 失敗，降級使用靜態分析
logger.warning(
@@ -1565,6 +1839,13 @@ async def receive_alert(
target_resource=alert.target_resource,
namespace=alert.namespace,
risk_level=_cs1_risk_level.value,
+ source_alert_id=alert_id,
+ source_alertname=_alertname_cs1,
+ source_severity=alert.severity,
+ source_message=alert.message,
+ source_labels=alert.labels or {},
+ source_fingerprint=fingerprint,
+ alert_category=_cs1_alert_category,
)
logger.info(
@@ -1830,6 +2111,7 @@ async def _process_new_alert_background(
alert_namespace=namespace,
alertname=alertname,
alert_category=alert_category,
+ target_resource=target_resource,
)
if _guarded_action_cs2.blocked:
rule_action = _guarded_action_cs2.action
@@ -1968,6 +2250,16 @@ async def _process_new_alert_background(
target_resource=target_resource,
namespace=namespace,
risk_level=rule_risk.value,
+ source_alert_id=alert_id,
+ source_alertname=alertname,
+ source_severity=severity,
+ source_message=message,
+ source_labels=traced_alert_labels,
+ source_annotations=alert_context.get("annotations", {}),
+ source_fingerprint=fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=str(alert_context.get("source_url") or "") or None,
)
elif not _is_heartbeat:
from src.repositories.alert_operation_log_repository import get_alert_operation_log_repository
@@ -2059,6 +2351,7 @@ async def _process_new_alert_background(
alert_namespace=namespace,
alertname=alertname,
alert_category=alert_category,
+ target_resource=target_resource,
)
_matched_playbook_id_cs3 = await resolve_playbook_id_for_alert(
rule_id=str(rule_response.get("rule_id", "")),
@@ -2200,6 +2493,16 @@ async def _process_new_alert_background(
target_resource=target_resource,
namespace=namespace,
risk_level=risk_level.value,
+ source_alert_id=alert_id,
+ source_alertname=alertname,
+ source_severity=severity,
+ source_message=message,
+ source_labels=traced_alert_labels,
+ source_annotations=alert_context.get("annotations", {}),
+ source_fingerprint=fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=str(alert_context.get("source_url") or "") or None,
)
elif not _is_heartbeat:
from src.repositories.alert_operation_log_repository import get_alert_operation_log_repository
@@ -2464,6 +2767,16 @@ async def _process_new_alert_background(
target_resource=target_resource,
namespace=namespace,
risk_level=fallback_create.risk_level.value,
+ source_alert_id=alert_id,
+ source_alertname=alertname,
+ source_severity=severity,
+ source_message=message,
+ source_labels=traced_alert_labels,
+ source_annotations=alert_context.get("annotations", {}),
+ source_fingerprint=fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=str(alert_context.get("source_url") or "") or None,
)
if _controlled_handoff.get("queued") is True:
telegram_root_cause = (
@@ -2959,24 +3272,60 @@ async def alertmanager_webhook(
converged=True,
)
- # Agent99 only receives the parent route after grouping. The bridge adds a
- # Redis NX fingerprint lock so concurrent duplicate deliveries cannot race
- # past this point and launch duplicate remediation.
- background_tasks.add_task(
- bridge_alertmanager_to_agent99,
- alert_id=alert_id,
+ # Backup/restore signals have one production dispatch owner: the raw
+ # Alertmanager ingress. Run it after storm grouping, but before generic
+ # approval convergence and the legacy TYPE-1 "notify then resolve" branch.
+ # The worker first persists/binds a canonical Incident and then sends one
+ # read-only BackupCheck through the PostgreSQL idempotency ledger. Telegram
+ # may only project that receipt and cannot dispatch a second copy.
+ backup_restore_route = resolve_agent99_durable_route(
alertname=alertname,
severity=severity,
- namespace=namespace,
target_resource=target_resource,
+ namespace=namespace,
message=message,
labels=dict(alert.labels) if alert.labels else {},
annotations=dict(alert.annotations) if alert.annotations else {},
- fingerprint=fingerprint,
- alert_category=alert_category,
- notification_type=notification_type,
- source_url=alert.generatorURL,
)
+ if (
+ isinstance(backup_restore_route, dict)
+ and backup_restore_route.get("kind") == "backup_health"
+ and backup_restore_route.get("suggested_mode") == "BackupCheck"
+ ):
+ background_tasks.add_task(
+ process_backup_restore_alertmanager_signal,
+ project_id="awoooi",
+ alert_id=alert_id,
+ alertname=alertname,
+ severity=severity,
+ namespace=namespace,
+ target_resource=target_resource,
+ message=message,
+ labels=dict(alert.labels) if alert.labels else {},
+ annotations=dict(alert.annotations) if alert.annotations else {},
+ source_fingerprint=fingerprint,
+ source_event_fingerprint=str(alert.fingerprint or fingerprint),
+ source_started_at=str(alert.startsAt or ""),
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=alert.generatorURL,
+ )
+ record_alert_chain_success("alertmanager")
+ _record_webhook_metric("success")
+ return AlertResponse(
+ success=True,
+ message=(
+ "Backup/restore signal queued for durable read-only BackupCheck "
+ "dispatch and verifier readback"
+ ),
+ alert_id=alert_id,
+ approval_created=False,
+ )
+
+ # Agent99 dispatch is deliberately deferred until an incident exists.
+ # _try_auto_repair_background is the sole owner and first reserves the
+ # deterministic PostgreSQL run/idempotency identity. Telegram only reads
+ # that durable receipt and never launches a second Recover.
try:
service = get_approval_service()
@@ -3019,6 +3368,18 @@ async def alertmanager_webhook(
target_resource=target_resource,
namespace=namespace,
risk_level=updated_approval.risk_level.value,
+ source_alert_id=alert_id,
+ source_alertname=alertname,
+ source_severity=severity,
+ source_message=message,
+ source_labels=dict(alert.labels) if alert.labels else {},
+ source_annotations=(
+ dict(alert.annotations) if alert.annotations else {}
+ ),
+ source_fingerprint=fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=alert.generatorURL,
)
# 2026-03-27 ogt: 收斂告警不重複發送 Telegram，只更新 hit_count
# 用戶可在 UI 查看聚合次數，避免 Telegram 洗版
@@ -3200,6 +3561,7 @@ async def alertmanager_webhook(
"fingerprint": fingerprint,
"message": message,
"annotations": dict(alert.annotations) if alert.annotations else {},
+ "source_url": alert.generatorURL or "",
"metrics": {},
"labels": alert.labels,
}
diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py
index 71926624a..aced8db6a 100644
--- a/apps/api/src/core/config.py
+++ b/apps/api/src/core/config.py
@@ -976,6 +976,39 @@ class Settings(BaseSettings):
default=3.0,
description="Timeout for fail-open Agent99 SRE alert relay POSTs.",
)
+ ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL: bool = Field(
+ default=True,
+ description=(
+ "Enable bounded durable BackupCheck replay for legacy backup/restore "
+ "Telegram receipt cards when a trusted Agent99 relay is configured."
+ ),
+ )
+ BACKUP_RESTORE_LEGACY_BACKFILL_STARTUP_SLEEP_SECONDS: float = Field(
+ default=5.0,
+ ge=0.0,
+ le=300.0,
+ description="Startup delay for the read-only legacy backup-card replay loop.",
+ )
+ BACKUP_RESTORE_LEGACY_BACKFILL_INTERVAL_SECONDS: float = Field(
+ default=15.0,
+ ge=5.0,
+ le=3600.0,
+ description="Interval between bounded legacy backup-card replay batches.",
+ )
+ BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT: int = Field(
+ default=20,
+ ge=1,
+ le=20,
+ description="Maximum legacy backup cards reserved and replayed per tick.",
+ )
+ BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS: int = Field(
+ default=100,
+ ge=1,
+ le=100,
+ description=(
+ "Maximum rows retained in one durable legacy replay snapshot cohort."
+ ),
+ )
# 2026-04-24 Claude Sonnet 4.6 (ADR-095 WS4): Hermes NL 自然語言閘道
# false=不啟用（預設），true=啟用 @mention 問答（需 ANTHROPIC_API_KEY）
HERMES_NL_ENABLED: bool = Field(
diff --git a/apps/api/src/db/awooop_models.py b/apps/api/src/db/awooop_models.py
index 3c132c9b8..6a60efe29 100644
--- a/apps/api/src/db/awooop_models.py
+++ b/apps/api/src/db/awooop_models.py
@@ -329,6 +329,8 @@ class AwoooPRunState(Base):
),
Index("idx_run_state_pending", "project_id", "created_at",
postgresql_where=text("state = 'pending' AND lease_until IS NULL")),
+ Index("idx_run_state_retry_due", "project_id", "next_attempt_at",
+ postgresql_where=text("state = 'pending' AND next_attempt_at IS NOT NULL")),
Index("idx_run_state_stale", "lease_until",
postgresql_where=text("state = 'running' AND lease_until IS NOT NULL")),
Index("idx_run_state_project_timeline", "project_id", "created_at"),
@@ -343,6 +345,7 @@ class AwoooPRunState(Base):
agent_id: Mapped[str] = mapped_column(String(128), nullable=False)
state: Mapped[str] = mapped_column(String(32), nullable=False, default="pending")
lease_until: Mapped[datetime | None] = mapped_column(nullable=True)
+ next_attempt_at: Mapped[datetime | None] = mapped_column(nullable=True)
heartbeat_at: Mapped[datetime | None] = mapped_column(nullable=True)
worker_id: Mapped[str | None] = mapped_column(String(128), nullable=True)
attempt_count: Mapped[int] = mapped_column(SmallInteger, nullable=False, default=0)
diff --git a/apps/api/src/db/base.py b/apps/api/src/db/base.py
index b7df07e39..652f310b5 100644
--- a/apps/api/src/db/base.py
+++ b/apps/api/src/db/base.py
@@ -368,6 +368,22 @@ async def _run_init_db_ddl(conn: AsyncConnection) -> None:
else:
raise
+ # Agent99 dispatch retry scheduling is read by the worker immediately
+ # after ``init_db`` returns. Normal CD does not apply arbitrary migration
+ # files, so this nullable compatibility column and bounded partial index
+ # must commit before any serving/background query can reference them.
+ # Keep the audited up/down migration as the explicit schema record.
+ async with conn.begin():
+ await conn.execute(text("""
+ ALTER TABLE awooop_run_state
+ ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMP NULL;
+ """))
+ await conn.execute(text(
+ "CREATE INDEX IF NOT EXISTS idx_run_state_retry_due "
+ "ON awooop_run_state(project_id, next_attempt_at) "
+ "WHERE state = 'pending' AND next_attempt_at IS NOT NULL;"
+ ))
+
async with conn.begin():
# 2026-04-02 Claude Code: 確保 risklevel enum 包含 'high' 值
# Phase 23 新增，避免舊 DB 缺少此值導致 InvalidTextRepresentation
diff --git a/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py
new file mode 100644
index 000000000..456ea2cbd
--- /dev/null
+++ b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py
@@ -0,0 +1,626 @@
+"""Reconcile durable Agent99 dispatches into verifier and learning closure."""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+from collections.abc import Awaitable, Callable
+from typing import Any
+
+import structlog
+from sqlalchemy import select, text
+
+from src.db.base import get_db_context
+from src.db.models import PlaybookRecord
+from src.models.knowledge import (
+ EntrySource,
+ EntryStatus,
+ EntryType,
+ KnowledgeEntryCreate,
+)
+from src.services.agent99_controlled_dispatch_ledger import (
+ Agent99DispatchIdentity,
+ get_agent99_dispatch_ledger,
+ record_agent99_learning_writeback,
+)
+from src.services.agent99_public_receipts import (
+ extract_agent99_outcome_evidence_refs,
+ sanitize_agent99_public_receipt_refs,
+)
+from src.services.agent99_sre_bridge import (
+ bridge_alertmanager_to_agent99,
+ read_agent99_sre_outcome,
+)
+from src.services.knowledge_service import get_knowledge_service
+from src.services.telegram_gateway import get_telegram_gateway
+from src.utils.timezone import now_taipei
+
+logger = structlog.get_logger(__name__)
+
+INTERVAL_SECONDS = 15
+RECONCILE_LIMIT = 20
+LEGACY_BACKFILL_LIMIT = 5
+LEGACY_COLD_START_INCIDENT_ID = "INC-20260711-11C751"
+
+LegacyIncidentFetcher = Callable[..., Awaitable[list[dict[str, Any]]]]
+Agent99Dispatcher = Callable[..., Awaitable[dict[str, Any]]]
+
+
+async def _fetch_legacy_cold_start_failures(
+ *,
+ project_id: str,
+ limit: int,
+) -> list[dict[str, Any]]:
+ """Read unresolved legacy Ansible failures that belong to Agent99.
+
+ This includes the production incident that exposed the invalid Kubernetes
+ namespace route. The query is bounded and read-only; PG/Redis dispatch
+ idempotency remains the sole authority for whether transport may run.
+ """
+
+ async with get_db_context(project_id) as db:
+ await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
+ result = await db.execute(
+ text("""
+ SELECT
+ incident.incident_id,
+ coalesce(incident.project_id, :project_id) AS project_id,
+ coalesce(incident.alertname, 'ColdStartGateBlocked') AS alertname,
+ coalesce(incident.severity::text, 'low') AS severity,
+ coalesce(incident.alert_category, 'general') AS alert_category,
+ coalesce(incident.notification_type, 'legacy_backfill')
+ AS notification_type,
+ approval_identity.id::text AS approval_id,
+ coalesce(approval_identity.fingerprint, '')
+ AS approval_fingerprint,
+ lower(approval_state.status::text) AS approval_status
+ FROM incidents incident
+ JOIN LATERAL (
+ SELECT
+ candidate.id,
+ candidate.fingerprint
+ FROM approval_records candidate
+ WHERE candidate.incident_id = incident.incident_id
+ ORDER BY candidate.created_at ASC, candidate.id ASC
+ LIMIT 1
+ ) approval_identity ON TRUE
+ JOIN LATERAL (
+ SELECT candidate.status
+ FROM approval_records candidate
+ WHERE candidate.incident_id = incident.incident_id
+ ORDER BY candidate.updated_at DESC, candidate.created_at DESC
+ LIMIT 1
+ ) approval_state ON TRUE
+ WHERE (incident.project_id = :project_id OR incident.project_id IS NULL)
+ AND incident.resolved_at IS NULL
+ AND upper(coalesce(incident.status::text, ''))
+ NOT IN ('RESOLVED', 'CLOSED')
+ AND (
+ incident.incident_id = :exact_incident_id
+ OR lower(coalesce(incident.alertname, '')) ~
+ 'cold[-_ ]?start|reboot[-_ ]?auto[-_ ]?recovery'
+ OR lower(CAST(incident.signals AS text)) ~
+ 'cold[-_ ]?start|reboot[-_ ]?auto[-_ ]?recovery'
+ )
+ AND (
+ lower(approval_state.status::text) = 'execution_failed'
+ OR EXISTS (
+ SELECT 1
+ FROM automation_operation_log legacy
+ WHERE coalesce(
+ legacy.incident_id::text,
+ legacy.input ->> 'incident_id'
+ ) = incident.incident_id
+ AND legacy.input ->> 'executor' = 'ansible'
+ AND lower(coalesce(legacy.status, ''))
+ IN ('failed', 'error')
+ )
+ )
+ ORDER BY
+ (incident.incident_id = :exact_incident_id) DESC,
+ incident.updated_at DESC
+ LIMIT :limit
+ """),
+ {
+ "project_id": project_id,
+ "exact_incident_id": LEGACY_COLD_START_INCIDENT_ID,
+ "limit": max(1, min(int(limit), 20)),
+ },
+ )
+ return [dict(row) for row in result.mappings().all()]
+
+
+def _legacy_source_fingerprint(row: dict[str, Any]) -> str:
+ """Use immutable incident identity; approval rows are metadata only."""
+
+ canonical = ":".join([
+ "legacy-agent99-cold-start-v1",
+ str(row.get("project_id") or "awoooi"),
+ str(row.get("incident_id") or ""),
+ "agent99:host_recovery:Recover",
+ "cold-start-gate",
+ ])
+ return f"legacy-cold-start:{hashlib.sha256(canonical.encode()).hexdigest()}"
+
+
+async def backfill_legacy_agent99_dispatches_once(
+ *,
+ project_id: str = "awoooi",
+ limit: int = LEGACY_BACKFILL_LIMIT,
+ fetcher: LegacyIncidentFetcher | None = None,
+ dispatcher: Agent99Dispatcher | None = None,
+) -> dict[str, int]:
+ """Route old generic-Ansible failures into the fenced Agent99 chain."""
+
+ fetch = fetcher or _fetch_legacy_cold_start_failures
+ dispatch = dispatcher or bridge_alertmanager_to_agent99
+ rows = await fetch(project_id=project_id, limit=limit)
+ counters = {
+ "scanned": len(rows),
+ "dispatch_performed": 0,
+ "deduplicated": 0,
+ "retryable": 0,
+ "blocked": 0,
+ }
+ for row in rows:
+ incident_id = str(row.get("incident_id") or "").strip()
+ if not incident_id:
+ counters["blocked"] += 1
+ continue
+ result = await dispatch(
+ alert_id=f"legacy-agent99-backfill-{incident_id}",
+ alertname=str(row.get("alertname") or "ColdStartGateBlocked"),
+ severity=str(row.get("severity") or "low"),
+ namespace="host_recovery",
+ target_resource="cold-start-gate",
+ message=(
+ "Legacy generic Ansible execution_failed rerouted to the "
+ "bounded Agent99 cold-start verifier chain."
+ ),
+ labels={
+ "event_type": "legacy_cold_start_agent99_backfill",
+ "legacy_executor": "ansible",
+ "legacy_status": str(row.get("approval_status") or "execution_failed"),
+ "legacy_approval_fingerprint": str(
+ row.get("approval_fingerprint") or ""
+ )[:64],
+ },
+ annotations={
+ "summary": "Bounded Agent99 recovery backfill for legacy failure",
+ },
+ fingerprint=_legacy_source_fingerprint(row),
+ alert_category=str(row.get("alert_category") or "general"),
+ notification_type=str(
+ row.get("notification_type") or "legacy_backfill"
+ ),
+ project_id=str(row.get("project_id") or project_id),
+ incident_id=incident_id,
+ approval_id=str(row.get("approval_id") or ""),
+ route_id="agent99:host_recovery:Recover",
+ work_item_id=(
+ f"agent99-dispatch:{project_id}:{incident_id}:legacy-cold-start"
+ ),
+ )
+ if result.get("dispatchPerformed") is True:
+ counters["dispatch_performed"] += 1
+ elif result.get("status") == "deduplicated":
+ counters["deduplicated"] += 1
+ elif result.get("status") == "retryable":
+ counters["retryable"] += 1
+ else:
+ counters["blocked"] += 1
+ return counters
+
+
+def _playbook_id(identity: Agent99DispatchIdentity) -> str:
+ digest = hashlib.sha256(
+ f"{identity.project_id}:{identity.route_id}".encode()
+ ).hexdigest()[:20]
+ return f"PB-A99-{digest}"
+
+
+async def _ensure_playbook_trust(
+ identity: Agent99DispatchIdentity,
+ *,
+ mode: str,
+) -> str | None:
+ playbook_id = _playbook_id(identity)
+ run_tag = f"agent99_run:{identity.run_id}"
+ now = now_taipei()
+ try:
+ async with get_db_context(identity.project_id) as db:
+ selected = await db.execute(
+ select(PlaybookRecord)
+ .where(
+ PlaybookRecord.playbook_id == playbook_id,
+ PlaybookRecord.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ playbook = selected.scalar_one_or_none()
+ if playbook is None:
+ playbook = PlaybookRecord(
+ playbook_id=playbook_id,
+ project_id=identity.project_id,
+ name=f"Agent99 {mode or 'Status'}: {identity.route_id}",
+ description=(
+ "Verified Agent99 controlled route with independent "
+ "outcome readback and same-run learning receipts."
+ ),
+ status="approved",
+ source="ai_extracted",
+ symptom_pattern={
+ "alert_names": [],
+ "affected_services": ["cold-start-gate"],
+ "severity_range": ["P0", "P1", "P2"],
+ "label_patterns": {},
+ "keywords": ["agent99", "cold-start", mode],
+ },
+ repair_steps=[{
+ "step_number": 1,
+ "action_type": "agent99_mode",
+ "command": mode,
+ "expected_result": "independent verifier passes",
+ "rollback_command": "Agent99 Status (no-write)",
+ "requires_approval": False,
+ "risk_level": "LOW",
+ }],
+ estimated_duration_minutes=10,
+ source_incident_ids=[identity.incident_id],
+ version=1,
+ ai_confidence=1.0,
+ success_count=1,
+ failure_count=0,
+ last_used_at=now,
+ trust_score=0.37,
+ approved_by="agent99_independent_verifier",
+ approved_at=now,
+ tags=["agent99", "controlled_dispatch", run_tag],
+ notes="No raw logs or secret values are persisted.",
+ requires_approval_level="auto",
+ stateful_targets=[],
+ requires_pre_backup=False,
+ review_required=False,
+ created_at=now,
+ updated_at=now,
+ )
+ db.add(playbook)
+ else:
+ tags = list(playbook.tags or [])
+ incidents = list(playbook.source_incident_ids or [])
+ if run_tag not in tags:
+ tags.append(run_tag)
+ playbook.success_count = int(playbook.success_count or 0) + 1
+ playbook.trust_score = min(
+ 1.0,
+ 0.9 * float(playbook.trust_score or 0.3) + 0.1,
+ )
+ playbook.version = int(playbook.version or 0) + 1
+ if identity.incident_id not in incidents:
+ incidents.append(identity.incident_id)
+ playbook.tags = tags
+ playbook.source_incident_ids = incidents
+ playbook.last_used_at = now
+ playbook.updated_at = now
+ await db.flush()
+ await db.refresh(playbook)
+ return f"playbooks:{playbook.playbook_id}:version:{playbook.version}"
+ except Exception as exc:
+ logger.warning(
+ "agent99_playbook_trust_writeback_failed",
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return None
+
+
+async def _ensure_km_writeback(
+ identity: Agent99DispatchIdentity,
+ *,
+ mode: str,
+) -> str | None:
+ knowledge = get_knowledge_service()
+ try:
+ entry = await knowledge.create_entry(
+ KnowledgeEntryCreate(
+ title=f"Agent99 verified recovery {identity.incident_id}",
+ content=(
+ f"Run {identity.run_id} on route {identity.route_id} "
+ f"completed Agent99 mode {mode or 'Status'} with independent "
+ "verifier success. Raw logs and secret values are not stored."
+ ),
+ entry_type=EntryType.INCIDENT_CASE,
+ category="AI自動化/Agent99受控修復",
+ tags=[
+ "agent99",
+ "controlled_dispatch",
+ f"automation_run_id:{identity.run_id}",
+ ],
+ source=EntrySource.AI_EXTRACTED,
+ status=EntryStatus.REVIEW,
+ related_incident_id=identity.incident_id,
+ related_playbook_id=_playbook_id(identity),
+ related_approval_id=identity.approval_id or None,
+ path_type=f"agent99:{identity.run_id}",
+ created_by="agent99_controlled_dispatch_reconciler",
+ ),
+ project_id=identity.project_id,
+ schedule_embedding=False,
+ )
+ embedded = await knowledge.ensure_entry_embedding(
+ entry.id,
+ entry.title,
+ entry.content,
+ project_id=identity.project_id,
+ )
+ if not embedded:
+ return None
+ return f"knowledge_entries:{entry.id}:embedding_verified"
+ except Exception as exc:
+ logger.warning(
+ "agent99_km_writeback_failed",
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return None
+
+
+async def _ensure_telegram_receipt(
+ identity: Agent99DispatchIdentity,
+ *,
+ mode: str,
+) -> str | None:
+ try:
+ response = await get_telegram_gateway().send_controlled_apply_result_receipt(
+ automation_run_id=str(identity.run_id),
+ incident_id=identity.incident_id,
+ catalog_id=f"agent99:{identity.route_id}",
+ apply_op_id=str(identity.run_id),
+ playbook_path=f"agent99://{mode or 'Status'}",
+ verification_result="success",
+ returncode=0,
+ duration_ms=0,
+ verifier_written=True,
+ # This receipt is one prerequisite for learning writeback; it must
+ # not claim the still-pending terminal ledger step is complete.
+ learning_written=False,
+ project_id=identity.project_id,
+ )
+ result = response.get("result") if isinstance(response, dict) else None
+ message_id = (
+ str(result.get("message_id") or "")
+ if isinstance(result, dict)
+ else ""
+ )
+ if (
+ not isinstance(response, dict)
+ or response.get("_awooop_outbound_mirror_acknowledged") is not True
+ or not message_id
+ ):
+ return None
+ return f"telegram_outbound:{message_id}:durable_ack"
+ except Exception as exc:
+ logger.warning(
+ "agent99_telegram_writeback_failed",
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return None
+
+
+async def _ensure_dr_scorecard(
+ identity: Agent99DispatchIdentity,
+ *,
+ evidence_refs: dict[str, str],
+) -> str | None:
+ if "backup_health" not in identity.route_id:
+ return "not_required"
+ try:
+ entry = await get_knowledge_service().create_entry(
+ KnowledgeEntryCreate(
+ title=f"DR scorecard {identity.incident_id}",
+ content=(
+ "Agent99 BackupCheck verified freshness, offsite, escrow, "
+ "restore-drill, and source-resolution receipts. "
+ + "; ".join(
+ f"{key}={value}"
+ for key, value in sorted(evidence_refs.items())
+ )
+ ),
+ entry_type=EntryType.BEST_PRACTICE,
+ category="資料保護/DR Scorecard",
+ tags=[
+ "backup_restore",
+ "dr_scorecard",
+ f"automation_run_id:{identity.run_id}",
+ ],
+ source=EntrySource.AI_EXTRACTED,
+ status=EntryStatus.REVIEW,
+ related_incident_id=identity.incident_id,
+ related_playbook_id=_playbook_id(identity),
+ related_approval_id=identity.approval_id or None,
+ path_type=f"agent99-dr:{identity.run_id}",
+ created_by="agent99_controlled_dispatch_reconciler",
+ ),
+ project_id=identity.project_id,
+ )
+ return f"knowledge_entries:{entry.id}:dr_scorecard"
+ except Exception as exc:
+ logger.warning(
+ "agent99_dr_scorecard_writeback_failed",
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return None
+
+
+async def _finalize_learning(
+ identity: Agent99DispatchIdentity,
+ *,
+ mode: str,
+ evidence_refs: dict[str, str],
+) -> dict[str, Any]:
+ ledger = get_agent99_dispatch_ledger()
+ checkpoint = await ledger.record_learning_asset_acks(
+ identity=identity,
+ receipt_refs={},
+ )
+ if checkpoint.get("receipt_persisted") is not True:
+ return {
+ "status": "learning_asset_checkpoint_unavailable",
+ "closed": False,
+ }
+ if checkpoint.get("runtime_closure_verified") is True:
+ return checkpoint
+ receipt_refs = dict(checkpoint.get("learning_receipt_refs") or {})
+
+ async def ensure_checkpointed(
+ key: str,
+ writer: Callable[[], Awaitable[str | None]],
+ ) -> bool:
+ nonlocal receipt_refs
+ if key in receipt_refs:
+ return True
+ receipt = await writer()
+ if not receipt:
+ return False
+ persisted = await ledger.record_learning_asset_acks(
+ identity=identity,
+ receipt_refs={key: receipt},
+ )
+ if persisted.get("receipt_persisted") is not True:
+ return False
+ receipt_refs = dict(persisted.get("learning_receipt_refs") or {})
+ return key in receipt_refs
+
+ assets = [
+ (
+ "playbook_trust_writeback_ack_id",
+ lambda: _ensure_playbook_trust(identity, mode=mode),
+ ),
+ (
+ "km_writeback_ack_id",
+ lambda: _ensure_km_writeback(identity, mode=mode),
+ ),
+ (
+ "telegram_lifecycle_receipt_id",
+ lambda: _ensure_telegram_receipt(identity, mode=mode),
+ ),
+ ]
+ if "backup_health" in identity.route_id:
+ assets.append((
+ "dr_scorecard_writeback_ack_id",
+ lambda: _ensure_dr_scorecard(
+ identity,
+ evidence_refs=evidence_refs,
+ ),
+ ))
+ for key, writer in assets:
+ if not await ensure_checkpointed(key, writer):
+ return {
+ "status": "learning_assets_pending",
+ "missing_asset": key,
+ "closed": False,
+ }
+ return await record_agent99_learning_writeback(
+ identity=identity,
+ receipt_refs=receipt_refs,
+ )
+
+
+async def reconcile_agent99_controlled_dispatches_once(
+ *,
+ project_id: str = "awoooi",
+ limit: int = RECONCILE_LIMIT,
+) -> dict[str, int]:
+ ledger = get_agent99_dispatch_ledger()
+ items = await ledger.list_reconcilable(project_id=project_id, limit=limit)
+ counters = {"scanned": len(items), "verifier_written": 0, "closed": 0}
+ for item in items:
+ identity = item["identity"]
+ receipt = item.get("receipt") if isinstance(item.get("receipt"), dict) else {}
+ mode = identity.route_id.rsplit(":", 1)[-1]
+ verifier_receipt = (
+ receipt.get("verifier")
+ if isinstance(receipt.get("verifier"), dict)
+ else {}
+ )
+ evidence_refs = {
+ **sanitize_agent99_public_receipt_refs(
+ verifier_receipt.get("evidence_refs")
+ if isinstance(verifier_receipt.get("evidence_refs"), dict)
+ else {}
+ )
+ }
+ if receipt.get("post_verifier_passed") is not True:
+ outcome = await asyncio.to_thread(
+ read_agent99_sre_outcome,
+ str(identity.run_id),
+ )
+ if not isinstance(outcome, dict):
+ continue
+ mode = str(outcome.get("mode") or mode)
+ outcome_contract = (
+ outcome.get("outcome")
+ if isinstance(outcome.get("outcome"), dict)
+ else {}
+ )
+ verified_at = str(outcome_contract.get("verifiedAt") or "unknown")
+ generic_refs = {
+ "agent99_outcome_receipt_id": (
+ f"agent99-relay:outcome:{identity.run_id}"
+ ),
+ "post_verifier_evidence_ref": (
+ f"agent99:{mode}:verified:{verified_at}"
+ ),
+ "source_event_evidence_ref": (
+ f"incident:{identity.incident_id}:source-resolved:{verified_at}"
+ ),
+ }
+ evidence_refs = extract_agent99_outcome_evidence_refs(
+ outcome,
+ generic_refs,
+ )
+ verifier = await ledger.record_verifier(
+ identity=identity,
+ outcome_receipt=outcome,
+ evidence_refs=evidence_refs,
+ )
+ if verifier.get("receipt_persisted") is not True:
+ continue
+ counters["verifier_written"] += 1
+ receipt = verifier
+ if receipt.get("post_verifier_passed") is True:
+ closure = await _finalize_learning(
+ identity,
+ mode=mode,
+ evidence_refs=evidence_refs,
+ )
+ if closure.get("runtime_closure_verified") is True:
+ counters["closed"] += 1
+ return counters
+
+
+async def run_agent99_controlled_dispatch_reconciler_loop() -> None:
+ while True:
+ try:
+ backfill = await backfill_legacy_agent99_dispatches_once()
+ if backfill["scanned"]:
+ logger.info("agent99_legacy_failure_backfill_tick", **backfill)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.warning(
+ "agent99_legacy_failure_backfill_tick_failed",
+ error=str(exc),
+ )
+ try:
+ await reconcile_agent99_controlled_dispatches_once()
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.warning(
+ "agent99_controlled_dispatch_reconciler_tick_failed",
+ error=str(exc),
+ )
+ await asyncio.sleep(INTERVAL_SECONDS)
diff --git a/apps/api/src/jobs/backup_restore_legacy_backfill_job.py b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py
new file mode 100644
index 000000000..df09f2067
--- /dev/null
+++ b/apps/api/src/jobs/backup_restore_legacy_backfill_job.py
@@ -0,0 +1,2004 @@
+"""Bounded durable replay for legacy backup/restore Telegram cards.
+
+The historical card is only a source receipt. This worker reserves each row
+in the RLS-protected AwoooP run/idempotency tables, binds a deterministic
+Incident/work item from ``message_id`` plus the persisted source fingerprint,
+and dispatches the existing read-only ``BackupCheck`` producer. Telegram is
+never a dispatch owner in this path.
+
+Safety boundary: this worker cannot run backup/restore, delete or prune data,
+change retention, write escrow markers, or read secrets.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import hashlib
+import json
+import urllib.parse
+from collections.abc import Awaitable, Callable, Mapping
+from dataclasses import dataclass
+from datetime import UTC, datetime
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
+
+import structlog
+from sqlalchemy import text
+
+from src.core.config import settings
+from src.core.context import clear_project_context, set_project_context
+from src.db.base import get_db_context
+from src.services.backup_restore_alertmanager_ingress import (
+ BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ BACKUP_RESTORE_PROHIBITED_ACTIONS,
+ build_backup_restore_ingress_owner,
+ process_backup_restore_alertmanager_signal,
+)
+
+logger = structlog.get_logger(__name__)
+
+BACKFILL_SCHEMA_VERSION = "backup_restore_legacy_backfill_v1"
+BACKFILL_CURSOR_SCHEMA_VERSION = "backup_restore_legacy_backfill_cursor_v1"
+BACKFILL_RESULT_SCHEMA_VERSION = "backup_restore_legacy_backfill_result_v1"
+BACKFILL_CURSOR_AGENT_ID = "backup_restore_legacy_backfill_cursor"
+BACKFILL_ITEM_AGENT_ID = "backup_restore_legacy_backfill"
+BACKFILL_IDEMPOTENCY_CHANNEL = "backup_restore_backfill"
+BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal"
+BACKUP_RESTORE_LANE = "backup_restore_escrow_triage"
+PROJECTION_SCHEMA_VERSION = "telegram_agent99_dispatch_receipt_projection_v1"
+CLAIM_LEASE_SECONDS = 120
+MAX_BATCH_LIMIT = 20
+MAX_SCOPE_ROWS = 100
+REPO_KNOWN_LAN_RELAY_HOST = "192.168.0.99"
+REPO_KNOWN_LAN_RELAY_PORT = 8787
+REPO_KNOWN_LAN_RELAY_PATH = "/agent99/sre-alert"
+
+Processor = Callable[..., Awaitable[dict[str, Any]]]
+
+
+def build_backup_restore_backfill_relay_preflight(
+ *,
+ relay_url: str | None = None,
+ relay_token: str | None = None,
+) -> dict[str, Any]:
+ """Classify the public-safe relay trust mode without exposing credentials."""
+
+ target_url = str(
+ settings.AGENT99_SRE_ALERT_RELAY_URL if relay_url is None else relay_url
+ ).strip()
+ token_present = bool(
+ str(
+ settings.AGENT99_SRE_ALERT_RELAY_TOKEN
+ if relay_token is None
+ else relay_token
+ ).strip()
+ )
+ try:
+ parsed = urllib.parse.urlsplit(target_url)
+ port = parsed.port
+ except ValueError:
+ parsed = urllib.parse.SplitResult("", "", "", "", "")
+ port = None
+ structurally_valid = bool(
+ parsed.scheme in {"http", "https"}
+ and parsed.hostname
+ and parsed.username is None
+ and parsed.password is None
+ and not parsed.query
+ and not parsed.fragment
+ )
+ lan_allowlisted = bool(
+ structurally_valid
+ and parsed.scheme == "http"
+ and parsed.hostname == REPO_KNOWN_LAN_RELAY_HOST
+ and port == REPO_KNOWN_LAN_RELAY_PORT
+ and parsed.path.rstrip("/") == REPO_KNOWN_LAN_RELAY_PATH
+ )
+ https_token_endpoint = bool(
+ structurally_valid and parsed.scheme == "https" and token_present
+ )
+ auth_mode = (
+ "token"
+ if https_token_endpoint
+ else "lan_allowlist"
+ if lan_allowlisted
+ else "blocked"
+ )
+ ready = auth_mode in {"token", "lan_allowlist"}
+ return {
+ "schema_version": "backup_restore_relay_preflight_v1",
+ "status": "ready" if ready else "blocked_fail_closed",
+ "ready": ready,
+ "auth_mode": auth_mode,
+ "endpoint_class": (
+ "repo_known_private_lan"
+ if lan_allowlisted
+ else "configured_token_endpoint"
+ if https_token_endpoint
+ else "untrusted_or_missing"
+ ),
+ "http_method": "POST",
+ "token_present": token_present,
+ "stores_secret": False,
+ }
+
+
+_CURSOR_INSERT_SQL = text(
+ """
+ INSERT INTO awooop_run_state (
+ run_id, project_id, agent_id, state,
+ attempt_count, max_attempts, trigger_type, trigger_ref,
+ is_shadow, input_sha256, step_count, error_detail, timeout_at
+ ) VALUES (
+ :run_id, :project_id, :agent_id, 'waiting_tool',
+ 0, 1, 'backfill_cursor', 'backup_restore_legacy_cards',
+ FALSE, :input_sha256, 0, :error_detail, NOW() + INTERVAL '30 days'
+ )
+ ON CONFLICT (run_id) DO NOTHING
+"""
+)
+
+_CURSOR_SELECT_SQL = text(
+ """
+ SELECT run_id, state, error_detail
+ FROM awooop_run_state
+ WHERE project_id = :project_id
+ AND run_id = :run_id
+ AND agent_id = :agent_id
+ FOR UPDATE
+"""
+)
+
+_SOURCE_SNAPSHOT_SQL = text(
+ """
+ SELECT
+ m.message_id,
+ m.queued_at,
+ COUNT(*) OVER () AS source_total_in_scope,
+ FIRST_VALUE(m.message_id) OVER (
+ ORDER BY m.queued_at DESC, m.message_id DESC
+ ) AS source_upper_message_id,
+ FIRST_VALUE(m.queued_at) OVER (
+ ORDER BY m.queued_at DESC, m.message_id DESC
+ ) AS source_upper_queued_at,
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{source_refs,fingerprints,0}' AS source_fingerprint,
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,event_type}' AS event_type,
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,lane}' AS lane,
+ COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,target}',
+ ''
+ ),
+ 'backup_restore'
+ ) AS target
+ FROM awooop_outbound_message m
+ WHERE m.project_id = :project_id
+ AND m.channel_type = 'telegram'
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,event_type}' = :event_type
+ AND COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,target}',
+ ''
+ ),
+ 'backup_restore'
+ ) = 'backup_restore'
+ AND (
+ CAST(:source_upper_queued_at AS TIMESTAMPTZ) IS NULL
+ OR (
+ m.queued_at,
+ m.message_id
+ ) <= (
+ CAST(:source_upper_queued_at AS TIMESTAMPTZ),
+ CAST(:source_upper_message_id AS UUID)
+ )
+ )
+ AND (
+ COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,schema_version}',
+ ''
+ ) <> :projection_schema
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,receipt_persisted}',
+ ''
+ ) <> 'true'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,accepted}',
+ ''
+ ) <> 'true'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,inbox_triggered}',
+ ''
+ ) <> 'true'
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,run_id}',
+ ''
+ ) IS NULL
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,trace_id}',
+ ''
+ ) IS NULL
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,work_item_id}',
+ ''
+ ) IS NULL
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,kind}',
+ ''
+ ) <> 'backup_health'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,suggested_mode}',
+ ''
+ ) <> 'BackupCheck'
+ )
+ ORDER BY m.queued_at ASC, m.message_id ASC
+ LIMIT :limit
+"""
+)
+
+_ITEM_RUN_INSERT_SQL = text(
+ """
+ INSERT INTO awooop_run_state (
+ run_id, project_id, agent_id, state,
+ attempt_count, max_attempts, trigger_type, trigger_ref,
+ is_shadow, input_sha256, step_count, error_detail, timeout_at
+ ) VALUES (
+ :run_id, :project_id, :agent_id, 'waiting_tool',
+ 0, 5, 'backfill_replay', :trigger_ref,
+ FALSE, :input_sha256, 1, :error_detail, NOW() + INTERVAL '7 days'
+ )
+ ON CONFLICT (run_id) DO NOTHING
+"""
+)
+
+_ITEM_IDEMPOTENCY_INSERT_SQL = text(
+ """
+ INSERT INTO awooop_run_idempotency (
+ project_id, channel_type, provider_event_id, run_id
+ ) VALUES (
+ :project_id, :channel_type, :provider_event_id, :run_id
+ )
+ ON CONFLICT (project_id, channel_type, provider_event_id) DO NOTHING
+ RETURNING run_id
+"""
+)
+
+_CURSOR_UPDATE_SQL = text(
+ """
+ UPDATE awooop_run_state
+ SET state = :state,
+ heartbeat_at = NOW(),
+ error_detail = :error_detail,
+ completed_at = CASE WHEN :state = 'completed' THEN NOW() ELSE NULL END
+ WHERE project_id = :project_id
+ AND run_id = :run_id
+ AND agent_id = :agent_id
+"""
+)
+
+_CLAIM_SQL = text(
+ """
+ WITH claimable AS (
+ SELECT run_id
+ FROM awooop_run_state
+ WHERE project_id = :project_id
+ AND agent_id = :agent_id
+ AND state = 'waiting_tool'
+ AND (lease_until IS NULL OR lease_until < NOW())
+ AND attempt_count < max_attempts
+ ORDER BY created_at ASC, run_id ASC
+ FOR UPDATE SKIP LOCKED
+ LIMIT :limit
+ )
+ UPDATE awooop_run_state item
+ SET state = 'waiting_tool',
+ worker_id = :claim_token,
+ lease_until = NOW() + (:lease_seconds * INTERVAL '1 second'),
+ heartbeat_at = NOW(),
+ started_at = COALESCE(started_at, NOW()),
+ attempt_count = attempt_count + 1,
+ error_code = NULL
+ FROM claimable
+ WHERE item.run_id = claimable.run_id
+ AND item.project_id = :project_id
+ AND item.agent_id = :agent_id
+ RETURNING
+ item.run_id,
+ item.attempt_count,
+ item.max_attempts,
+ item.error_detail
+"""
+)
+
+_EXHAUST_EXPIRED_CLAIMS_SQL = text(
+ """
+ UPDATE awooop_run_state
+ SET state = 'failed',
+ worker_id = NULL,
+ lease_until = NULL,
+ heartbeat_at = NOW(),
+ error_code = 'E-BACKFILL-LEASE-EXHAUSTED',
+ completed_at = NOW()
+ WHERE project_id = :project_id
+ AND agent_id = :agent_id
+ AND state = 'waiting_tool'
+ AND worker_id IS NOT NULL
+ AND lease_until < NOW()
+ AND attempt_count >= max_attempts
+ AND error_code IS NULL
+ RETURNING run_id
+"""
+)
+
+_SOURCE_LOAD_SQL = text(
+ """
+ SELECT
+ m.message_id,
+ m.queued_at,
+ COALESCE(m.source_envelope, '{}'::jsonb) AS source_envelope
+ FROM awooop_outbound_message m
+ WHERE m.project_id = :project_id
+ AND m.message_id = CAST(:message_id AS UUID)
+ AND m.channel_type = 'telegram'
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,event_type}' = :event_type
+ LIMIT 1
+"""
+)
+
+_SOURCE_PROJECTION_UPDATE_SQL = text(
+ """
+ WITH active_claim AS (
+ SELECT item.run_id
+ FROM awooop_run_state item
+ WHERE item.project_id = :project_id
+ AND item.run_id = :claim_run_id
+ AND item.agent_id = :agent_id
+ AND item.state = 'waiting_tool'
+ AND item.worker_id = :claim_token
+ AND item.lease_until > clock_timestamp()
+ FOR UPDATE
+ )
+ UPDATE awooop_outbound_message m
+ SET source_envelope =
+ JSONB_SET(
+ JSONB_SET(
+ COALESCE(m.source_envelope, '{}'::jsonb),
+ '{agent99_dispatch_receipt}',
+ CAST(:projection AS jsonb),
+ TRUE
+ ),
+ '{backup_restore_backfill}',
+ CAST(:backfill_receipt AS jsonb),
+ TRUE
+ )
+ FROM active_claim
+ WHERE m.project_id = :project_id
+ AND m.message_id = CAST(:message_id AS UUID)
+ AND m.queued_at = CAST(:source_queued_at AS TIMESTAMPTZ)
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{source_refs,fingerprints,0}' = :source_fingerprint
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,event_type}' = :event_type
+ AND COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,lane}',
+ ''
+ ),
+ :default_lane
+ ) = :source_lane
+ AND COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,target}',
+ ''
+ ),
+ 'backup_restore'
+ ) = :source_target
+ AND (
+ NOT (
+ COALESCE(m.source_envelope, '{}'::jsonb)
+ ? 'agent99_dispatch_receipt'
+ )
+ OR COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,schema_version}' =
+ 'telegram_agent99_dispatch_receipt_v1'
+ OR (
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,schema_version}' =
+ :projection_schema
+ AND (
+ COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,receipt_persisted}',
+ ''
+ ) <> 'true'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,accepted}',
+ ''
+ ) <> 'true'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,inbox_triggered}',
+ ''
+ ) <> 'true'
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,run_id}',
+ ''
+ ) IS NULL
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,trace_id}',
+ ''
+ ) IS NULL
+ OR NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,work_item_id}',
+ ''
+ ) IS NULL
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,kind}',
+ ''
+ ) <> 'backup_health'
+ OR COALESCE(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,suggested_mode}',
+ ''
+ ) <> 'BackupCheck'
+ )
+ )
+ OR COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,run_id}' = :run_id
+ OR (
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{backup_restore_backfill,occurrence_id}' = :occurrence_id
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{backup_restore_backfill,source_message_id}' =
+ CAST(:message_id AS TEXT)
+ AND (
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,run_id}',
+ ''
+ ) IS NULL
+ OR COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{agent99_dispatch_receipt,run_id}' = :run_id
+ )
+ )
+ )
+ RETURNING m.message_id
+"""
+)
+
+_PROJECTION_CONFLICT_READBACK_SQL = text(
+ """
+ SELECT
+ EXISTS (
+ SELECT 1
+ FROM awooop_run_state item
+ WHERE item.project_id = :project_id
+ AND item.run_id = :claim_run_id
+ AND item.agent_id = :agent_id
+ AND item.state = 'waiting_tool'
+ AND item.worker_id = :claim_token
+ AND item.lease_until > clock_timestamp()
+ ) AS claim_active,
+ EXISTS (
+ SELECT 1
+ FROM awooop_outbound_message m
+ WHERE m.project_id = :project_id
+ AND m.message_id = CAST(:message_id AS UUID)
+ ) AS source_exists,
+ EXISTS (
+ SELECT 1
+ FROM awooop_outbound_message m
+ WHERE m.project_id = :project_id
+ AND m.message_id = CAST(:message_id AS UUID)
+ AND m.queued_at = CAST(:source_queued_at AS TIMESTAMPTZ)
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{source_refs,fingerprints,0}' = :source_fingerprint
+ AND COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,event_type}' = :event_type
+ AND COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,lane}',
+ ''
+ ),
+ :default_lane
+ ) = :source_lane
+ AND COALESCE(
+ NULLIF(
+ COALESCE(m.source_envelope, '{}'::jsonb) #>>
+ '{ai_automation_alert_card,target}',
+ ''
+ ),
+ 'backup_restore'
+ ) = :source_target
+ ) AS source_identity_matches
+"""
+)
+
+_ITEM_FINISH_SQL = text(
+ """
+ UPDATE awooop_run_state
+ SET state = :state,
+ worker_id = NULL,
+ lease_until = NULL,
+ heartbeat_at = NOW(),
+ output_sha256 = :output_sha256,
+ error_code = :error_code,
+ error_detail = :error_detail,
+ completed_at = CASE
+ WHEN :state IN ('completed', 'failed') THEN NOW()
+ ELSE NULL
+ END
+ WHERE project_id = :project_id
+ AND run_id = :run_id
+ AND agent_id = :agent_id
+ AND state = 'waiting_tool'
+ AND worker_id = :claim_token
+ RETURNING run_id
+"""
+)
+
+_ITEM_STATE_COUNTS_SQL = text(
+ """
+ SELECT
+ COUNT(*) AS item_total,
+ COUNT(*) FILTER (
+ WHERE state NOT IN ('completed', 'failed', 'cancelled', 'timeout')
+ ) AS remaining_total,
+ COUNT(*) FILTER (WHERE state = 'completed') AS completed_total,
+ COUNT(*) FILTER (
+ WHERE state IN ('failed', 'cancelled', 'timeout')
+ ) AS failed_total
+ FROM awooop_run_state
+ WHERE project_id = :project_id
+ AND agent_id = :agent_id
+"""
+)
+
+
+@dataclass(frozen=True)
+class LegacyBackupClaim:
+ run_id: UUID
+ claim_token: str
+ attempt_count: int
+ max_attempts: int
+ item: dict[str, Any]
+
+
+@dataclass(frozen=True)
+class LegacyBackupCard:
+ message_id: str
+ queued_at: str
+ source_fingerprint: str
+ event_type: str
+ lane: str
+ target: str
+ source_envelope: dict[str, Any]
+
+
+def _stable_json(value: Mapping[str, Any]) -> str:
+ return json.dumps(
+ dict(value),
+ ensure_ascii=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ )
+
+
+def _sha256(value: str) -> str:
+ return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def _json_object(value: Any) -> dict[str, Any]:
+ if isinstance(value, dict):
+ return dict(value)
+ if not value:
+ return {}
+ try:
+ parsed = json.loads(str(value))
+ except (TypeError, ValueError, json.JSONDecodeError):
+ return {}
+ return parsed if isinstance(parsed, dict) else {}
+
+
+def _row_mapping(row: Any) -> dict[str, Any]:
+ if isinstance(row, Mapping):
+ return dict(row)
+ mapping = getattr(row, "_mapping", None)
+ return dict(mapping) if isinstance(mapping, Mapping) else {}
+
+
+def _timestamp(value: Any) -> str:
+ if isinstance(value, datetime):
+ if value.tzinfo is None:
+ value = value.replace(tzinfo=UTC)
+ return value.isoformat()
+ return str(value or "").strip()
+
+
+def _database_timestamp(value: Any) -> datetime | None:
+ if value is None or str(value).strip() == "":
+ return None
+ if isinstance(value, datetime):
+ parsed = value
+ else:
+ raw = str(value).strip()
+ if raw.endswith("Z"):
+ raw = f"{raw[:-1]}+00:00"
+ try:
+ parsed = datetime.fromisoformat(raw)
+ except ValueError as exc:
+ raise ValueError("backup_backfill_source_timestamp_invalid") from exc
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=UTC)
+ return parsed
+
+
+def _source_refs(envelope: Mapping[str, Any]) -> dict[str, Any]:
+ refs = envelope.get("source_refs")
+ return dict(refs) if isinstance(refs, Mapping) else {}
+
+
+def _alert_card(envelope: Mapping[str, Any]) -> dict[str, Any]:
+ card = envelope.get("ai_automation_alert_card")
+ return dict(card) if isinstance(card, Mapping) else {}
+
+
+def _first_string(value: Any) -> str:
+ if isinstance(value, list):
+ for item in value:
+ text_value = str(item or "").strip()
+ if text_value:
+ return text_value
+ return ""
+
+
+def _cursor_run_id(project_id: str) -> UUID:
+ return uuid5(
+ NAMESPACE_URL,
+ f"{BACKFILL_CURSOR_SCHEMA_VERSION}:{project_id}",
+ )
+
+
+def build_legacy_backup_occurrence(
+ *,
+ project_id: str,
+ message_id: str,
+ source_fingerprint: str,
+ queued_at: str,
+) -> dict[str, Any]:
+ """Build the stable source occurrence, Incident, and work-item identity."""
+
+ project = str(project_id or "").strip()
+ source_message = str(message_id or "").strip()
+ fingerprint = str(source_fingerprint or "").strip()
+ queued = str(queued_at or "").strip()
+ if not project:
+ raise ValueError("backup_backfill_project_id_required")
+ if not source_message:
+ raise ValueError("backup_backfill_message_id_required")
+ if not fingerprint:
+ raise ValueError("backup_backfill_source_fingerprint_required")
+ fingerprint_hash = _sha256(fingerprint)[:16]
+ occurrence_id = f"legacy-backup:{source_message}:{fingerprint_hash}"
+ dispatch_source_fingerprint = "legacy-backup-occurrence:" + _sha256(
+ f"{project}\x1f{source_message}\x1f{fingerprint}"
+ )
+ owner = build_backup_restore_ingress_owner(
+ project_id=project,
+ source_fingerprint=dispatch_source_fingerprint,
+ source_event_fingerprint=occurrence_id,
+ source_started_at=queued,
+ )
+ item_run_id = uuid5(
+ NAMESPACE_URL,
+ f"{BACKFILL_SCHEMA_VERSION}:{project}:{occurrence_id}",
+ )
+ return {
+ "schema_version": "backup_restore_legacy_occurrence_v1",
+ "project_id": project,
+ "source_message_id": source_message,
+ "source_fingerprint": fingerprint,
+ "source_fingerprint_hash": fingerprint_hash,
+ "dispatch_source_fingerprint": dispatch_source_fingerprint,
+ "source_queued_at": queued,
+ "occurrence_id": occurrence_id,
+ "backfill_run_id": str(item_run_id),
+ "incident_id": owner["incident_id"],
+ "approval_id": owner["approval_id"],
+ "work_item_id": owner["work_item_id"],
+ "route_id": BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ "controlled_apply": False,
+ "telegram_dispatch_performed": False,
+ }
+
+
+def _initial_cursor(project_id: str) -> dict[str, Any]:
+ return {
+ "schema_version": BACKFILL_CURSOR_SCHEMA_VERSION,
+ "project_id": project_id,
+ "cursor": {"queued_at": None, "message_id": None},
+ "scanned_total": 0,
+ "reserved_total": 0,
+ "deduplicated_total": 0,
+ "completed_total": 0,
+ "failed_total": 0,
+ "retry_total": 0,
+ "source_total_at_start": None,
+ "source_upper_bound": None,
+ "source_snapshot": None,
+ "source_snapshot_sha256": None,
+ "source_snapshot_truncated": False,
+ "cohort_index": 0,
+ "cohort_source_total": 0,
+ "cohort_scanned_total": 0,
+ "cohort_exhausted": False,
+ "cohort_counted": False,
+ "completed_cohort_count": 0,
+ "source_exhausted": False,
+ "source_drift_detected": False,
+ "scope_cap_reached": False,
+ "runtime_execution_authorized": False,
+ "telegram_dispatch_performed": False,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ }
+
+
+def _cursor_terminal_state(
+ detail: Mapping[str, Any],
+ *,
+ remaining: int,
+) -> tuple[str, str]:
+ if detail.get("source_drift_detected") is True:
+ return "blocked_source_drift_fail_closed", "failed"
+ if detail.get("scope_cap_reached") is True:
+ return "blocked_scope_cap", "waiting_tool"
+ if (
+ detail.get("cohort_exhausted") is True
+ and remaining == 0
+ and int(detail.get("failed_total") or 0) > 0
+ ):
+ return "blocked_failed_items_fail_closed", "failed"
+ if (
+ detail.get("source_exhausted") is True
+ and remaining == 0
+ and int(detail.get("item_total") or 0)
+ != int(detail.get("source_total_at_start") or 0)
+ ):
+ return "blocked_accounting_drift_fail_closed", "failed"
+ if (
+ detail.get("source_exhausted") is True
+ and remaining == 0
+ and int(detail.get("failed_total") or 0) > 0
+ ):
+ return "blocked_failed_items_fail_closed", "failed"
+ if detail.get("source_exhausted") is True and remaining == 0:
+ return "completed", "completed"
+ return "in_progress", "waiting_tool"
+
+
+def _item_payload_from_row(
+ *,
+ project_id: str,
+ row: Mapping[str, Any],
+) -> dict[str, Any]:
+ message_id = str(row.get("message_id") or "").strip()
+ source_fingerprint = str(row.get("source_fingerprint") or "").strip()
+ queued_at = _timestamp(row.get("queued_at"))
+ if source_fingerprint:
+ occurrence = build_legacy_backup_occurrence(
+ project_id=project_id,
+ message_id=message_id,
+ source_fingerprint=source_fingerprint,
+ queued_at=queued_at,
+ )
+ else:
+ missing_hash = _sha256(f"missing:{project_id}:{message_id}")[:16]
+ occurrence = {
+ "schema_version": "backup_restore_legacy_occurrence_v1",
+ "project_id": project_id,
+ "source_message_id": message_id,
+ "source_fingerprint": "",
+ "source_fingerprint_hash": "",
+ "dispatch_source_fingerprint": "",
+ "source_queued_at": queued_at,
+ "occurrence_id": f"legacy-backup-blocked:{message_id}:{missing_hash}",
+ "backfill_run_id": str(
+ uuid5(
+ NAMESPACE_URL,
+ f"{BACKFILL_SCHEMA_VERSION}:{project_id}:{message_id}:missing",
+ )
+ ),
+ "incident_id": None,
+ "approval_id": None,
+ "work_item_id": None,
+ "route_id": BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ "controlled_apply": False,
+ "telegram_dispatch_performed": False,
+ }
+ return {
+ "schema_version": "backup_restore_legacy_backfill_item_v1",
+ **occurrence,
+ "event_type": str(row.get("event_type") or ""),
+ "lane": str(row.get("lane") or BACKUP_RESTORE_LANE),
+ "target": str(row.get("target") or "backup_restore"),
+ "identity_ready": bool(source_fingerprint),
+ "runtime_execution_authorized": False,
+ "production_write_performed": False,
+ "telegram_dispatch_performed": False,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ }
+
+
+async def _reserve_legacy_backup_page(
+ *,
+ project_id: str,
+ scan_limit: int,
+ max_rows: int,
+) -> dict[str, Any]:
+ """Reserve one source page before advancing the durable cursor."""
+
+ bounded_limit = max(1, min(int(scan_limit), MAX_BATCH_LIMIT))
+ bounded_max = max(1, min(int(max_rows), MAX_SCOPE_ROWS))
+ cursor_run_id = _cursor_run_id(project_id)
+ initial = _initial_cursor(project_id)
+ async with get_db_context(project_id) as db:
+ await db.execute(
+ _CURSOR_INSERT_SQL,
+ {
+ "run_id": cursor_run_id,
+ "project_id": project_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ "input_sha256": _sha256(f"{BACKFILL_SCHEMA_VERSION}:{project_id}"),
+ "error_detail": _stable_json(initial),
+ },
+ )
+ selected = await db.execute(
+ _CURSOR_SELECT_SQL,
+ {
+ "project_id": project_id,
+ "run_id": cursor_run_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ },
+ )
+ selected_row = selected.mappings().one_or_none()
+ if selected_row is None:
+ raise RuntimeError("backup_backfill_cursor_row_missing")
+ detail = _json_object(selected_row.get("error_detail"))
+ if detail.get("schema_version") != BACKFILL_CURSOR_SCHEMA_VERSION:
+ raise RuntimeError("backup_backfill_cursor_schema_invalid")
+ if (
+ detail.get("scope_cap_reached") is True
+ and int(detail.get("scanned_total") or 0) == 0
+ and int(detail.get("reserved_total") or 0) == 0
+ ):
+ # The pre-cohort implementation captured max+1 rows and blocked
+ # before reserving any work. It is safe to replace only that
+ # zero-progress cursor with the bounded cohort shape.
+ detail = _initial_cursor(project_id)
+ if detail.get("source_snapshot") is None:
+ upper_bound = detail.get("source_upper_bound")
+ upper = dict(upper_bound) if isinstance(upper_bound, Mapping) else {}
+ snapshot_result = await db.execute(
+ _SOURCE_SNAPSHOT_SQL,
+ {
+ "project_id": project_id,
+ "event_type": BACKUP_RESTORE_EVENT_TYPE,
+ "projection_schema": PROJECTION_SCHEMA_VERSION,
+ "source_upper_queued_at": _database_timestamp(
+ upper.get("queued_at")
+ ),
+ "source_upper_message_id": upper.get("message_id"),
+ "limit": bounded_max,
+ },
+ )
+ raw_snapshot = [
+ _row_mapping(row) for row in snapshot_result.mappings().all()
+ ]
+ snapshot = [
+ {
+ "message_id": str(mapping.get("message_id") or ""),
+ "queued_at": _timestamp(mapping.get("queued_at")),
+ "source_fingerprint": str(mapping.get("source_fingerprint") or ""),
+ "event_type": str(mapping.get("event_type") or ""),
+ "lane": str(mapping.get("lane") or BACKUP_RESTORE_LANE),
+ "target": str(mapping.get("target") or "backup_restore"),
+ }
+ for mapping in raw_snapshot
+ ]
+ remaining_in_scope = (
+ int(raw_snapshot[0].get("source_total_in_scope") or len(snapshot))
+ if raw_snapshot
+ else 0
+ )
+ scanned_before_cohort = int(detail.get("scanned_total") or 0)
+ if detail.get("source_total_at_start") is None:
+ detail["source_total_at_start"] = remaining_in_scope
+ if raw_snapshot:
+ upper_queued_at = _timestamp(
+ raw_snapshot[0].get("source_upper_queued_at")
+ )
+ upper_message_id = str(
+ raw_snapshot[0].get("source_upper_message_id") or ""
+ ).strip()
+ if (
+ not upper_queued_at or not upper_message_id
+ ) and remaining_in_scope == len(snapshot):
+ upper_queued_at = snapshot[-1]["queued_at"]
+ upper_message_id = snapshot[-1]["message_id"]
+ if not upper_queued_at or not upper_message_id:
+ detail["source_drift_detected"] = True
+ else:
+ detail["source_upper_bound"] = {
+ "queued_at": upper_queued_at,
+ "message_id": upper_message_id,
+ }
+ expected_remaining = max(
+ 0,
+ int(detail.get("source_total_at_start") or 0) - scanned_before_cohort,
+ )
+ if remaining_in_scope != expected_remaining:
+ detail["source_drift_detected"] = True
+ detail["source_snapshot"] = snapshot
+ detail["source_snapshot_sha256"] = _sha256(_stable_json({"rows": snapshot}))
+ detail["source_snapshot_truncated"] = remaining_in_scope > len(snapshot)
+ detail["cohort_index"] = int(detail.get("cohort_index") or 0) + 1
+ detail["cohort_source_total"] = len(snapshot)
+ detail["cohort_scanned_total"] = 0
+ detail["cohort_exhausted"] = not snapshot
+ detail["cohort_counted"] = False
+ detail["source_exhausted"] = bool(
+ not snapshot
+ and scanned_before_cohort
+ == int(detail.get("source_total_at_start") or 0)
+ )
+ detail["scope_cap_reached"] = False
+ snapshot_value = detail.get("source_snapshot")
+ if not isinstance(snapshot_value, list) or any(
+ not isinstance(row, Mapping) for row in snapshot_value
+ ):
+ raise RuntimeError("backup_backfill_source_snapshot_invalid")
+ snapshot_rows = [dict(row) for row in snapshot_value]
+ snapshot_message_ids = [
+ str(row.get("message_id") or "").strip() for row in snapshot_rows
+ ]
+ if (
+ any(not message_id for message_id in snapshot_message_ids)
+ or len(snapshot_message_ids) != len(set(snapshot_message_ids))
+ or any(
+ str(row.get("event_type") or "") != BACKUP_RESTORE_EVENT_TYPE
+ for row in snapshot_rows
+ )
+ ):
+ raise RuntimeError("backup_backfill_source_snapshot_identity_invalid")
+ snapshot_sha256 = _sha256(_stable_json({"rows": snapshot_rows}))
+ if snapshot_sha256 != str(detail.get("source_snapshot_sha256") or ""):
+ raise RuntimeError("backup_backfill_source_snapshot_hash_mismatch")
+ if int(detail.get("cohort_source_total") or 0) != len(snapshot_rows):
+ raise RuntimeError("backup_backfill_source_cohort_count_mismatch")
+ source_total = int(detail.get("source_total_at_start") or 0)
+ scanned_total = int(detail.get("scanned_total") or 0)
+ cohort_scanned_total = int(detail.get("cohort_scanned_total") or 0)
+ cohort_source_total = int(detail.get("cohort_source_total") or 0)
+ if detail.get("source_drift_detected") is True:
+ await db.execute(
+ _CURSOR_UPDATE_SQL,
+ {
+ "state": "waiting_tool",
+ "error_detail": _stable_json(detail),
+ "project_id": project_id,
+ "run_id": cursor_run_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ },
+ )
+ return {
+ "scanned": 0,
+ "reserved": 0,
+ "deduplicated": 0,
+ "source_exhausted": bool(detail.get("source_exhausted")),
+ "source_drift_detected": True,
+ "scope_cap_reached": False,
+ "source_total_at_start": source_total,
+ "cohort_index": int(detail.get("cohort_index") or 0),
+ "cohort_source_total": cohort_source_total,
+ "cursor": dict(detail.get("cursor") or {}),
+ }
+
+ if (
+ scanned_total > source_total
+ or cohort_scanned_total > cohort_source_total
+ or len(snapshot_rows) > bounded_max
+ ):
+ detail["source_drift_detected"] = True
+ raise RuntimeError("backup_backfill_cursor_beyond_source_snapshot")
+ page_end = min(cohort_scanned_total + bounded_limit, cohort_source_total)
+ rows = snapshot_rows[cohort_scanned_total:page_end]
+ reserved = 0
+ deduplicated = 0
+ for row in rows:
+ item = _item_payload_from_row(project_id=project_id, row=row)
+ item_run_id = UUID(str(item["backfill_run_id"]))
+ await db.execute(
+ _ITEM_RUN_INSERT_SQL,
+ {
+ "run_id": item_run_id,
+ "project_id": project_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ "trigger_ref": (f"legacy-backup:{item['source_message_id']}")[:256],
+ "input_sha256": _sha256(_stable_json(item)),
+ "error_detail": _stable_json({"item": item}),
+ },
+ )
+ idempotency = await db.execute(
+ _ITEM_IDEMPOTENCY_INSERT_SQL,
+ {
+ "project_id": project_id,
+ "channel_type": BACKFILL_IDEMPOTENCY_CHANNEL,
+ "provider_event_id": item["occurrence_id"],
+ "run_id": item_run_id,
+ },
+ )
+ if idempotency.scalar_one_or_none() is not None:
+ reserved += 1
+ else:
+ deduplicated += 1
+
+ new_scanned_total = scanned_total + len(rows)
+ new_cohort_scanned_total = cohort_scanned_total + len(rows)
+ cohort_exhausted = new_cohort_scanned_total == cohort_source_total
+ source_exhausted = new_scanned_total == source_total
+ source_drift_detected = False
+ if rows:
+ last = rows[-1]
+ detail["cursor"] = {
+ "queued_at": _timestamp(last.get("queued_at")),
+ "message_id": str(last.get("message_id") or ""),
+ }
+ detail["scanned_total"] = new_scanned_total
+ detail["cohort_scanned_total"] = new_cohort_scanned_total
+ detail["cohort_exhausted"] = cohort_exhausted
+ detail["reserved_total"] = int(detail.get("reserved_total") or 0) + reserved
+ detail["deduplicated_total"] = (
+ int(detail.get("deduplicated_total") or 0) + deduplicated
+ )
+ detail["source_exhausted"] = source_exhausted
+ detail["source_drift_detected"] = source_drift_detected
+ detail["scope_cap_reached"] = False
+ await db.execute(
+ _CURSOR_UPDATE_SQL,
+ {
+ "state": "waiting_tool",
+ "error_detail": _stable_json(detail),
+ "project_id": project_id,
+ "run_id": cursor_run_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ },
+ )
+ return {
+ "scanned": len(rows),
+ "reserved": reserved,
+ "deduplicated": deduplicated,
+ "source_exhausted": source_exhausted,
+ "source_drift_detected": source_drift_detected,
+ "scope_cap_reached": bool(detail.get("scope_cap_reached")),
+ "source_total_at_start": source_total,
+ "cohort_index": int(detail.get("cohort_index") or 0),
+ "cohort_source_total": cohort_source_total,
+ "cohort_exhausted": cohort_exhausted,
+ "cursor": dict(detail.get("cursor") or {}),
+ }
+
+
+async def _claim_legacy_backup_items(
+ *,
+ project_id: str,
+ limit: int,
+) -> list[LegacyBackupClaim]:
+ bounded_limit = max(1, min(int(limit), MAX_BATCH_LIMIT))
+ claim_token = str(uuid4())
+ async with get_db_context(project_id) as db:
+ exhausted_result = await db.execute(
+ _EXHAUST_EXPIRED_CLAIMS_SQL,
+ {
+ "project_id": project_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ },
+ )
+ exhausted_run_ids = exhausted_result.mappings().all()
+ if exhausted_run_ids:
+ logger.warning(
+ "backup_restore_legacy_backfill_expired_claims_exhausted",
+ project_id=project_id,
+ exhausted_count=len(exhausted_run_ids),
+ telegram_dispatch_performed=False,
+ )
+ result = await db.execute(
+ _CLAIM_SQL,
+ {
+ "project_id": project_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ "limit": bounded_limit,
+ "claim_token": claim_token,
+ "lease_seconds": CLAIM_LEASE_SECONDS,
+ },
+ )
+ rows = result.mappings().all()
+ claims: list[LegacyBackupClaim] = []
+ for row in rows:
+ mapping = _row_mapping(row)
+ envelope = _json_object(mapping.get("error_detail"))
+ item = envelope.get("item")
+ if not isinstance(item, Mapping):
+ item = {}
+ claims.append(
+ LegacyBackupClaim(
+ run_id=UUID(str(mapping.get("run_id"))),
+ claim_token=claim_token,
+ attempt_count=int(mapping.get("attempt_count") or 0),
+ max_attempts=int(mapping.get("max_attempts") or 0),
+ item=dict(item),
+ )
+ )
+ return claims
+
+
+async def _load_legacy_backup_card(
+ *,
+ project_id: str,
+ claim: LegacyBackupClaim,
+) -> LegacyBackupCard | None:
+ message_id = str(claim.item.get("source_message_id") or "").strip()
+ if not message_id:
+ return None
+ async with get_db_context(project_id) as db:
+ result = await db.execute(
+ _SOURCE_LOAD_SQL,
+ {
+ "project_id": project_id,
+ "message_id": message_id,
+ "event_type": BACKUP_RESTORE_EVENT_TYPE,
+ },
+ )
+ row = result.mappings().one_or_none()
+ if row is None:
+ return None
+ mapping = _row_mapping(row)
+ envelope = _json_object(mapping.get("source_envelope"))
+ refs = _source_refs(envelope)
+ card = _alert_card(envelope)
+ return LegacyBackupCard(
+ message_id=str(mapping.get("message_id") or ""),
+ queued_at=_timestamp(mapping.get("queued_at")),
+ source_fingerprint=_first_string(refs.get("fingerprints")),
+ event_type=str(card.get("event_type") or ""),
+ lane=str(card.get("lane") or BACKUP_RESTORE_LANE),
+ target=str(card.get("target") or "backup_restore"),
+ source_envelope=envelope,
+ )
+
+
+def _existing_projection_matches(
+ card: LegacyBackupCard,
+ *,
+ occurrence: Mapping[str, Any],
+) -> bool:
+ projection = card.source_envelope.get("agent99_dispatch_receipt")
+ if not isinstance(projection, Mapping):
+ return False
+ schema_version = str(projection.get("schema_version") or "")
+ if schema_version == "telegram_agent99_dispatch_receipt_v1":
+ return False
+ if schema_version != PROJECTION_SCHEMA_VERSION:
+ raise ValueError("backup_backfill_existing_projection_schema_conflict")
+ durable_shape = bool(
+ projection.get("receipt_persisted") is True
+ and projection.get("accepted") is True
+ and projection.get("inbox_triggered") is True
+ and str(projection.get("run_id") or "").strip()
+ and str(projection.get("trace_id") or "").strip()
+ and str(projection.get("work_item_id") or "").strip()
+ and projection.get("kind") == "backup_health"
+ and projection.get("suggested_mode") == "BackupCheck"
+ )
+ if not durable_shape:
+ return False
+ backfill = card.source_envelope.get("backup_restore_backfill")
+ if isinstance(backfill, Mapping):
+ if str(backfill.get("occurrence_id") or "") != str(
+ occurrence.get("occurrence_id") or ""
+ ):
+ raise ValueError("backup_backfill_existing_occurrence_conflict")
+ required = {
+ "incident_id": occurrence.get("incident_id"),
+ "work_item_id": occurrence.get("work_item_id"),
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ }
+ if any(
+ str(projection.get(key) or "") != str(expected or "")
+ for key, expected in required.items()
+ ):
+ raise ValueError("backup_backfill_existing_projection_identity_conflict")
+ return True
+
+
+def _validated_projection(
+ *,
+ result: Mapping[str, Any],
+ occurrence: Mapping[str, Any],
+ relay_auth_mode: str,
+) -> dict[str, Any]:
+ owner = result.get("owner")
+ identity = result.get("identity")
+ if not isinstance(owner, Mapping) or not isinstance(identity, Mapping):
+ raise ValueError("backup_backfill_dispatch_identity_missing")
+ expected_owner = {
+ "incident_id": occurrence.get("incident_id"),
+ "approval_id": occurrence.get("approval_id"),
+ "work_item_id": occurrence.get("work_item_id"),
+ }
+ if any(
+ str(owner.get(key) or "") != str(expected or "")
+ for key, expected in expected_owner.items()
+ ):
+ raise ValueError("backup_backfill_dispatch_owner_mismatch")
+ if (
+ owner.get("controlled_apply") is not False
+ or owner.get("read_only") is not True
+ or str(owner.get("route_id") or "") != BACKUP_RESTORE_AGENT99_ROUTE_ID
+ or str(owner.get("kind") or "") != "backup_health"
+ or str(owner.get("mode") or "") != "BackupCheck"
+ ):
+ raise ValueError("backup_backfill_dispatch_owner_policy_mismatch")
+ if any(
+ str(identity.get(key) or "") != str(expected or "")
+ for key, expected in {
+ "incident_id": occurrence.get("incident_id"),
+ "work_item_id": occurrence.get("work_item_id"),
+ }.items()
+ ):
+ raise ValueError("backup_backfill_dispatch_stage_identity_mismatch")
+ if (
+ result.get("accepted") is not True
+ or result.get("receipt_persisted") is not True
+ or result.get("runtime_execution_authorized") is not False
+ or result.get("production_write_performed") is not False
+ or str(result.get("status") or "")
+ not in {
+ "backupcheck_dispatched_verifier_pending",
+ "backupcheck_deduplicated_verifier_pending",
+ }
+ ):
+ raise ValueError("backup_backfill_dispatch_not_durable_fail_closed")
+ run_id = str(identity.get("run_id") or "").strip()
+ trace_id = str(identity.get("trace_id") or "").strip()
+ work_item_id = str(identity.get("work_item_id") or "").strip()
+ if not run_id or not trace_id or not work_item_id:
+ raise ValueError("backup_backfill_dispatch_stage_identity_incomplete")
+ if relay_auth_mode not in {"token", "lan_allowlist", "injected_processor"}:
+ raise ValueError("backup_backfill_relay_auth_mode_invalid")
+ verifier = result.get("verifier")
+ verifier_receipt = (
+ dict(verifier.get("receipt"))
+ if isinstance(verifier, Mapping)
+ and isinstance(verifier.get("receipt"), Mapping)
+ else {}
+ )
+ return {
+ "schema_version": PROJECTION_SCHEMA_VERSION,
+ "status": str(result.get("status") or ""),
+ "transport": "durable_agent99_ledger",
+ "auth_mode": relay_auth_mode,
+ "alert_id": f"legacy-backup-card-{occurrence['source_message_id']}",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ # Ingress accepted is intentionally the conjunction of relay accepted
+ # and inboxTriggered; projecting both values is therefore not an
+ # inference from transport text.
+ "accepted": True,
+ "inbox_triggered": True,
+ "incident_id": str(occurrence["incident_id"]),
+ "run_id": run_id,
+ "trace_id": trace_id,
+ "work_item_id": work_item_id,
+ "idempotency_key": str(identity.get("idempotency_key") or ""),
+ "receipt_persisted": True,
+ "dispatch_performed": bool(result.get("dispatch_performed") is True),
+ "controlled_apply": False,
+ "read_only": True,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": bool(
+ result.get("runtime_closure_verified") is True
+ ),
+ "current_verifier": verifier_receipt,
+ "source": BACKFILL_SCHEMA_VERSION,
+ "telegram_dispatch_performed": False,
+ "stores_raw_response": False,
+ }
+
+
+async def _persist_legacy_dispatch_projection(
+ *,
+ project_id: str,
+ claim: LegacyBackupClaim,
+ card: LegacyBackupCard,
+ occurrence: Mapping[str, Any],
+ projection: Mapping[str, Any],
+) -> str:
+ backfill_receipt = {
+ "schema_version": BACKFILL_SCHEMA_VERSION,
+ "status": "dispatch_receipt_projected",
+ "source_message_id": card.message_id,
+ "source_fingerprint_hash": occurrence["source_fingerprint_hash"],
+ "occurrence_id": occurrence["occurrence_id"],
+ "backfill_run_id": occurrence["backfill_run_id"],
+ "incident_id": occurrence["incident_id"],
+ "work_item_id": occurrence["work_item_id"],
+ "agent99_run_id": str(projection.get("run_id") or ""),
+ "agent99_trace_id": str(projection.get("trace_id") or ""),
+ "auth_mode": str(projection.get("auth_mode") or ""),
+ "controlled_apply": False,
+ "telegram_dispatch_performed": False,
+ "runtime_closure_verified": False,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ }
+ source_identity_params = {
+ "project_id": project_id,
+ "message_id": card.message_id,
+ "source_queued_at": _database_timestamp(
+ claim.item.get("source_queued_at")
+ ),
+ "source_fingerprint": str(claim.item.get("source_fingerprint") or ""),
+ "event_type": str(claim.item.get("event_type") or ""),
+ "default_lane": BACKUP_RESTORE_LANE,
+ "source_lane": str(claim.item.get("lane") or BACKUP_RESTORE_LANE),
+ "source_target": str(claim.item.get("target") or "backup_restore"),
+ "claim_run_id": claim.run_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ "claim_token": claim.claim_token,
+ }
+ async with get_db_context(project_id) as db:
+ result = await db.execute(
+ _SOURCE_PROJECTION_UPDATE_SQL,
+ {
+ **source_identity_params,
+ "projection_schema": PROJECTION_SCHEMA_VERSION,
+ "projection": _stable_json(projection),
+ "backfill_receipt": _stable_json(backfill_receipt),
+ "run_id": str(projection.get("run_id") or ""),
+ "occurrence_id": occurrence["occurrence_id"],
+ },
+ )
+ if result.scalar_one_or_none() is not None:
+ return "projected"
+ readback_result = await db.execute(
+ _PROJECTION_CONFLICT_READBACK_SQL,
+ source_identity_params,
+ )
+ readback = readback_result.mappings().one_or_none()
+ readback_mapping = _row_mapping(readback) if readback is not None else {}
+ if readback_mapping.get("claim_active") is not True:
+ return "lease_lost"
+ if (
+ readback_mapping.get("source_exists") is not True
+ or readback_mapping.get("source_identity_matches") is not True
+ ):
+ return "source_identity_drift"
+ return "projection_conflict"
+
+
+def _public_result(
+ *,
+ status: str,
+ occurrence: Mapping[str, Any],
+ projection: Mapping[str, Any] | None = None,
+ reason: str = "",
+ auth_mode: str = "injected_processor",
+) -> dict[str, Any]:
+ return {
+ "schema_version": BACKFILL_RESULT_SCHEMA_VERSION,
+ "status": status,
+ "reason": reason,
+ "source_message_id": occurrence.get("source_message_id"),
+ "source_fingerprint_hash": occurrence.get("source_fingerprint_hash"),
+ "occurrence_id": occurrence.get("occurrence_id"),
+ "backfill_run_id": occurrence.get("backfill_run_id"),
+ "incident_id": occurrence.get("incident_id"),
+ "work_item_id": occurrence.get("work_item_id"),
+ "agent99_run_id": (
+ projection.get("run_id") if isinstance(projection, Mapping) else None
+ ),
+ "controlled_apply": False,
+ "auth_mode": auth_mode,
+ "runtime_execution_authorized": False,
+ "telegram_dispatch_performed": False,
+ "runtime_closure_verified": False,
+ "stores_raw_response": False,
+ }
+
+
+async def _finish_backfill_claim(
+ *,
+ project_id: str,
+ claim: LegacyBackupClaim,
+ state: str,
+ result: Mapping[str, Any],
+ error_code: str | None,
+) -> bool:
+ if state not in {"waiting_tool", "completed", "failed"}:
+ raise ValueError("backup_backfill_finish_state_invalid")
+ envelope = {"item": claim.item, "last_result": dict(result)}
+ output = _stable_json(dict(result))
+ async with get_db_context(project_id) as db:
+ updated = await db.execute(
+ _ITEM_FINISH_SQL,
+ {
+ "state": state,
+ "output_sha256": _sha256(output),
+ "error_code": error_code,
+ "error_detail": _stable_json(envelope),
+ "project_id": project_id,
+ "run_id": claim.run_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ "claim_token": claim.claim_token,
+ },
+ )
+ return updated.scalar_one_or_none() is not None
+
+
+async def _replay_backfill_claim(
+ *,
+ project_id: str,
+ claim: LegacyBackupClaim,
+ processor: Processor,
+ relay_auth_mode: str = "injected_processor",
+) -> str:
+ card = await _load_legacy_backup_card(project_id=project_id, claim=claim)
+ if card is None:
+ result = _public_result(
+ status="source_row_missing_fail_closed",
+ occurrence=claim.item,
+ reason="source_row_missing_or_not_backup_card",
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-SOURCE",
+ )
+ return "failed" if finished else "lease_lost"
+ if not card.source_fingerprint:
+ result = _public_result(
+ status="source_fingerprint_missing_fail_closed",
+ occurrence=claim.item,
+ reason="message_id_and_source_fingerprint_required",
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-FINGERPRINT",
+ )
+ return "failed" if finished else "lease_lost"
+ occurrence = build_legacy_backup_occurrence(
+ project_id=project_id,
+ message_id=card.message_id,
+ source_fingerprint=card.source_fingerprint,
+ queued_at=card.queued_at,
+ )
+ if any(
+ str(claim.item.get(key) or "") != str(occurrence.get(key) or "")
+ for key in (
+ "source_message_id",
+ "source_fingerprint",
+ "dispatch_source_fingerprint",
+ "occurrence_id",
+ "incident_id",
+ "work_item_id",
+ )
+ ) or any(
+ str(claim.item.get(key) or "") != str(current_value or "")
+ for key, current_value in {
+ "event_type": card.event_type,
+ "lane": card.lane,
+ "target": card.target,
+ }.items()
+ ):
+ result = _public_result(
+ status="source_identity_drift_fail_closed",
+ occurrence=occurrence,
+ reason="reserved_identity_does_not_match_current_source_row",
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-IDENTITY",
+ )
+ return "failed" if finished else "lease_lost"
+
+ try:
+ if _existing_projection_matches(card, occurrence=occurrence):
+ result = _public_result(
+ status="already_projected_deduplicated",
+ occurrence=occurrence,
+ projection=card.source_envelope.get("agent99_dispatch_receipt"),
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="completed",
+ result=result,
+ error_code=None,
+ )
+ return "deduplicated" if finished else "lease_lost"
+ except ValueError as exc:
+ result = _public_result(
+ status="existing_projection_conflict_fail_closed",
+ occurrence=occurrence,
+ reason=str(exc),
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-PROJECTION-CONFLICT",
+ )
+ return "failed" if finished else "lease_lost"
+
+ context_tokens = set_project_context(
+ project_id,
+ source="backup_restore_legacy_backfill",
+ request_id=str(claim.run_id),
+ )
+ try:
+ dispatch_result = await processor(
+ project_id=project_id,
+ alert_id=f"legacy-backup-card-{card.message_id}",
+ alertname="BackupRestoreEscrowSignalLegacyReplay",
+ severity="warning",
+ namespace="awoooi",
+ target_resource=card.target or "backup_restore",
+ message=(
+ "Legacy backup/restore card durable readback replay; collect "
+ "status, freshness, offsite, escrow and restore-drill evidence only."
+ ),
+ labels={
+ "event_type": BACKUP_RESTORE_EVENT_TYPE,
+ "lane": card.lane or BACKUP_RESTORE_LANE,
+ "legacy_source_message_id": card.message_id,
+ "legacy_source_fingerprint_hash": occurrence["source_fingerprint_hash"],
+ },
+ annotations={"summary": "Read-only BackupCheck historical evidence replay"},
+ source_fingerprint=occurrence["dispatch_source_fingerprint"],
+ source_event_fingerprint=occurrence["occurrence_id"],
+ source_started_at=card.queued_at,
+ alert_category="backup",
+ notification_type="TYPE-1",
+ source_url=None,
+ )
+ if (
+ str(dispatch_result.get("status") or "")
+ not in {
+ "backupcheck_dispatched_verifier_pending",
+ "backupcheck_deduplicated_verifier_pending",
+ }
+ or dispatch_result.get("accepted") is not True
+ or dispatch_result.get("receipt_persisted") is not True
+ ):
+ raise RuntimeError("durable_backupcheck_dispatch_not_ready")
+ projection = _validated_projection(
+ result=dispatch_result,
+ occurrence=occurrence,
+ relay_auth_mode=relay_auth_mode,
+ )
+ projection_status = await _persist_legacy_dispatch_projection(
+ project_id=project_id,
+ claim=claim,
+ card=card,
+ occurrence=occurrence,
+ projection=projection,
+ )
+ if projection_status == "lease_lost":
+ return "lease_lost"
+ if projection_status == "source_identity_drift":
+ result = _public_result(
+ status="source_identity_drift_after_dispatch_fail_closed",
+ occurrence=occurrence,
+ reason="reserved_identity_changed_before_projection_write",
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-IDENTITY",
+ )
+ return "failed" if finished else "lease_lost"
+ if projection_status != "projected":
+ raise ValueError("backup_backfill_projection_write_conflict")
+ except ValueError as exc:
+ result = _public_result(
+ status="dispatch_validation_failed_fail_closed",
+ occurrence=occurrence,
+ reason=str(exc),
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="failed",
+ result=result,
+ error_code="E-BACKFILL-VALIDATION",
+ )
+ return "failed" if finished else "lease_lost"
+ except Exception as exc:
+ retryable = claim.attempt_count < claim.max_attempts
+ result = _public_result(
+ status=(
+ "dispatch_retry_pending"
+ if retryable
+ else "dispatch_retry_exhausted_fail_closed"
+ ),
+ occurrence=occurrence,
+ reason=f"processor_error:{type(exc).__name__}",
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="waiting_tool" if retryable else "failed",
+ result=result,
+ error_code="E-BACKFILL-RETRY" if retryable else "E-BACKFILL-EXHAUSTED",
+ )
+ logger.warning(
+ "backup_restore_legacy_backfill_processor_failed",
+ project_id=project_id,
+ source_message_id=card.message_id,
+ error_type=type(exc).__name__,
+ retryable=retryable,
+ )
+ if not finished:
+ return "lease_lost"
+ return "retried" if retryable else "failed"
+ finally:
+ clear_project_context(context_tokens)
+
+ result = _public_result(
+ status="dispatch_receipt_projected",
+ occurrence=occurrence,
+ projection=projection,
+ auth_mode=relay_auth_mode,
+ )
+ finished = await _finish_backfill_claim(
+ project_id=project_id,
+ claim=claim,
+ state="completed",
+ result=result,
+ error_code=None,
+ )
+ return "completed" if finished else "lease_lost"
+
+
+async def _update_cursor_counters(
+ *,
+ project_id: str,
+ completed: int,
+ deduplicated: int,
+ failed: int,
+ retried: int,
+) -> dict[str, Any]:
+ cursor_run_id = _cursor_run_id(project_id)
+ async with get_db_context(project_id) as db:
+ selected = await db.execute(
+ _CURSOR_SELECT_SQL,
+ {
+ "project_id": project_id,
+ "run_id": cursor_run_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ },
+ )
+ row = selected.mappings().one_or_none()
+ if row is None:
+ raise RuntimeError("backup_backfill_cursor_row_missing")
+ detail = _json_object(row.get("error_detail"))
+ detail["last_tick"] = {
+ "completed": int(completed),
+ "deduplicated": int(deduplicated),
+ "failed": int(failed),
+ "retried": int(retried),
+ }
+ detail["deduplicated_total"] = int(detail.get("deduplicated_total") or 0) + int(
+ deduplicated
+ )
+ detail["retry_total"] = int(detail.get("retry_total") or 0) + int(retried)
+ state_counts_result = await db.execute(
+ _ITEM_STATE_COUNTS_SQL,
+ {
+ "project_id": project_id,
+ "agent_id": BACKFILL_ITEM_AGENT_ID,
+ },
+ )
+ state_counts_row = state_counts_result.mappings().one_or_none()
+ if state_counts_row is None:
+ raise RuntimeError("backup_backfill_item_state_counts_missing")
+ state_counts = _row_mapping(state_counts_row)
+ detail["item_total"] = int(state_counts.get("item_total") or 0)
+ detail["completed_total"] = int(state_counts.get("completed_total") or 0)
+ detail["failed_total"] = int(state_counts.get("failed_total") or 0)
+ remaining = int(state_counts.get("remaining_total") or 0)
+ detail["remaining_item_total"] = remaining
+ cohort_ready = bool(detail.get("cohort_exhausted") is True and remaining == 0)
+ if cohort_ready and int(detail.get("item_total") or 0) != int(
+ detail.get("scanned_total") or 0
+ ):
+ detail["source_drift_detected"] = True
+ status, cursor_state = _cursor_terminal_state(detail, remaining=remaining)
+ if (
+ status == "in_progress"
+ and cohort_ready
+ and detail.get("source_exhausted") is not True
+ ):
+ if detail.get("cohort_counted") is not True:
+ detail["completed_cohort_count"] = (
+ int(detail.get("completed_cohort_count") or 0) + 1
+ )
+ detail["cohort_counted"] = True
+ detail["source_snapshot"] = None
+ detail["source_snapshot_sha256"] = None
+ detail["source_snapshot_truncated"] = False
+ detail["cohort_source_total"] = 0
+ detail["cohort_scanned_total"] = 0
+ detail["cohort_exhausted"] = False
+ status = "cohort_completed_next_pending"
+ cursor_state = "waiting_tool"
+ elif status == "completed" and detail.get("cohort_counted") is not True:
+ if int(detail.get("cohort_source_total") or 0) > 0:
+ detail["completed_cohort_count"] = (
+ int(detail.get("completed_cohort_count") or 0) + 1
+ )
+ detail["cohort_counted"] = True
+ detail["status"] = status
+ await db.execute(
+ _CURSOR_UPDATE_SQL,
+ {
+ "state": cursor_state,
+ "error_detail": _stable_json(detail),
+ "project_id": project_id,
+ "run_id": cursor_run_id,
+ "agent_id": BACKFILL_CURSOR_AGENT_ID,
+ },
+ )
+ return detail
+
+
+async def run_backup_restore_legacy_backfill_once(
+ *,
+ project_id: str = "awoooi",
+ scan_limit: int = 20,
+ process_limit: int = 20,
+ max_rows: int = MAX_SCOPE_ROWS,
+ processor: Processor = process_backup_restore_alertmanager_signal,
+) -> dict[str, Any]:
+ """Reserve and process one bounded historical replay batch."""
+
+ normalized_project = str(project_id or "").strip()
+ if not normalized_project:
+ return {
+ "schema_version": BACKFILL_RESULT_SCHEMA_VERSION,
+ "status": "project_context_missing_fail_closed",
+ "dispatch_total": 0,
+ "runtime_execution_authorized": False,
+ "telegram_dispatch_performed": False,
+ }
+ relay_auth_mode = "injected_processor"
+ relay_preflight: dict[str, Any] | None = None
+ if processor is process_backup_restore_alertmanager_signal:
+ relay_preflight = build_backup_restore_backfill_relay_preflight()
+ if relay_preflight["ready"] is not True:
+ return {
+ "schema_version": BACKFILL_RESULT_SCHEMA_VERSION,
+ "status": "relay_preflight_failed_fail_closed",
+ "dispatch_total": 0,
+ "auth_mode": relay_preflight["auth_mode"],
+ "relay_preflight": relay_preflight,
+ "runtime_execution_authorized": False,
+ "telegram_dispatch_performed": False,
+ "error": "trusted_relay_not_configured",
+ }
+ relay_auth_mode = str(relay_preflight["auth_mode"])
+ stats: dict[str, Any] = {
+ "schema_version": BACKFILL_RESULT_SCHEMA_VERSION,
+ "project_id": normalized_project,
+ "status": "in_progress",
+ "auth_mode": relay_auth_mode,
+ "relay_preflight": relay_preflight,
+ "scanned": 0,
+ "reserved": 0,
+ "claimed": 0,
+ "completed": 0,
+ "deduplicated": 0,
+ "failed": 0,
+ "retried": 0,
+ "lease_lost": 0,
+ "dispatch_total": 0,
+ "source_total_at_start": None,
+ "source_exhausted": False,
+ "source_drift_detected": False,
+ "scope_cap_reached": False,
+ "cohort_index": 0,
+ "cohort_source_total": 0,
+ "cohort_exhausted": False,
+ "cursor": {},
+ "runtime_execution_authorized": False,
+ "telegram_dispatch_performed": False,
+ "production_write_performed": False,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ "error": None,
+ }
+ try:
+ reserved = await _reserve_legacy_backup_page(
+ project_id=normalized_project,
+ scan_limit=scan_limit,
+ max_rows=max_rows,
+ )
+ stats.update(
+ {
+ "scanned": int(reserved.get("scanned") or 0),
+ "reserved": int(reserved.get("reserved") or 0),
+ "deduplicated": int(reserved.get("deduplicated") or 0),
+ "source_total_at_start": reserved.get("source_total_at_start"),
+ "source_exhausted": bool(reserved.get("source_exhausted")),
+ "source_drift_detected": bool(reserved.get("source_drift_detected")),
+ "scope_cap_reached": bool(reserved.get("scope_cap_reached")),
+ "cohort_index": int(reserved.get("cohort_index") or 0),
+ "cohort_source_total": int(reserved.get("cohort_source_total") or 0),
+ "cohort_exhausted": bool(reserved.get("cohort_exhausted")),
+ "cursor": dict(reserved.get("cursor") or {}),
+ }
+ )
+ claims = await _claim_legacy_backup_items(
+ project_id=normalized_project,
+ limit=process_limit,
+ )
+ stats["claimed"] = len(claims)
+ for claim in claims:
+ outcome = await _replay_backfill_claim(
+ project_id=normalized_project,
+ claim=claim,
+ processor=processor,
+ relay_auth_mode=relay_auth_mode,
+ )
+ if outcome == "completed":
+ stats["completed"] += 1
+ stats["dispatch_total"] += 1
+ elif outcome == "deduplicated":
+ stats["deduplicated"] += 1
+ elif outcome == "retried":
+ stats["retried"] += 1
+ elif outcome == "lease_lost":
+ stats["lease_lost"] += 1
+ else:
+ stats["failed"] += 1
+ cursor_detail = await _update_cursor_counters(
+ project_id=normalized_project,
+ completed=stats["completed"],
+ deduplicated=(
+ stats["deduplicated"] - int(reserved.get("deduplicated") or 0)
+ ),
+ failed=stats["failed"],
+ retried=stats["retried"],
+ )
+ stats["status"] = str(cursor_detail.get("status") or "in_progress")
+ stats["remaining_item_total"] = int(
+ cursor_detail.get("remaining_item_total") or 0
+ )
+ except Exception as exc:
+ stats["status"] = "database_or_worker_failure_fail_closed"
+ stats["error"] = f"{type(exc).__name__}"
+ logger.warning(
+ "backup_restore_legacy_backfill_tick_failed",
+ project_id=normalized_project,
+ error_type=type(exc).__name__,
+ runtime_execution_authorized=False,
+ telegram_dispatch_performed=False,
+ )
+ logger.info("backup_restore_legacy_backfill_tick", **stats)
+ return stats
+
+
+async def run_backup_restore_legacy_backfill_loop() -> None:
+ """Drain the historical backlog in bounded, restart-safe batches."""
+
+ if not settings.ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL:
+ logger.info("backup_restore_legacy_backfill_disabled")
+ return
+ relay_preflight = build_backup_restore_backfill_relay_preflight()
+ if relay_preflight["ready"] is not True:
+ logger.warning(
+ "backup_restore_legacy_backfill_relay_preflight_failed",
+ status=relay_preflight["status"],
+ auth_mode=relay_preflight["auth_mode"],
+ endpoint_class=relay_preflight["endpoint_class"],
+ telegram_dispatch_performed=False,
+ )
+ return
+ logger.info(
+ "backup_restore_legacy_backfill_loop_started",
+ auth_mode=relay_preflight["auth_mode"],
+ batch_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
+ max_rows=settings.BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS,
+ telegram_dispatch_performed=False,
+ )
+ await asyncio.sleep(settings.BACKUP_RESTORE_LEGACY_BACKFILL_STARTUP_SLEEP_SECONDS)
+ while True:
+ sleep_seconds = float(settings.BACKUP_RESTORE_LEGACY_BACKFILL_INTERVAL_SECONDS)
+ try:
+ result = await run_backup_restore_legacy_backfill_once(
+ project_id="awoooi",
+ scan_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
+ process_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
+ max_rows=settings.BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS,
+ )
+ if result.get("status") == "completed":
+ logger.info(
+ "backup_restore_legacy_backfill_loop_completed",
+ source_total_at_start=result.get("source_total_at_start"),
+ telegram_dispatch_performed=False,
+ )
+ return
+ if result.get("status") in {
+ "blocked_scope_cap",
+ "blocked_failed_items_fail_closed",
+ "blocked_source_drift_fail_closed",
+ "blocked_accounting_drift_fail_closed",
+ }:
+ logger.warning(
+ "backup_restore_legacy_backfill_loop_terminal_blocked",
+ status=result.get("status"),
+ telegram_dispatch_performed=False,
+ )
+ return
+ if result.get("error"):
+ sleep_seconds = max(sleep_seconds, 60.0)
+ except asyncio.CancelledError:
+ raise
+ except Exception as exc:
+ logger.warning(
+ "backup_restore_legacy_backfill_loop_failed",
+ error_type=type(exc).__name__,
+ )
+ sleep_seconds = max(sleep_seconds, 60.0)
+ await asyncio.sleep(sleep_seconds)
diff --git a/apps/api/src/main.py b/apps/api/src/main.py
index 9e203f95e..7e43b3e49 100644
--- a/apps/api/src/main.py
+++ b/apps/api/src/main.py
@@ -465,6 +465,37 @@ async def lifespan(_app: FastAPI) -> AsyncGenerator[None, None]:
handoff="awoooi-worker deployment owns Redis stream consumption",
)
+ # Agent99 durable dispatch outcome reconciler. The worker reads only the
+ # authenticated public-safe relay outcome and closes the same PG run after
+ # verifier, KM/PlayBook, Telegram, and incident receipts are durable.
+ if (
+ settings.AGENT99_SRE_ALERT_RELAY_URL
+ and settings.AGENT99_SRE_ALERT_RELAY_TOKEN
+ ):
+ try:
+ from src.jobs.agent99_controlled_dispatch_reconciler_job import (
+ run_agent99_controlled_dispatch_reconciler_loop,
+ )
+
+ scheduled_task = schedule_api_background_task(
+ run_agent99_controlled_dispatch_reconciler_loop()
+ )
+ logger.info(
+ "agent99_controlled_dispatch_reconciler_schedule_evaluated",
+ scheduled_in_process=scheduled_task is not None,
+ interval_seconds=15,
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_controlled_dispatch_reconciler_schedule_failed",
+ error=str(exc),
+ )
+ else:
+ logger.info(
+ "agent99_controlled_dispatch_reconciler_skipped",
+ reason="authenticated_relay_not_configured",
+ )
+
# BUG-005 修復 2026-04-11: 啟動時掃描 Redis 中所有 state=ready 但未送 Telegram 的 token
# dedup TTL 10 分鐘過期後，ready decisions 就沒有補送機制 → 長期卡在 ready 無人審核
try:
diff --git a/apps/api/src/models/incident.py b/apps/api/src/models/incident.py
index 98296bf48..b648a8446 100644
--- a/apps/api/src/models/incident.py
+++ b/apps/api/src/models/incident.py
@@ -22,7 +22,7 @@ C-Suite 戰略會議決議 (2026-03-22):
from datetime import datetime, timezone
from enum import Enum
-from typing import Literal
+from typing import Literal, cast, get_args
from uuid import UUID, uuid4
from pydantic import BaseModel, Field, field_validator
@@ -70,6 +70,33 @@ class IncidentStatus(str, Enum):
ESCALATED = "escalated" # 已升級 - 需要人工介入
+# Signal sources are a durable contract: values written by the host sensor and
+# the public signal ingress can remain in PostgreSQL long after a producer is
+# upgraded. Keep this list bounded, but include every repo-owned producer
+# vocabulary instead of silently rewriting provenance during readback.
+SignalSource = Literal[
+ "prometheus",
+ "signoz",
+ "alertmanager",
+ "manual",
+ "telegram",
+ "journal",
+ "node-exporter",
+ "sensor-probe",
+ "sensor-agent",
+]
+SIGNAL_SOURCE_VALUES: frozenset[str] = frozenset(get_args(SignalSource))
+
+
+def parse_signal_source(value: object) -> SignalSource:
+ """Return a canonical known source or reject an unowned durable value."""
+
+ normalized = str(value or "").strip().lower()
+ if normalized not in SIGNAL_SOURCE_VALUES:
+ raise ValueError("unsupported_signal_source")
+ return cast(SignalSource, normalized)
+
+
# =============================================================================
# Signal (原始告警)
# =============================================================================
@@ -89,9 +116,7 @@ class Signal(BaseModel):
)
alert_name: str = Field(..., description="告警名稱 (如 HighCPUUsage)")
severity: Severity = Field(..., description="告警嚴重度")
- source: Literal["prometheus", "signoz", "alertmanager", "manual", "telegram"] = (
- Field(..., description="告警來源")
- )
+ source: SignalSource = Field(..., description="告警來源")
fired_at: datetime = Field(..., description="告警觸發時間")
resolved_at: datetime | None = Field(None, description="告警解除時間")
labels: dict[str, str] = Field(
@@ -107,9 +132,26 @@ class Signal(BaseModel):
description="告警指紋 Hash，用於去重與聚合",
)
+ @field_validator("source", mode="before")
+ @classmethod
+ def validate_source(cls, value: object) -> SignalSource:
+ return parse_signal_source(value)
+
# [首席架構師] 移除 json_encoders (Pydantic v2 已 deprecated)，原生序列化輸出格式與 .isoformat() 一致 v1.1 2026-04-01 Asia/Taipei
+class IncidentReadDegradation(BaseModel):
+ """Public-safe metadata for one durable row excluded from a response."""
+
+ incident_id: str
+ reason_code: Literal[
+ "incident_signal_source_unsupported",
+ "incident_signal_schema_invalid",
+ "incident_record_schema_invalid",
+ ]
+ quarantined_from_response: bool = True
+
+
# =============================================================================
# AI Decision Chain (CISO 要求：可稽核性)
# =============================================================================
diff --git a/apps/api/src/repositories/incident_repository.py b/apps/api/src/repositories/incident_repository.py
index 67cb5047d..bd06937f2 100644
--- a/apps/api/src/repositories/incident_repository.py
+++ b/apps/api/src/repositories/incident_repository.py
@@ -11,10 +11,12 @@ Phase 16 R3.3: Repository 層實作
建立者: Claude Code (Phase 16 架構重構)
"""
+from dataclasses import dataclass, field
from datetime import UTC, datetime
from typing import Any
import structlog
+from pydantic import ValidationError
from sqlalchemy import select
from src.db.base import get_db_context
@@ -22,6 +24,7 @@ from src.db.models import IncidentRecord
from src.models.incident import (
Incident,
IncidentFrequencyStats,
+ IncidentReadDegradation,
IncidentStatus,
Severity,
)
@@ -29,6 +32,37 @@ from src.repositories.interfaces import IIncidentRepository
logger = structlog.get_logger(__name__)
+_ACTIVE_INCIDENT_STATUS_VALUES = frozenset({
+ IncidentStatus.INVESTIGATING.value,
+ IncidentStatus.MITIGATING.value,
+})
+
+
+@dataclass
+class ActiveIncidentReadResult:
+ """Valid active incidents plus public-safe per-row quarantine metadata."""
+
+ incidents: list[Incident] = field(default_factory=list)
+ degraded_records: list[IncidentReadDegradation] = field(default_factory=list)
+
+ @property
+ def degraded(self) -> bool:
+ return bool(self.degraded_records)
+
+
+def _record_validation_reason(exc: ValidationError) -> str:
+ errors = exc.errors(include_input=False)
+ source_errors = [
+ error
+ for error in errors
+ if tuple(error.get("loc") or ())[-1:] == ("source",)
+ ]
+ if source_errors and all(error.get("type") != "missing" for error in source_errors):
+ return "incident_signal_source_unsupported"
+ if any(tuple(error.get("loc") or ())[:1] == ("signals",) for error in errors):
+ return "incident_signal_schema_invalid"
+ return "incident_record_schema_invalid"
+
# =============================================================================
# Conversion Helpers
@@ -168,8 +202,13 @@ class IncidentDBRepository(IIncidentRepository):
record = result.scalar_one_or_none()
return _record_to_incident(record) if record else None
- async def get_active(self, *, project_id: str | None = None) -> list[Incident]:
- """取得所有活躍的 Incident"""
+ async def get_active_readback(
+ self,
+ *,
+ project_id: str | None = None,
+ ) -> ActiveIncidentReadResult:
+ """Read valid active rows and quarantine only schema-invalid rows."""
+
active_statuses = [
IncidentStatus.INVESTIGATING,
IncidentStatus.MITIGATING,
@@ -181,7 +220,48 @@ class IncidentDBRepository(IIncidentRepository):
.order_by(IncidentRecord.created_at.desc())
)
records = result.scalars().all()
- return [_record_to_incident(r) for r in records]
+ # SQL is authoritative, while the defensive filter prevents a
+ # terminal row from leaking into the active surface if an enum or
+ # test adapter returns a wider result set. In particular, verified
+ # terminal incident INC-20260711-D037E5 must never reappear here.
+ active_records = [
+ record
+ for record in records
+ if _normalize_status(record.status)
+ in _ACTIVE_INCIDENT_STATUS_VALUES
+ ]
+ incidents: list[Incident] = []
+ degraded_records: list[IncidentReadDegradation] = []
+ for record in active_records:
+ try:
+ incidents.append(_record_to_incident(record))
+ except ValidationError as exc:
+ reason_code = _record_validation_reason(exc)
+ logger.warning(
+ "active_incident_row_quarantined",
+ incident_id=record.incident_id,
+ project_id=project_id,
+ reason_code=reason_code,
+ validation_error_count=len(exc.errors()),
+ redis_fallback_used=False,
+ )
+ degraded_records.append(
+ IncidentReadDegradation(
+ incident_id=record.incident_id,
+ reason_code=reason_code,
+ )
+ )
+ return ActiveIncidentReadResult(
+ incidents=incidents,
+ degraded_records=degraded_records,
+ )
+
+ async def get_active(self, *, project_id: str | None = None) -> list[Incident]:
+ """取得所有可安全讀取的活躍 Incident。"""
+
+ return (
+ await self.get_active_readback(project_id=project_id)
+ ).incidents
async def update(self, incident: Incident) -> Incident | None:
"""更新 Incident"""
diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py
new file mode 100644
index 000000000..fab3e692f
--- /dev/null
+++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py
@@ -0,0 +1,1962 @@
+"""Durable single-owner ledger for Agent99 controlled dispatch.
+
+One logical recovery action must have one identity across Alertmanager,
+Agent99, Telegram projection, verifier, and learning writeback. This module
+uses the existing AwoooP run/idempotency/step tables as a PostgreSQL-backed
+reservation and receipt store. It never performs the transport dispatch.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass
+from datetime import UTC, datetime, timedelta
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid4, uuid5
+
+import structlog
+from sqlalchemy import and_, or_, select, update
+from sqlalchemy.dialects.postgresql import insert as pg_insert
+
+from src.db.awooop_models import (
+ AwoooPRunIdempotency,
+ AwoooPRunState,
+ AwoooPRunStepJournal,
+)
+from src.db.base import get_db_context
+from src.db.models import AlertOperationLog, IncidentRecord
+from src.models.incident import IncidentStatus
+from src.services.agent99_public_receipts import (
+ AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS,
+ AGENT99_LEARNING_RECEIPT_REFS,
+ AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS,
+ sanitize_agent99_public_receipt_refs,
+)
+
+logger = structlog.get_logger(__name__)
+
+AGENT99_DISPATCH_AGENT_ID = "agent99_controlled_dispatch"
+AGENT99_DISPATCH_CHANNEL = "agent99_controlled"
+AGENT99_DISPATCH_TRIGGER_TYPE = "agent99_dispatch"
+AGENT99_DISPATCH_LEASE_SECONDS = 120
+AGENT99_DISPATCH_RETRY_SECONDS = 600
+AGENT99_DISPATCH_TIMEOUT_MINUTES = 30
+AGENT99_MAX_EXECUTION_GENERATION = 3
+
+
+def _utc_now_naive() -> datetime:
+ """Return PostgreSQL-compatible UTC for the existing naive columns."""
+
+ return datetime.now(UTC).replace(tzinfo=None)
+
+
+def _stable_json(value: dict[str, Any]) -> str:
+ return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+
+
+def _sha256(value: str) -> str:
+ return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class Agent99DispatchIdentity:
+ project_id: str
+ incident_id: str
+ source_fingerprint: str
+ route_id: str
+ execution_generation: str
+ approval_id: str
+ run_id: UUID
+ trace_id: str
+ work_item_id: str
+ idempotency_key: str
+ single_flight_key: str
+ lock_owner: str
+ trigger_ref: str
+
+ def public_dict(self) -> dict[str, str]:
+ public = {
+ "schema_version": "agent99_controlled_dispatch_identity_v1",
+ "project_id": self.project_id,
+ "incident_id": self.incident_id,
+ "source_fingerprint": self.source_fingerprint,
+ "route_id": self.route_id,
+ "execution_generation": self.execution_generation,
+ "approval_id": self.approval_id,
+ "run_id": str(self.run_id),
+ "trace_id": self.trace_id,
+ "work_item_id": self.work_item_id,
+ "idempotency_key": self.idempotency_key,
+ "single_flight_key": self.single_flight_key,
+ "lock_owner": self.lock_owner,
+ }
+ public["canonical_digest"] = _sha256(_stable_json(public))
+ return public
+
+
+def build_agent99_dispatch_identity(
+ *,
+ project_id: str,
+ incident_id: str,
+ source_fingerprint: str,
+ route_id: str,
+ execution_generation: str = "1",
+ approval_id: str = "",
+ work_item_id: str = "",
+) -> Agent99DispatchIdentity:
+ """Build one deterministic identity for every projection of a dispatch."""
+
+ resolved_project_id = str(project_id or "awoooi").strip() or "awoooi"
+ resolved_incident_id = str(incident_id or "").strip()
+ resolved_route_id = str(route_id or "").strip()
+ resolved_work_item_id = (
+ str(work_item_id or "").strip()
+ or (
+ f"agent99-dispatch:{resolved_project_id}:"
+ f"{resolved_incident_id}:{resolved_route_id}"
+ )
+ )[:160]
+ normalized = {
+ "project_id": resolved_project_id,
+ "incident_id": resolved_incident_id,
+ "source_fingerprint": str(source_fingerprint or "").strip(),
+ "route_id": resolved_route_id,
+ "execution_generation": str(execution_generation or "1").strip() or "1",
+ "approval_id": str(approval_id or "").strip()[:128],
+ "work_item_id": resolved_work_item_id,
+ }
+ if not normalized["incident_id"]:
+ raise ValueError("agent99_dispatch_incident_id_required")
+ if not normalized["source_fingerprint"]:
+ raise ValueError("agent99_dispatch_source_fingerprint_required")
+ if not normalized["route_id"]:
+ raise ValueError("agent99_dispatch_route_id_required")
+
+ canonical = _stable_json(normalized)
+ digest = _sha256(canonical)
+ # The transport lock protects one logical incident/source/route across
+ # approval refreshes and bounded generations. Run/idempotency identity is
+ # stricter and includes both approval and work-item identity above.
+ single_flight_digest = _sha256(_stable_json({
+ "project_id": normalized["project_id"],
+ "incident_id": normalized["incident_id"],
+ "source_fingerprint": normalized["source_fingerprint"],
+ "route_id": normalized["route_id"],
+ }))
+ run_id = uuid5(NAMESPACE_URL, f"agent99-controlled-dispatch:{canonical}")
+ trace_id = f"00-{digest[:32]}-{digest[32:48]}-01"
+ idempotency_key = f"agent99-controlled:{digest}"
+ trigger_ref = (
+ f"agent99:{normalized['incident_id']}:{normalized['route_id']}:"
+ f"{normalized['execution_generation']}"
+ )[:256]
+ return Agent99DispatchIdentity(
+ project_id=normalized["project_id"],
+ incident_id=normalized["incident_id"],
+ source_fingerprint=normalized["source_fingerprint"],
+ route_id=normalized["route_id"],
+ execution_generation=normalized["execution_generation"],
+ approval_id=normalized["approval_id"],
+ run_id=run_id,
+ trace_id=trace_id,
+ work_item_id=normalized["work_item_id"],
+ idempotency_key=idempotency_key,
+ single_flight_key=f"agent99-controlled-flight:{single_flight_digest}",
+ lock_owner=str(run_id),
+ trigger_ref=trigger_ref,
+ )
+
+
+def parse_agent99_dispatch_identity(
+ value: dict[str, Any],
+) -> Agent99DispatchIdentity:
+ """Rebuild and validate a complete public identity from an outcome caller."""
+
+ required_nonempty = {
+ "schema_version",
+ "project_id",
+ "incident_id",
+ "source_fingerprint",
+ "route_id",
+ "execution_generation",
+ "run_id",
+ "trace_id",
+ "work_item_id",
+ "idempotency_key",
+ "single_flight_key",
+ "lock_owner",
+ "canonical_digest",
+ }
+ normalized = {
+ str(key): str(item or "").strip()
+ for key, item in (value or {}).items()
+ }
+ missing = sorted(
+ key for key in required_nonempty if not normalized.get(key)
+ )
+ if "approval_id" not in normalized:
+ missing.append("approval_id")
+ if missing:
+ raise ValueError(
+ "agent99_dispatch_identity_fields_required:" + ",".join(missing)
+ )
+ if normalized["schema_version"] != "agent99_controlled_dispatch_identity_v1":
+ raise ValueError("agent99_dispatch_identity_schema_invalid")
+ identity = build_agent99_dispatch_identity(
+ project_id=normalized["project_id"],
+ incident_id=normalized["incident_id"],
+ source_fingerprint=normalized["source_fingerprint"],
+ route_id=normalized["route_id"],
+ execution_generation=normalized["execution_generation"],
+ approval_id=normalized.get("approval_id", ""),
+ work_item_id=normalized["work_item_id"],
+ )
+ expected = identity.public_dict()
+ compared = required_nonempty | {"approval_id"}
+ mismatched = sorted(
+ key for key in (compared - {"canonical_digest"})
+ if normalized.get(key, "") != str(expected.get(key) or "")
+ )
+ if mismatched:
+ raise ValueError(
+ "agent99_dispatch_identity_mismatch:" + ",".join(mismatched)
+ )
+ if normalized["canonical_digest"] != expected["canonical_digest"]:
+ raise ValueError("agent99_dispatch_identity_mismatch:canonical_digest")
+ return identity
+
+
+def validate_agent99_outcome_identity(
+ identity: Agent99DispatchIdentity,
+ receipt: dict[str, Any],
+) -> bool:
+ """Require the complete canonical identity at both receipt layers."""
+
+ outer = receipt.get("identity")
+ outcome = receipt.get("outcome")
+ nested = outcome.get("identity") if isinstance(outcome, dict) else None
+ if not isinstance(outer, dict) or not isinstance(nested, dict):
+ return False
+ expected = identity.public_dict()
+ for candidate in (outer, nested):
+ try:
+ parsed = parse_agent99_dispatch_identity(candidate)
+ except ValueError:
+ return False
+ if parsed != identity or parsed.public_dict() != expected:
+ return False
+ return True
+
+
+def agent99_dispatch_identity_from_public(
+ value: dict[str, Any],
+) -> Agent99DispatchIdentity:
+ """Rebuild and cryptographically validate a public dispatch identity."""
+
+ return parse_agent99_dispatch_identity(value)
+
+
+def attach_agent99_dispatch_identity(
+ payload: dict[str, Any],
+ identity: Agent99DispatchIdentity,
+) -> dict[str, Any]:
+ """Attach the shared identity without exposing secrets or raw responses."""
+
+ enriched = dict(payload)
+ routing = dict(enriched.get("routing") or {})
+ awoooi = dict(enriched.get("awoooi") or {})
+ public_identity = identity.public_dict()
+ routing.update({
+ "idempotencyKey": identity.idempotency_key,
+ "runId": str(identity.run_id),
+ "traceId": identity.trace_id,
+ "workItemId": identity.work_item_id,
+ "executionGeneration": identity.execution_generation,
+ "correlationKey": identity.idempotency_key,
+ })
+ awoooi.update({
+ "incidentId": identity.incident_id,
+ "approvalId": identity.approval_id,
+ "agent99DispatchIdentity": public_identity,
+ })
+ enriched.update({
+ "id": f"awoooi-agent99-{identity.run_id}",
+ "routing": routing,
+ "awoooi": awoooi,
+ })
+ return enriched
+
+
+def build_agent99_dispatch_receipt_envelope(
+ *,
+ identity: Agent99DispatchIdentity,
+ dispatch_receipt: dict[str, Any],
+ controlled_apply_authorized: bool | None = None,
+) -> dict[str, Any]:
+ accepted = bool(dispatch_receipt.get("accepted") is True)
+ inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True)
+ dispatch_promoted = bool(accepted and inbox_triggered)
+ delivery_certainty = str(
+ dispatch_receipt.get("delivery_certainty")
+ or (
+ "delivered"
+ if dispatch_promoted
+ else "unknown"
+ )
+ )
+ if delivery_certainty not in {"delivered", "not_delivered", "unknown"}:
+ delivery_certainty = "unknown"
+ retry_safe = bool(
+ not dispatch_promoted
+ and not accepted
+ and delivery_certainty == "not_delivered"
+ )
+ public_identity = identity.public_dict()
+ stage_identity = {
+ "run_id": str(identity.run_id),
+ "trace_id": identity.trace_id,
+ "work_item_id": identity.work_item_id,
+ }
+ return {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": (
+ "dispatch_accepted_verifier_pending"
+ if dispatch_promoted
+ else "dispatch_not_delivered_retry_safe"
+ if retry_safe
+ else "dispatch_delivery_unknown_reconcile_only"
+ ),
+ "identity": public_identity,
+ "dispatch_receipt": {
+ "schema_version": str(
+ dispatch_receipt.get("schema_version")
+ or "agent99_sre_dispatch_receipt_v1"
+ ),
+ "status": str(dispatch_receipt.get("status") or "unknown"),
+ "transport": str(dispatch_receipt.get("transport") or "unknown"),
+ "alert_id": str(dispatch_receipt.get("alert_id") or ""),
+ "kind": str(dispatch_receipt.get("kind") or ""),
+ "http_status": int(dispatch_receipt.get("http_status") or 0),
+ "accepted": accepted,
+ "inbox_triggered": inbox_triggered,
+ "delivery_certainty": delivery_certainty,
+ "stores_raw_response": False,
+ },
+ # Transport acceptance proves only that Agent99 accepted the inbox
+ # item. Runtime execution stays unproven until the independent verifier
+ # and durable learning acknowledgements write into this same run.
+ "dispatch_accepted": accepted,
+ "inbox_triggered": inbox_triggered,
+ "dispatch_promoted": dispatch_promoted,
+ "controlled_apply_authorized": bool(
+ dispatch_promoted
+ and (
+ dispatch_receipt.get("controlled_apply_authorized") is True
+ if controlled_apply_authorized is None
+ else controlled_apply_authorized
+ )
+ ),
+ "runtime_execution_authorized": False,
+ "runtime_execution_attempted": False,
+ "post_verifier_passed": False,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "needs_human": False,
+ "retry_policy": {
+ "safe_to_retry": retry_safe,
+ "requires_generation_advance": retry_safe,
+ "reconcile_only": bool(not dispatch_promoted and not retry_safe),
+ "max_generation": AGENT99_MAX_EXECUTION_GENERATION,
+ },
+ "outbox": {
+ "status": (
+ "delivered"
+ if dispatch_promoted
+ else "not_delivered"
+ if retry_safe
+ else "delivery_unknown"
+ ),
+ "durable_reservation": True,
+ "step_seq": 1,
+ **stage_identity,
+ },
+ "verifier": {
+ "status": (
+ "pending"
+ if dispatch_promoted
+ else "blocked_dispatch_not_promoted"
+ ),
+ "step_seq": 2,
+ **stage_identity,
+ "required_receipts": [
+ "agent99_outcome_terminal_readback",
+ "full_stack_cold_start_scorecard_rerun",
+ "source_fingerprint_resolution",
+ ],
+ },
+ "learning_writeback": {
+ "status": (
+ "pending_verifier"
+ if dispatch_promoted
+ else "blocked_dispatch_not_promoted"
+ ),
+ "step_seq": 3,
+ **stage_identity,
+ "required_receipts": [
+ "incident_closure_receipt",
+ "telegram_lifecycle_receipt",
+ "km_writeback_ack",
+ "playbook_trust_writeback_ack",
+ ],
+ },
+ }
+
+
+def _stage_identity(identity: Agent99DispatchIdentity) -> dict[str, str]:
+ return {
+ "run_id": str(identity.run_id),
+ "trace_id": identity.trace_id,
+ "work_item_id": identity.work_item_id,
+ }
+
+
+def _stage_identity_matches(
+ identity: Agent99DispatchIdentity,
+ receipt: dict[str, Any],
+) -> bool:
+ """Validate one complete canonical identity projection."""
+
+ supplied_identity = (
+ receipt.get("identity")
+ if isinstance(receipt.get("identity"), dict)
+ else receipt
+ )
+ if not isinstance(supplied_identity, dict):
+ return False
+ try:
+ parsed = parse_agent99_dispatch_identity(supplied_identity)
+ except ValueError:
+ return False
+ return parsed == identity and parsed.public_dict() == identity.public_dict()
+
+
+def _safe_receipt_refs(receipt_refs: dict[str, Any] | None) -> dict[str, str]:
+ """Keep only bounded durable identifiers, never raw evidence bodies."""
+
+ return sanitize_agent99_public_receipt_refs(receipt_refs)
+
+
+class PostgresAgent99DispatchLedger:
+ """PostgreSQL reservation and public-safe receipt store."""
+
+ async def reserve(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ payload: dict[str, Any],
+ ) -> dict[str, Any]:
+ identity_payload = identity.public_dict()
+ input_payload = {
+ "schema_version": "agent99_controlled_dispatch_input_v1",
+ "identity": identity_payload,
+ "kind": str(payload.get("kind") or ""),
+ "suggested_mode": str(payload.get("suggestedMode") or "Status"),
+ "controlled_apply": bool(payload.get("controlledApply") is True),
+ "target_resource": str((payload.get("awoooi") or {}).get("targetResource") or ""),
+ "stores_secrets": False,
+ }
+ input_json = _stable_json(input_payload)
+ now = _utc_now_naive()
+ claim_token = str(uuid4())
+ stage_identity = {
+ "run_id": str(identity.run_id),
+ "trace_id": identity.trace_id,
+ "work_item_id": identity.work_item_id,
+ }
+ reservation_envelope = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "dispatch_reserved",
+ "identity": identity_payload,
+ "dispatch_accepted": False,
+ "controlled_apply_requested": bool(input_payload["controlled_apply"]),
+ "controlled_apply_authorized": False,
+ "runtime_execution_authorized": False,
+ "runtime_execution_attempted": False,
+ "post_verifier_passed": False,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "outbox": {
+ "status": "pending_delivery",
+ "durable_reservation": True,
+ "step_seq": 1,
+ **stage_identity,
+ },
+ "verifier": {
+ "status": "pending_dispatch",
+ "step_seq": 2,
+ **stage_identity,
+ },
+ "learning_writeback": {
+ "status": "pending_verifier",
+ "step_seq": 3,
+ **stage_identity,
+ },
+ }
+
+ try:
+ async with get_db_context(identity.project_id) as db:
+ await db.execute(
+ pg_insert(AwoooPRunState)
+ .values(
+ run_id=identity.run_id,
+ project_id=identity.project_id,
+ agent_id=AGENT99_DISPATCH_AGENT_ID,
+ state="pending",
+ attempt_count=0,
+ max_attempts=3,
+ trace_id=identity.trace_id,
+ trigger_type=AGENT99_DISPATCH_TRIGGER_TYPE,
+ trigger_ref=identity.trigger_ref,
+ is_shadow=False,
+ input_sha256=_sha256(input_json),
+ step_count=3,
+ error_detail=_stable_json(reservation_envelope),
+ timeout_at=(
+ now + timedelta(minutes=AGENT99_DISPATCH_TIMEOUT_MINUTES)
+ ),
+ next_attempt_at=None,
+ )
+ .on_conflict_do_nothing(index_elements=[AwoooPRunState.run_id])
+ )
+ await db.execute(
+ pg_insert(AwoooPRunIdempotency)
+ .values(
+ project_id=identity.project_id,
+ channel_type=AGENT99_DISPATCH_CHANNEL,
+ provider_event_id=identity.idempotency_key,
+ run_id=identity.run_id,
+ )
+ .on_conflict_do_nothing(constraint="uix_run_idempotency_key")
+ )
+ claim = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.attempt_count < AwoooPRunState.max_attempts,
+ # A running claim is delivery-unknown after a worker
+ # crash and is therefore reconciliation-only. Only a
+ # durable pending row whose not-before elapsed may be
+ # claimed for transport.
+ and_(
+ AwoooPRunState.state == "pending",
+ or_(
+ AwoooPRunState.next_attempt_at.is_(None),
+ AwoooPRunState.next_attempt_at <= now,
+ ),
+ ),
+ )
+ .values(
+ state="running",
+ # A fresh token fences every claim/reclaim. The stable
+ # run identity must never be used as a worker token.
+ worker_id=claim_token,
+ started_at=now,
+ heartbeat_at=now,
+ lease_until=(
+ now + timedelta(seconds=AGENT99_DISPATCH_LEASE_SECONDS)
+ ),
+ next_attempt_at=None,
+ attempt_count=AwoooPRunState.attempt_count + 1,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ claimed = claim.scalar_one_or_none() is not None
+ await db.execute(
+ pg_insert(AwoooPRunStepJournal)
+ .values([
+ {
+ "run_id": identity.run_id,
+ "project_id": identity.project_id,
+ "step_seq": 1,
+ "tool_name": "agent99.controlled_dispatch.outbox",
+ "input_hash": _sha256(input_json),
+ "compensation_json": reservation_envelope,
+ "result_status": "pending",
+ "was_blocked": False,
+ },
+ {
+ "run_id": identity.run_id,
+ "project_id": identity.project_id,
+ "step_seq": 2,
+ "tool_name": "agent99.independent_post_verifier",
+ "input_hash": _sha256(identity.trace_id),
+ "compensation_json": reservation_envelope["verifier"],
+ "result_status": "pending",
+ "was_blocked": False,
+ },
+ {
+ "run_id": identity.run_id,
+ "project_id": identity.project_id,
+ "step_seq": 3,
+ "tool_name": "agent99.learning_writeback",
+ "input_hash": _sha256(identity.work_item_id),
+ "compensation_json": reservation_envelope[
+ "learning_writeback"
+ ],
+ "result_status": "pending",
+ "was_blocked": False,
+ },
+ ])
+ .on_conflict_do_nothing(constraint="uix_run_step_seq")
+ )
+ state_result = await db.execute(
+ select(
+ AwoooPRunState.state,
+ AwoooPRunState.error_detail,
+ AwoooPRunState.next_attempt_at,
+ ).where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ )
+ state_row = state_result.one_or_none()
+ except Exception as exc:
+ logger.warning(
+ "agent99_dispatch_pg_reservation_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "ledger_unavailable_fail_closed",
+ "claimed": False,
+ "identity": identity_payload,
+ "receipt": None,
+ }
+
+ existing = _parse_envelope(state_row.error_detail if state_row else None)
+ if claimed:
+ return {
+ "status": "reserved",
+ "claimed": True,
+ "identity": identity_payload,
+ "receipt": reservation_envelope,
+ "claim_token": claim_token,
+ }
+ next_attempt_at = getattr(state_row, "next_attempt_at", None)
+ retry_not_before = bool(next_attempt_at and next_attempt_at > now)
+ return {
+ "status": (
+ "retry_not_before"
+ if retry_not_before
+ else f"idempotent_{str(state_row.state if state_row else 'unknown')}"
+ ),
+ "claimed": False,
+ "identity": identity_payload,
+ "receipt": existing,
+ "next_attempt_at": (
+ next_attempt_at.isoformat() if next_attempt_at else None
+ ),
+ }
+
+ async def defer_claim_retryable(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ claim_token: str,
+ reason: str,
+ retry_after_seconds: int,
+ ) -> dict[str, Any]:
+ """Return an undispatched claim to pending without losing fencing.
+
+ A Redis lock collision or Redis outage occurs before transport. The run
+ therefore remains retryable; it must not be marked as a failed repair.
+ The claim token prevents an expired worker from changing a newer claim.
+ """
+
+ now = _utc_now_naive()
+ safe_reason = str(reason or "dispatch_lock_unavailable")[:128]
+ retry_after = max(1, min(int(retry_after_seconds or 1), 3600))
+ next_attempt_at = now + timedelta(seconds=retry_after)
+ try:
+ async with get_db_context(identity.project_id) as db:
+ current_result = await db.execute(
+ select(
+ AwoooPRunState.state,
+ AwoooPRunState.worker_id,
+ AwoooPRunState.attempt_count,
+ AwoooPRunState.max_attempts,
+ AwoooPRunState.error_detail,
+ )
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ current = current_result.one_or_none()
+ if (
+ current is None
+ or current.state != "running"
+ or str(current.worker_id or "") != str(claim_token or "")
+ ):
+ return {
+ "status": "claim_fenced_no_write",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ envelope = _parse_envelope(current.error_detail) or {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "identity": identity.public_dict(),
+ }
+ attempts_exhausted = bool(
+ int(current.attempt_count or 0) >= int(current.max_attempts or 0)
+ )
+ envelope.update({
+ "status": "dispatch_lock_retryable",
+ "dispatch_accepted": False,
+ "inbox_triggered": False,
+ "dispatch_promoted": False,
+ "controlled_apply_authorized": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "retry_policy": {
+ "safe_to_retry": True,
+ "retry_same_generation": not attempts_exhausted,
+ "requires_generation_advance": attempts_exhausted,
+ "reconcile_only": False,
+ "max_generation": AGENT99_MAX_EXECUTION_GENERATION,
+ "retry_after_seconds": retry_after,
+ "next_attempt_at": next_attempt_at.isoformat(),
+ "reason": safe_reason,
+ },
+ "outbox": {
+ "status": "lock_wait_retryable",
+ "durable_reservation": True,
+ "step_seq": 1,
+ **_stage_identity(identity),
+ },
+ })
+ envelope_json = _stable_json(envelope)
+ update_result = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state == "running",
+ AwoooPRunState.worker_id == claim_token,
+ )
+ .values(
+ state="pending",
+ worker_id=None,
+ lease_until=None,
+ next_attempt_at=next_attempt_at,
+ heartbeat_at=now,
+ error_detail=envelope_json,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ persisted = update_result.scalar_one_or_none() is not None
+ if persisted:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 1,
+ )
+ .values(
+ compensation_json=envelope,
+ result_status="pending",
+ error_code=None,
+ was_blocked=True,
+ block_reason=safe_reason,
+ completed_at=None,
+ )
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_dispatch_retry_defer_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "retry_defer_persistence_failed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "receipt_persisted": persisted,
+ "next_attempt_at": next_attempt_at.isoformat(),
+ "runtime_closure_verified": False,
+ }
+
+ async def complete(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ claim_token: str,
+ dispatch_receipt: dict[str, Any],
+ controlled_apply_authorized: bool | None = None,
+ ) -> dict[str, Any]:
+ envelope = build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt=dispatch_receipt,
+ controlled_apply_authorized=controlled_apply_authorized,
+ )
+ accepted = bool(dispatch_receipt.get("accepted") is True)
+ inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True)
+ dispatch_promoted = bool(accepted and inbox_triggered)
+ retry_safe = bool(
+ isinstance(envelope.get("retry_policy"), dict)
+ and envelope["retry_policy"].get("safe_to_retry") is True
+ )
+ now = _utc_now_naive()
+ next_attempt_at = (
+ now + timedelta(seconds=AGENT99_DISPATCH_RETRY_SECONDS)
+ if retry_safe
+ else None
+ )
+ if retry_safe:
+ envelope["retry_policy"].update({
+ "retry_after_seconds": AGENT99_DISPATCH_RETRY_SECONDS,
+ "next_attempt_at": next_attempt_at.isoformat(),
+ })
+ envelope_json = _stable_json(envelope)
+ try:
+ async with get_db_context(identity.project_id) as db:
+ result = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state.in_(["running", "waiting_tool"]),
+ AwoooPRunState.worker_id == claim_token,
+ )
+ .values(
+ # Delivery is not the terminal run state. A successful
+ # dispatch waits for the independent verifier and KM /
+ # PlayBook acknowledgements in steps 2 and 3.
+ state=(
+ "waiting_tool"
+ if dispatch_promoted or not retry_safe
+ else "failed"
+ ),
+ output_sha256=_sha256(envelope_json),
+ error_code=(
+ None
+ if dispatch_promoted
+ else "E-AGENT99-DISPATCH-NOT-DELIVERED"
+ if retry_safe
+ else "E-AGENT99-DISPATCH-UNKNOWN"
+ ),
+ error_detail=envelope_json,
+ completed_at=now if retry_safe else None,
+ heartbeat_at=now,
+ lease_until=None,
+ next_attempt_at=next_attempt_at,
+ worker_id=None,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ finalized = result.scalar_one_or_none() is not None
+ if finalized:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 1,
+ )
+ .values(
+ output_hash=_sha256(envelope_json),
+ compensation_json=envelope,
+ result_status=(
+ "success"
+ if dispatch_promoted
+ else "failed"
+ if retry_safe
+ else "pending"
+ ),
+ error_code=(
+ None
+ if dispatch_promoted
+ else "E-AGENT99-DISPATCH-NOT-DELIVERED"
+ if retry_safe
+ else None
+ ),
+ was_blocked=not dispatch_promoted,
+ block_reason=(
+ None
+ if dispatch_promoted
+ else "dispatch_not_delivered"
+ if retry_safe
+ else "dispatch_delivery_reconcile_pending"
+ ),
+ completed_at=now if dispatch_promoted or retry_safe else None,
+ )
+ )
+ if retry_safe:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq.in_([2, 3]),
+ AwoooPRunStepJournal.result_status == "pending",
+ )
+ .values(
+ result_status="failed",
+ error_code="E-AGENT99-DISPATCH-NOT-DELIVERED",
+ was_blocked=True,
+ block_reason="dispatch_not_accepted",
+ completed_at=now,
+ )
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_dispatch_pg_receipt_write_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ **envelope,
+ "status": "receipt_persistence_failed_no_runtime_closure",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "receipt_persisted": finalized,
+ "claim_fenced": not finalized,
+ "status": (
+ str(envelope.get("status") or "unknown")
+ if finalized
+ else "claim_fenced_no_write"
+ ),
+ "runtime_closure_verified": False,
+ }
+
+ async def mark_delivery_attempt_started(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ claim_token: str,
+ ) -> dict[str, Any]:
+ """Fence transport ambiguity before the first network/file write.
+
+ After this durable transition a worker crash is delivery-unknown. The
+ run stays in ``waiting_tool`` and only authenticated outcome readback
+ may advance it; an expired lease can never redispatch the mutation.
+ """
+
+ now = _utc_now_naive()
+ try:
+ async with get_db_context(identity.project_id) as db:
+ current_result = await db.execute(
+ select(
+ AwoooPRunState.state,
+ AwoooPRunState.worker_id,
+ AwoooPRunState.error_detail,
+ )
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ current = current_result.one_or_none()
+ if (
+ current is None
+ or current.state != "running"
+ or str(current.worker_id or "") != str(claim_token or "")
+ ):
+ return {
+ "status": "claim_fenced_no_write",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ envelope = _parse_envelope(current.error_detail) or {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "identity": identity.public_dict(),
+ }
+ envelope.update({
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "dispatch_accepted": False,
+ "inbox_triggered": False,
+ "dispatch_promoted": False,
+ "controlled_apply_authorized": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "retry_policy": {
+ "safe_to_retry": False,
+ "requires_generation_advance": False,
+ "reconcile_only": True,
+ "max_generation": AGENT99_MAX_EXECUTION_GENERATION,
+ },
+ "outbox": {
+ "status": "delivery_attempt_started_unknown",
+ "durable_reservation": True,
+ "step_seq": 1,
+ **_stage_identity(identity),
+ },
+ })
+ envelope_json = _stable_json(envelope)
+ updated = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state == "running",
+ AwoooPRunState.worker_id == claim_token,
+ )
+ .values(
+ state="waiting_tool",
+ error_code="E-AGENT99-DISPATCH-UNKNOWN",
+ error_detail=envelope_json,
+ heartbeat_at=now,
+ next_attempt_at=None,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ persisted = updated.scalar_one_or_none() is not None
+ if persisted:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 1,
+ )
+ .values(
+ compensation_json=envelope["outbox"],
+ result_status="pending",
+ error_code=None,
+ was_blocked=True,
+ block_reason="dispatch_delivery_reconcile_pending",
+ completed_at=None,
+ )
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_delivery_attempt_fence_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "delivery_attempt_fence_persistence_failed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "receipt_persisted": persisted,
+ "runtime_closure_verified": False,
+ }
+
+ async def record_verifier(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ outcome_receipt: dict[str, Any],
+ evidence_refs: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
+ """Persist Agent99 outcome + independent verifier on the same run."""
+
+ if not validate_agent99_outcome_identity(identity, outcome_receipt):
+ return {
+ "status": "verifier_identity_mismatch_fail_closed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ outcome = (
+ outcome_receipt.get("outcome")
+ if isinstance(outcome_receipt.get("outcome"), dict)
+ else outcome_receipt
+ )
+ schema_version = str(
+ outcome.get("schemaVersion") or outcome.get("schema_version") or ""
+ )
+ if schema_version != "agent99_outcome_contract_v1":
+ return {
+ "status": "verifier_schema_invalid_fail_closed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+
+ outcome_state = str(outcome.get("state") or "unknown")
+ verifier_passed = bool(outcome.get("verifierPassed") is True)
+ source_resolved = bool(outcome.get("sourceEventResolved") is True)
+ transport_ok = bool(outcome.get("transportOk") is True)
+ safe_refs = _safe_receipt_refs(evidence_refs)
+ required_evidence_refs = set(AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS)
+ mode = str(outcome_receipt.get("mode") or "")
+ if mode == "BackupCheck" or "backup_health" in identity.route_id:
+ required_evidence_refs.update(
+ AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS
+ )
+ missing_evidence_refs = sorted(required_evidence_refs - set(safe_refs))
+ if missing_evidence_refs:
+ return {
+ "status": "verifier_evidence_missing_fail_closed",
+ "missing_evidence_refs": missing_evidence_refs,
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ passed = bool(
+ outcome_state == "resolved"
+ and transport_ok
+ and verifier_passed
+ and source_resolved
+ )
+ execution_attempted = transport_ok
+ now = _utc_now_naive()
+
+ try:
+ async with get_db_context(identity.project_id) as db:
+ current_result = await db.execute(
+ select(AwoooPRunState.state, AwoooPRunState.error_detail)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ current = current_result.one_or_none()
+ envelope = _parse_envelope(
+ current.error_detail if current is not None else None
+ )
+ if (
+ current is None
+ or current.state not in {"pending", "running", "waiting_tool"}
+ or not isinstance(envelope, dict)
+ or not _stage_identity_matches(
+ identity,
+ envelope.get("identity")
+ if isinstance(envelope.get("identity"), dict)
+ else {},
+ )
+ ):
+ return {
+ "status": "verifier_run_not_ready_fail_closed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+
+ if (
+ current.state in {"pending", "running"}
+ or envelope.get("dispatch_promoted") is not True
+ ):
+ # A signed/allowlisted Agent99 outcome is the authoritative
+ # reconciliation receipt when transport acceptance was
+ # observed but the API could not persist its response.
+ envelope.update({
+ "dispatch_accepted": True,
+ "inbox_triggered": True,
+ "dispatch_promoted": True,
+ "controlled_apply_authorized": bool(
+ outcome_receipt.get("controlledApply") is True
+ ),
+ "dispatch_receipt": {
+ "schema_version": "agent99_sre_dispatch_receipt_v1",
+ "status": "outcome_readback_proves_delivery",
+ "transport": "agent99_relay_readback",
+ "alert_id": str(identity.run_id),
+ "kind": str(outcome_receipt.get("mode") or ""),
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ "stores_raw_response": False,
+ },
+ "outbox": {
+ "status": "delivered_reconciled_from_outcome",
+ "durable_reservation": True,
+ "step_seq": 1,
+ **_stage_identity(identity),
+ },
+ "retry_policy": {
+ "safe_to_retry": False,
+ "requires_generation_advance": False,
+ "reconcile_only": False,
+ "max_generation": AGENT99_MAX_EXECUTION_GENERATION,
+ },
+ })
+
+ verifier_receipt = {
+ "schema_version": "agent99_independent_verifier_receipt_v1",
+ "status": "success" if passed else "failed",
+ **_stage_identity(identity),
+ "outcome_schema_version": schema_version,
+ "outcome_state": outcome_state,
+ "transport_ok": transport_ok,
+ "verifier_passed": verifier_passed,
+ "source_event_resolved": source_resolved,
+ "verified_at": str(
+ outcome.get("verifiedAt")
+ or outcome.get("verified_at")
+ or ""
+ )[:64],
+ "evidence_refs": safe_refs,
+ "stores_raw_evidence": False,
+ }
+ envelope.update({
+ "status": (
+ "verifier_passed_learning_writeback_pending"
+ if passed
+ else "verifier_failed_no_runtime_closure"
+ ),
+ "runtime_execution_authorized": bool(
+ envelope.get("controlled_apply_authorized") is True
+ ),
+ "runtime_execution_attempted": execution_attempted,
+ "post_verifier_passed": passed,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "verifier": verifier_receipt,
+ "learning_writeback": {
+ **_stage_identity(identity),
+ "step_seq": 3,
+ "status": (
+ "pending_writeback"
+ if passed
+ else "blocked_verifier_failed"
+ ),
+ "required_receipts": [
+ "incident_closure_receipt",
+ "telegram_lifecycle_receipt",
+ "km_writeback_ack",
+ "playbook_trust_writeback_ack",
+ ],
+ },
+ })
+ envelope_json = _stable_json(envelope)
+ update_result = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state.in_([
+ "pending",
+ "running",
+ "waiting_tool",
+ ]),
+ )
+ .values(
+ state="waiting_tool" if passed else "failed",
+ output_sha256=_sha256(envelope_json),
+ error_code=None if passed else "E-AGENT99-VERIFY",
+ error_detail=envelope_json,
+ heartbeat_at=now,
+ completed_at=None if passed else now,
+ lease_until=None,
+ next_attempt_at=None,
+ worker_id=None,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ persisted = update_result.scalar_one_or_none() is not None
+ if persisted:
+ if current.state == "running":
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 1,
+ )
+ .values(
+ output_hash=_sha256(envelope_json),
+ compensation_json=envelope["outbox"],
+ result_status="success",
+ error_code=None,
+ was_blocked=False,
+ block_reason=None,
+ completed_at=now,
+ )
+ )
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 2,
+ )
+ .values(
+ output_hash=_sha256(_stable_json(verifier_receipt)),
+ compensation_json=verifier_receipt,
+ result_status="success" if passed else "failed",
+ error_code=None if passed else "E-AGENT99-VERIFY",
+ was_blocked=not passed,
+ block_reason=(
+ None if passed else "independent_verifier_failed"
+ ),
+ completed_at=now,
+ )
+ )
+ if not passed:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 3,
+ AwoooPRunStepJournal.result_status == "pending",
+ )
+ .values(
+ result_status="failed",
+ error_code="E-AGENT99-VERIFY",
+ was_blocked=True,
+ block_reason="independent_verifier_failed",
+ completed_at=now,
+ )
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_verifier_receipt_write_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "verifier_receipt_persistence_failed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "receipt_persisted": persisted,
+ "runtime_closure_verified": False,
+ }
+
+ async def record_learning_writeback(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ receipt_refs: dict[str, Any],
+ ) -> dict[str, Any]:
+ """Atomically close the incident, operation log, and same-run ledger.
+
+ KM, PlayBook, DR, and Telegram each provide a durable idempotent
+ acknowledgement first. The incident terminal state, immutable closure
+ event, run state, and step-3 receipt then commit in one PostgreSQL
+ transaction. Redis is only a post-commit projection.
+ """
+
+ required_refs = {
+ "telegram_lifecycle_receipt_id",
+ "km_writeback_ack_id",
+ "playbook_trust_writeback_ack_id",
+ }
+ if "backup_health" in identity.route_id:
+ required_refs.add("dr_scorecard_writeback_ack_id")
+ supplied_refs = sanitize_agent99_public_receipt_refs(
+ receipt_refs,
+ allowed_keys=(
+ AGENT99_LEARNING_RECEIPT_REFS
+ - {"incident_closure_receipt_id"}
+ ),
+ )
+ supplied_missing = sorted(required_refs - set(supplied_refs))
+ if supplied_refs and supplied_missing:
+ return {
+ "status": "learning_writeback_incomplete_fail_closed",
+ "missing_receipts": supplied_missing,
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ now = _utc_now_naive()
+ incident_now = datetime.now(UTC)
+ closure_event_id = str(uuid5(
+ NAMESPACE_URL,
+ f"agent99-terminal-bundle:{identity.project_id}:{identity.run_id}",
+ ))
+ try:
+ async with get_db_context(identity.project_id) as db:
+ current_result = await db.execute(
+ select(AwoooPRunState.state, AwoooPRunState.error_detail)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ current = current_result.one_or_none()
+ envelope = _parse_envelope(
+ current.error_detail if current is not None else None
+ )
+ if (
+ current is not None
+ and current.state == "completed"
+ and isinstance(envelope, dict)
+ and envelope.get("runtime_closure_verified") is True
+ and envelope.get("closure_complete") is True
+ and _stage_identity_matches(
+ identity,
+ envelope.get("identity")
+ if isinstance(envelope.get("identity"), dict)
+ else {},
+ )
+ ):
+ return {
+ **envelope,
+ "status": "closed_verified_learning_written_idempotent",
+ "receipt_persisted": True,
+ "postgres_terminal_bundle_committed": True,
+ "redis_projection_pending": True,
+ "runtime_closure_verified": True,
+ }
+ if (
+ current is None
+ or current.state != "waiting_tool"
+ or not isinstance(envelope, dict)
+ or envelope.get("post_verifier_passed") is not True
+ or not _stage_identity_matches(
+ identity,
+ envelope.get("identity")
+ if isinstance(envelope.get("identity"), dict)
+ else {},
+ )
+ ):
+ return {
+ "status": "learning_writeback_run_not_ready_fail_closed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+
+ learning_checkpoint = (
+ envelope.get("learning_writeback")
+ if isinstance(envelope.get("learning_writeback"), dict)
+ else {}
+ )
+ checkpoint_refs = sanitize_agent99_public_receipt_refs(
+ learning_checkpoint.get("receipt_refs")
+ if isinstance(learning_checkpoint.get("receipt_refs"), dict)
+ else {},
+ allowed_keys=(
+ AGENT99_LEARNING_RECEIPT_REFS
+ - {"incident_closure_receipt_id"}
+ ),
+ )
+ missing = sorted(required_refs - set(checkpoint_refs))
+ if missing:
+ return {
+ "status": (
+ "learning_writeback_checkpoint_incomplete_fail_closed"
+ ),
+ "missing_receipts": missing,
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ if supplied_refs and supplied_refs != checkpoint_refs:
+ return {
+ "status": (
+ "learning_writeback_checkpoint_mismatch_fail_closed"
+ ),
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ # Only the refs already committed by record_learning_asset_acks
+ # are allowed into the terminal bundle. Caller values are a
+ # consistency check, never the source of truth.
+ safe_refs = dict(checkpoint_refs)
+ safe_refs["incident_closure_receipt_id"] = (
+ f"alert_operation_log:{closure_event_id}"
+ )
+
+ incident_result = await db.execute(
+ select(IncidentRecord)
+ .where(
+ IncidentRecord.incident_id == identity.incident_id,
+ IncidentRecord.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ incident = incident_result.scalar_one_or_none()
+ if incident is None:
+ return {
+ "status": "incident_terminal_bundle_target_missing",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ incident_outcome = dict(incident.outcome or {})
+ incident_outcome.update({
+ "execution_success": True,
+ "effectiveness_score": max(
+ 4,
+ int(incident_outcome.get("effectiveness_score") or 0),
+ ),
+ "learning_notes": (
+ f"Agent99 same-run verifier passed: {identity.run_id}"
+ ),
+ "should_remember": True,
+ "automation_run_id": str(identity.run_id),
+ "trace_id": identity.trace_id,
+ "work_item_id": identity.work_item_id,
+ })
+
+ learning_receipt = {
+ "schema_version": "agent99_learning_writeback_receipt_v1",
+ "status": "success",
+ "step_seq": 3,
+ **_stage_identity(identity),
+ "receipt_refs": safe_refs,
+ "stores_raw_evidence": False,
+ }
+ envelope.update({
+ "status": "closed_verified_learning_written",
+ "runtime_closure_verified": True,
+ "closure_complete": True,
+ "learning_writeback": learning_receipt,
+ })
+ envelope_json = _stable_json(envelope)
+ await db.execute(
+ pg_insert(AlertOperationLog)
+ .values(
+ id=closure_event_id,
+ incident_id=identity.incident_id,
+ approval_id=identity.approval_id or None,
+ event_type="RESOLVED",
+ actor="agent99_controlled_dispatch_reconciler",
+ action_detail=(
+ "Agent99 verifier and durable learning receipts closed"
+ ),
+ success=True,
+ context={
+ "automation_run_id": str(identity.run_id),
+ "trace_id": identity.trace_id,
+ "work_item_id": identity.work_item_id,
+ "runtime_closure_verified": True,
+ "receipt_refs": safe_refs,
+ },
+ created_at=incident_now,
+ )
+ .on_conflict_do_nothing(
+ index_elements=[AlertOperationLog.id]
+ )
+ )
+ incident_update = await db.execute(
+ update(IncidentRecord)
+ .where(
+ IncidentRecord.incident_id == identity.incident_id,
+ IncidentRecord.project_id == identity.project_id,
+ )
+ .values(
+ status=IncidentStatus.RESOLVED,
+ outcome=incident_outcome,
+ resolved_at=incident_now,
+ updated_at=incident_now,
+ )
+ .returning(IncidentRecord.incident_id)
+ )
+ if incident_update.scalar_one_or_none() is None:
+ raise RuntimeError("agent99_incident_terminal_update_missing")
+ update_result = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state == "waiting_tool",
+ )
+ .values(
+ state="completed",
+ output_sha256=_sha256(envelope_json),
+ error_code=None,
+ error_detail=envelope_json,
+ heartbeat_at=now,
+ completed_at=now,
+ lease_until=None,
+ next_attempt_at=None,
+ worker_id=None,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ persisted = update_result.scalar_one_or_none() is not None
+ if not persisted:
+ raise RuntimeError("agent99_terminal_run_update_fenced")
+ terminal_step = await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 3,
+ )
+ .values(
+ output_hash=_sha256(_stable_json(learning_receipt)),
+ compensation_json=learning_receipt,
+ result_status="success",
+ error_code=None,
+ was_blocked=False,
+ block_reason=None,
+ completed_at=now,
+ )
+ .returning(AwoooPRunStepJournal.step_id)
+ )
+ if terminal_step.scalar_one_or_none() is None:
+ raise RuntimeError("agent99_terminal_step_update_missing")
+ except Exception as exc:
+ logger.warning(
+ "agent99_learning_writeback_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "learning_writeback_persistence_failed",
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "receipt_persisted": persisted,
+ "postgres_terminal_bundle_committed": persisted,
+ "redis_projection_pending": persisted,
+ "runtime_closure_verified": bool(
+ persisted and envelope.get("runtime_closure_verified") is True
+ ),
+ }
+
+ async def record_learning_asset_acks(
+ self,
+ *,
+ identity: Agent99DispatchIdentity,
+ receipt_refs: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
+ """Checkpoint per-run learning side effects before terminal closure.
+
+ Reconciliation may crash after KM, PlayBook, Telegram, or DR succeeds.
+ Persisting each public acknowledgement on the same run lets the next
+ tick resume at the first missing asset instead of replaying completed
+ side effects. The asset writers retain their own deterministic keys as
+ the final crash fence between side effect and this checkpoint.
+ """
+
+ supplied = sanitize_agent99_public_receipt_refs(
+ receipt_refs,
+ allowed_keys=(
+ AGENT99_LEARNING_RECEIPT_REFS
+ - {"incident_closure_receipt_id"}
+ ),
+ )
+ now = _utc_now_naive()
+ try:
+ async with get_db_context(identity.project_id) as db:
+ current_result = await db.execute(
+ select(AwoooPRunState.state, AwoooPRunState.error_detail)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ )
+ .with_for_update()
+ )
+ current = current_result.one_or_none()
+ envelope = _parse_envelope(
+ current.error_detail if current is not None else None
+ )
+ if (
+ current is None
+ or current.state not in {"waiting_tool", "completed"}
+ or not isinstance(envelope, dict)
+ or envelope.get("post_verifier_passed") is not True
+ or not _stage_identity_matches(
+ identity,
+ envelope.get("identity")
+ if isinstance(envelope.get("identity"), dict)
+ else {},
+ )
+ ):
+ return {
+ "status": "learning_asset_ack_run_not_ready_fail_closed",
+ "receipt_persisted": False,
+ "learning_receipt_refs": {},
+ "runtime_closure_verified": False,
+ }
+
+ learning = (
+ dict(envelope.get("learning_writeback"))
+ if isinstance(envelope.get("learning_writeback"), dict)
+ else {}
+ )
+ existing = sanitize_agent99_public_receipt_refs(
+ learning.get("receipt_refs")
+ if isinstance(learning.get("receipt_refs"), dict)
+ else {},
+ allowed_keys=AGENT99_LEARNING_RECEIPT_REFS,
+ )
+ merged = {**existing, **supplied}
+ if current.state == "completed":
+ return {
+ **envelope,
+ "status": "learning_asset_acks_terminal_readback",
+ "receipt_persisted": True,
+ "learning_receipt_refs": merged,
+ "runtime_closure_verified": bool(
+ envelope.get("runtime_closure_verified") is True
+ ),
+ }
+
+ learning.update({
+ "status": "pending_writeback",
+ "step_seq": 3,
+ **_stage_identity(identity),
+ "receipt_refs": merged,
+ "acknowledged_receipts": sorted(merged),
+ "stores_raw_evidence": False,
+ })
+ envelope["learning_writeback"] = learning
+ envelope_json = _stable_json(envelope)
+ updated = await db.execute(
+ update(AwoooPRunState)
+ .where(
+ AwoooPRunState.run_id == identity.run_id,
+ AwoooPRunState.project_id == identity.project_id,
+ AwoooPRunState.state == "waiting_tool",
+ )
+ .values(
+ error_detail=envelope_json,
+ heartbeat_at=now,
+ )
+ .returning(AwoooPRunState.run_id)
+ )
+ persisted = updated.scalar_one_or_none() is not None
+ if persisted:
+ await db.execute(
+ update(AwoooPRunStepJournal)
+ .where(
+ AwoooPRunStepJournal.run_id == identity.run_id,
+ AwoooPRunStepJournal.step_seq == 3,
+ )
+ .values(
+ compensation_json=learning,
+ result_status="pending",
+ error_code=None,
+ was_blocked=False,
+ block_reason=None,
+ completed_at=None,
+ )
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_learning_asset_ack_failed_fail_closed",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ "status": "learning_asset_ack_persistence_failed",
+ "receipt_persisted": False,
+ "learning_receipt_refs": {},
+ "runtime_closure_verified": False,
+ }
+ return {
+ **envelope,
+ "status": "learning_asset_acks_recorded",
+ "receipt_persisted": persisted,
+ "learning_receipt_refs": merged,
+ "runtime_closure_verified": False,
+ }
+
+ async def read_latest_for_incident(
+ self,
+ *,
+ project_id: str,
+ incident_id: str,
+ ) -> dict[str, Any] | None:
+ if not incident_id:
+ return None
+ try:
+ async with get_db_context(project_id or "awoooi") as db:
+ result = await db.execute(
+ select(
+ AwoooPRunState.run_id,
+ AwoooPRunState.state,
+ AwoooPRunState.trace_id,
+ AwoooPRunState.error_detail,
+ )
+ .where(
+ AwoooPRunState.project_id == (project_id or "awoooi"),
+ AwoooPRunState.agent_id == AGENT99_DISPATCH_AGENT_ID,
+ AwoooPRunState.trigger_ref.like(f"agent99:{incident_id}:%"),
+ )
+ .order_by(AwoooPRunState.created_at.desc())
+ .limit(1)
+ )
+ row = result.one_or_none()
+ envelope = _parse_envelope(row.error_detail if row else None)
+ if not isinstance(envelope, dict) or row is None:
+ return None
+ return {
+ **envelope,
+ "run_state": str(row.state),
+ "run_id": str(row.run_id),
+ "trace_id": str(row.trace_id or ""),
+ }
+ except Exception as exc:
+ logger.warning(
+ "agent99_dispatch_receipt_read_failed",
+ incident_id=incident_id,
+ error=str(exc),
+ )
+ return None
+
+ async def list_reconcilable(
+ self,
+ *,
+ project_id: str = "awoooi",
+ limit: int = 20,
+ ) -> list[dict[str, Any]]:
+ """List durable deliveries that need outcome or learning reconciliation."""
+
+ try:
+ async with get_db_context(project_id or "awoooi") as db:
+ result = await db.execute(
+ select(
+ AwoooPRunState.run_id,
+ AwoooPRunState.state,
+ AwoooPRunState.error_detail,
+ )
+ .where(
+ AwoooPRunState.project_id == (project_id or "awoooi"),
+ AwoooPRunState.agent_id == AGENT99_DISPATCH_AGENT_ID,
+ AwoooPRunState.state.in_([
+ "pending",
+ "running",
+ "waiting_tool",
+ ]),
+ )
+ # Prioritize observed delivery/outcome lanes; pending crash
+ # recovery candidates are polled after them.
+ .order_by(
+ (AwoooPRunState.state == "pending").asc(),
+ AwoooPRunState.created_at.asc(),
+ )
+ .limit(max(1, min(int(limit), 100)))
+ )
+ rows = result.all()
+ except Exception as exc:
+ logger.warning(
+ "agent99_dispatch_reconciliation_list_failed",
+ project_id=project_id,
+ error=str(exc),
+ )
+ return []
+
+ items: list[dict[str, Any]] = []
+ for row in rows:
+ envelope = _parse_envelope(row.error_detail)
+ public_identity = (
+ envelope.get("identity")
+ if isinstance(envelope, dict)
+ and isinstance(envelope.get("identity"), dict)
+ else None
+ )
+ if not isinstance(public_identity, dict):
+ continue
+ try:
+ identity = agent99_dispatch_identity_from_public(public_identity)
+ except ValueError:
+ logger.warning(
+ "agent99_dispatch_reconciliation_identity_invalid",
+ run_id=str(row.run_id),
+ )
+ continue
+ items.append({
+ "identity": identity,
+ "run_state": str(row.state),
+ "receipt": envelope,
+ })
+ return items
+
+
+def _parse_envelope(value: Any) -> dict[str, Any] | None:
+ if isinstance(value, dict):
+ return value
+ if not value:
+ return None
+ try:
+ parsed = json.loads(str(value))
+ except (TypeError, ValueError, json.JSONDecodeError):
+ return None
+ return parsed if isinstance(parsed, dict) else None
+
+
+_ledger = PostgresAgent99DispatchLedger()
+
+
+def get_agent99_dispatch_ledger() -> PostgresAgent99DispatchLedger:
+ return _ledger
+
+
+async def read_agent99_dispatch_receipt(
+ *,
+ project_id: str,
+ incident_id: str,
+) -> dict[str, Any] | None:
+ return await _ledger.read_latest_for_incident(
+ project_id=project_id,
+ incident_id=incident_id,
+ )
+
+
+async def list_agent99_reconcilable_dispatches(
+ *,
+ project_id: str = "awoooi",
+ limit: int = 20,
+) -> list[dict[str, Any]]:
+ return await _ledger.list_reconcilable(
+ project_id=project_id,
+ limit=limit,
+ )
+
+
+async def record_agent99_verifier_receipt(
+ *,
+ identity: Agent99DispatchIdentity,
+ outcome_receipt: dict[str, Any],
+ evidence_refs: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+ return await _ledger.record_verifier(
+ identity=identity,
+ outcome_receipt=outcome_receipt,
+ evidence_refs=evidence_refs,
+ )
+
+
+async def record_agent99_learning_writeback(
+ *,
+ identity: Agent99DispatchIdentity,
+ receipt_refs: dict[str, Any],
+) -> dict[str, Any]:
+ result = await _ledger.record_learning_writeback(
+ identity=identity,
+ receipt_refs=receipt_refs,
+ )
+ if result.get("postgres_terminal_bundle_committed") is not True:
+ return result
+
+ projection_acknowledged = False
+ try:
+ # PostgreSQL is source-of-truth. Working memory is refreshed only
+ # after the terminal transaction above has committed successfully.
+ from src.services.incident_service import get_incident_service
+
+ incident_service = get_incident_service()
+ incident = await incident_service.get_from_episodic_memory(
+ identity.incident_id,
+ project_id=identity.project_id,
+ )
+ projection_acknowledged = bool(
+ incident is not None
+ and await incident_service.save_to_working_memory(incident)
+ )
+ except Exception as exc:
+ logger.warning(
+ "agent99_incident_redis_projection_pending",
+ incident_id=identity.incident_id,
+ run_id=str(identity.run_id),
+ error=str(exc),
+ )
+ return {
+ **result,
+ "redis_projection_pending": not projection_acknowledged,
+ "redis_projection_acknowledged": projection_acknowledged,
+ }
diff --git a/apps/api/src/services/agent99_outcome_ingestion.py b/apps/api/src/services/agent99_outcome_ingestion.py
new file mode 100644
index 000000000..cab4a1d61
--- /dev/null
+++ b/apps/api/src/services/agent99_outcome_ingestion.py
@@ -0,0 +1,119 @@
+"""Production ingestion for Agent99 outcome, verifier, and learning receipts."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from src.services.agent99_controlled_dispatch_ledger import (
+ parse_agent99_dispatch_identity,
+ record_agent99_verifier_receipt,
+ validate_agent99_outcome_identity,
+)
+from src.services.agent99_public_receipts import (
+ extract_agent99_outcome_evidence_refs,
+)
+
+
+def _public_evidence_refs(
+ payload_refs: dict[str, Any],
+ outcome_receipt: dict[str, Any],
+) -> dict[str, str]:
+ """Merge only bounded, public-safe identifiers from authenticated input."""
+
+ return extract_agent99_outcome_evidence_refs(
+ outcome_receipt,
+ payload_refs,
+ )
+
+
+async def ingest_agent99_outcome_receipt(
+ payload: dict[str, Any],
+) -> dict[str, Any]:
+ """Write the Agent99 outcome and optional learning acks to one run.
+
+ The caller must carry the complete deterministic identity both in the
+ request identity and in the Agent99 outcome receipt. This prevents a valid
+ outcome from being rebound to another incident or execution generation.
+ """
+
+ identity_payload = (
+ payload.get("identity")
+ if isinstance(payload.get("identity"), dict)
+ else {}
+ )
+ identity = parse_agent99_dispatch_identity(identity_payload)
+ outcome_receipt = (
+ payload.get("outcome_receipt")
+ if isinstance(payload.get("outcome_receipt"), dict)
+ else {}
+ )
+ if not outcome_receipt:
+ raise ValueError("agent99_outcome_receipt_required")
+ if not validate_agent99_outcome_identity(identity, outcome_receipt):
+ raise ValueError("agent99_outcome_identity_mismatch")
+ payload_evidence_refs = (
+ payload.get("evidence_refs")
+ if isinstance(payload.get("evidence_refs"), dict)
+ else {}
+ )
+ evidence_refs = _public_evidence_refs(
+ payload_evidence_refs,
+ outcome_receipt,
+ )
+ verifier = await record_agent99_verifier_receipt(
+ identity=identity,
+ outcome_receipt=outcome_receipt,
+ evidence_refs=evidence_refs,
+ )
+ if verifier.get("receipt_persisted") is not True:
+ return {
+ "schema_version": "agent99_outcome_ingestion_receipt_v1",
+ "status": str(verifier.get("status") or "verifier_rejected"),
+ "accepted": False,
+ "identity": identity.public_dict(),
+ "verifier": verifier,
+ "learning_writeback": {
+ "status": "blocked_verifier_not_persisted",
+ "receipt_persisted": False,
+ },
+ "runtime_closure_verified": False,
+ }
+
+ # The relay is authoritative for the Agent99 outcome, but it is not the
+ # authority for Telegram/KM/PlayBook/DR delivery acknowledgements. Those
+ # side effects are created and checkpointed by the server-side reconciler.
+ # Keep the compatibility field in the webhook schema, but never let its
+ # caller-provided values enter the terminal transaction.
+ untrusted_learning_refs = (
+ payload.get("learning_receipt_refs")
+ if isinstance(payload.get("learning_receipt_refs"), dict)
+ else {}
+ )
+ learning = {
+ "status": (
+ "untrusted_external_receipts_ignored_reconciler_pending"
+ if verifier.get("post_verifier_passed") is True
+ and untrusted_learning_refs
+ else "pending_reconciler_durable_receipts"
+ if verifier.get("post_verifier_passed") is True
+ else "blocked_verifier_failed"
+ ),
+ "receipt_persisted": False,
+ "runtime_closure_verified": False,
+ }
+ return {
+ "schema_version": "agent99_outcome_ingestion_receipt_v1",
+ "status": (
+ "verifier_recorded_learning_pending"
+ if verifier.get("post_verifier_passed") is True
+ else "verifier_failed_no_runtime_closure"
+ ),
+ "accepted": True,
+ "identity": identity.public_dict(),
+ "verifier": verifier,
+ "learning_writeback": learning,
+ "untrusted_learning_receipt_refs_ignored": bool(
+ untrusted_learning_refs
+ ),
+ "runtime_closure_verified": False,
+ }
diff --git a/apps/api/src/services/agent99_public_receipts.py b/apps/api/src/services/agent99_public_receipts.py
new file mode 100644
index 000000000..5851358fd
--- /dev/null
+++ b/apps/api/src/services/agent99_public_receipts.py
@@ -0,0 +1,108 @@
+"""Strict public receipt identifiers shared by every Agent99 readback path.
+
+Receipt references are durable identifiers, never log bodies, filesystem
+paths, URLs with query strings, credentials, or arbitrary caller metadata.
+Keeping the grammar in one module prevents the webhook and relay reconciler
+from accepting different evidence for the same run.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterable, Mapping
+from typing import Any
+
+AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS = frozenset({
+ "agent99_outcome_receipt_id",
+ "post_verifier_evidence_ref",
+ "source_event_evidence_ref",
+})
+AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS = frozenset({
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+})
+AGENT99_LEARNING_RECEIPT_REFS = frozenset({
+ "incident_closure_receipt_id",
+ "telegram_lifecycle_receipt_id",
+ "km_writeback_ack_id",
+ "playbook_trust_writeback_ack_id",
+ "dr_scorecard_writeback_ack_id",
+})
+AGENT99_PUBLIC_RECEIPT_REF_KEYS = (
+ AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS
+ | AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS
+ | AGENT99_LEARNING_RECEIPT_REFS
+)
+
+# URN-like public identifiers only. Deliberately excludes whitespace,
+# backslashes, query delimiters, fragments, and assignment (``=``).
+_PUBLIC_RECEIPT_REF_RE = re.compile(
+ r"^[A-Za-z0-9][A-Za-z0-9._:/@+%-]{0,255}$"
+)
+_SECRET_SHAPED_RE = re.compile(
+ r"(?:authorization|bearer|password|passwd|token|secret|api[_-]?key|"
+ r"private[_-]?key|cookie|session)(?:\s|:|=)|"
+ r"-----BEGIN[^-]*PRIVATE KEY-----|(?:^|[/.])\.env(?:$|[/.])",
+ re.IGNORECASE,
+)
+
+
+def normalize_agent99_public_receipt_ref(value: Any) -> str | None:
+ """Return one bounded public receipt id, or ``None`` when unsafe."""
+
+ ref = str(value or "").strip()
+ if not ref or len(ref) > 256 or not ref.isascii():
+ return None
+ if _SECRET_SHAPED_RE.search(ref) or not _PUBLIC_RECEIPT_REF_RE.fullmatch(ref):
+ return None
+ if ".." in ref or "://" in ref:
+ return None
+ return ref
+
+
+def sanitize_agent99_public_receipt_refs(
+ *sources: Mapping[str, Any] | None,
+ allowed_keys: Iterable[str] = AGENT99_PUBLIC_RECEIPT_REF_KEYS,
+) -> dict[str, str]:
+ """Merge allowlisted refs using the same strict grammar everywhere."""
+
+ allowed = frozenset(str(key) for key in allowed_keys)
+ safe: dict[str, str] = {}
+ for source in sources:
+ for raw_key, raw_value in (source or {}).items():
+ key = str(raw_key or "").strip()
+ if key not in allowed:
+ continue
+ ref = normalize_agent99_public_receipt_ref(raw_value)
+ if ref is not None:
+ safe[key] = ref
+ return safe
+
+
+def extract_agent99_outcome_evidence_refs(
+ outcome_receipt: Mapping[str, Any] | None,
+ *additional_sources: Mapping[str, Any] | None,
+) -> dict[str, str]:
+ """Extract generic and BackupCheck refs from one outcome envelope."""
+
+ receipt = outcome_receipt or {}
+ outcome = receipt.get("outcome")
+ if not isinstance(outcome, Mapping):
+ outcome = receipt
+ nested_refs = outcome.get("evidenceRefs")
+ if not isinstance(nested_refs, Mapping):
+ nested_refs = outcome.get("evidence_refs")
+ if not isinstance(nested_refs, Mapping):
+ nested_refs = {}
+ return sanitize_agent99_public_receipt_refs(
+ *additional_sources,
+ nested_refs,
+ allowed_keys=(
+ AGENT99_REQUIRED_VERIFIER_EVIDENCE_REFS
+ | AGENT99_BACKUP_REQUIRED_VERIFIER_EVIDENCE_REFS
+ ),
+ )
diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py
index 1c733dd79..8dabf1df3 100644
--- a/apps/api/src/services/agent99_sre_bridge.py
+++ b/apps/api/src/services/agent99_sre_bridge.py
@@ -13,11 +13,30 @@ from typing import Any
from src.core.config import settings
from src.core.logging import get_logger
+from src.services.agent99_controlled_dispatch_ledger import (
+ attach_agent99_dispatch_identity,
+ build_agent99_dispatch_identity,
+ get_agent99_dispatch_ledger,
+ parse_agent99_dispatch_identity,
+)
+from src.services.agent99_public_receipts import (
+ extract_agent99_outcome_evidence_refs,
+)
from src.utils.timezone import now_taipei
logger = get_logger("awoooi.agent99_sre_bridge")
AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600
+AGENT99_DURABLE_ROUTE_KINDS = {
+ "provider_freshness_signal",
+ "public_route_502",
+ "performance_pressure",
+ "backup_health",
+ "security_health",
+ "harbor_registry",
+ "awoooi_k3s",
+ "host_recovery",
+}
_SENSITIVE_KEY_RE = re.compile(
r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)",
@@ -95,6 +114,25 @@ def resolve_agent99_alert_kind(
annotations=annotations or {},
)
+ # Explicit backup identities win over generic freshness words. Backup
+ # failure cards intentionally mention freshness, but must stay in the
+ # read-only BackupCheck lane with its stricter no-delete/no-restore policy.
+ backup_identity = " ".join(
+ [
+ alertname,
+ target_resource,
+ str((labels or {}).get("event_type") or ""),
+ str((labels or {}).get("lane") or ""),
+ ]
+ ).lower()
+ if re.search(
+ r"backup_restore_escrow|backup[-_ ]restore|backupcredentialescrow|"
+ r"backupaggregaterun|backupconfigcapture|escrow_missing|"
+ r"restore[_ -]drill|restic|rclone|offsite",
+ backup_identity,
+ ):
+ return "backup_health"
+
if re.search(
r"host112guest(?:readbackmissing|notready)|host112_guest|"
r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)",
@@ -181,6 +219,37 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
}.get(kind)
+def resolve_agent99_durable_route(
+ *,
+ alertname: str,
+ severity: str,
+ target_resource: str,
+ namespace: str,
+ message: str,
+ labels: dict[str, Any] | None = None,
+ annotations: dict[str, Any] | None = None,
+) -> dict[str, str] | None:
+ """Return the single-owner route for Agent99-actionable domains."""
+
+ kind = resolve_agent99_alert_kind(
+ alertname=alertname,
+ severity=severity,
+ target_resource=target_resource,
+ namespace=namespace,
+ message=message,
+ labels=labels,
+ annotations=annotations,
+ )
+ if kind not in AGENT99_DURABLE_ROUTE_KINDS:
+ return None
+ suggested_mode = _suggested_mode_for_kind(kind) or "Status"
+ return {
+ "kind": kind,
+ "suggested_mode": suggested_mode,
+ "route_id": f"agent99:{kind}:{suggested_mode}",
+ }
+
+
def _resolution_policy_for_kind(kind: str) -> str:
if kind in {
"provider_freshness_signal",
@@ -221,10 +290,39 @@ async def try_acquire_agent99_sre_single_flight(
*,
ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
) -> bool:
- """Grant one Agent99 dispatch per fingerprint inside the controlled window."""
+ """Compatibility wrapper for read-only/non-mutating callers."""
+
+ decision = await acquire_agent99_sre_single_flight(
+ fingerprint,
+ alert_id,
+ ttl_seconds=ttl_seconds,
+ fail_closed=False,
+ )
+ return bool(decision["acquired"])
+
+
+async def acquire_agent99_sre_single_flight(
+ fingerprint: str,
+ alert_id: str,
+ *,
+ ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
+ fail_closed: bool,
+) -> dict[str, Any]:
+ """Acquire the shared Redis lock with explicit failure semantics.
+
+ Mutating modes pass ``fail_closed=True``. Redis unavailability therefore
+ produces zero dispatches instead of silently allowing two Recover actions.
+ """
if not fingerprint:
- return True
+ return {
+ "acquired": not fail_closed,
+ "reason": (
+ "missing_idempotency_identity_fail_closed"
+ if fail_closed
+ else "missing_identity_readonly_allowed"
+ ),
+ }
try:
from src.core.redis_client import get_redis
@@ -235,15 +333,29 @@ async def try_acquire_agent99_sre_single_flight(
ex=ttl_seconds,
nx=True,
)
- return bool(acquired)
+ return {
+ "acquired": bool(acquired),
+ "reason": "acquired" if acquired else "single_flight_duplicate",
+ }
except Exception as exc:
logger.warning(
- "agent99_sre_single_flight_failed_fail_open",
+ (
+ "agent99_sre_single_flight_failed_fail_closed"
+ if fail_closed
+ else "agent99_sre_single_flight_failed_readonly_fail_open"
+ ),
fingerprint=fingerprint,
alert_id=alert_id,
error=str(exc),
)
- return True
+ return {
+ "acquired": not fail_closed,
+ "reason": (
+ "redis_unavailable_fail_closed"
+ if fail_closed
+ else "redis_unavailable_readonly_fail_open"
+ ),
+ }
async def release_agent99_sre_single_flight(fingerprint: str, alert_id: str) -> None:
@@ -335,6 +447,8 @@ def build_agent99_sre_alert(
)
if kind == "provider_freshness_signal":
safe_labels.extend(["provider-freshness", "no-false-green"])
+ elif kind == "backup_health":
+ safe_labels.extend(["backup-readback", "no-false-green", "read-only"])
instruction = None
if kind == "provider_freshness_signal":
@@ -343,6 +457,13 @@ def build_agent99_sre_alert(
"directReference,verifierReadback; mark missing,expired,timeout,"
"no_false_green; do not switch provider, call paid model, edit env, or reload."
)
+ elif kind == "backup_health":
+ instruction = (
+ "BackupCheck readback candidate only; collect backup status, freshness, "
+ "offsite verification, escrow missing count, restore drill and source "
+ "resolution receipts; no_false_green; do not run backup/restore, prune, "
+ "remote delete, change retention, write escrow markers, or read secrets."
+ )
elif kind == "host112_guest_recovery":
instruction = (
"Run the allowlisted host 112 guest recovery; verify graphical.target, "
@@ -495,6 +616,130 @@ def post_agent99_sre_alert(
return {"status": status, "body": body}
+def read_agent99_sre_outcome(
+ run_id: str,
+ *,
+ relay_url: str | None = None,
+ relay_token: str | None = None,
+ timeout_seconds: float | None = None,
+) -> dict[str, Any] | None:
+ """Read one public-safe Agent99 outcome from the authenticated relay."""
+
+ target_url = (
+ relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
+ )
+ token = (
+ relay_token
+ if relay_token is not None
+ else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
+ )
+ if not target_url or not token:
+ return None
+ timeout = (
+ timeout_seconds
+ if timeout_seconds is not None
+ else settings.AGENT99_SRE_ALERT_RELAY_TIMEOUT_SECONDS
+ )
+ parsed = urllib.parse.urlsplit(target_url)
+ query = urllib.parse.urlencode({"run_id": str(run_id)})
+ readback_url = urllib.parse.urlunsplit(
+ (parsed.scheme, parsed.netloc, parsed.path, query, "")
+ )
+ request = urllib.request.Request(
+ readback_url,
+ headers={
+ "Accept": "application/json",
+ "User-Agent": "awoooi-agent99-outcome-reconciler/1",
+ "X-Agent99-Relay-Token": token,
+ },
+ method="GET",
+ )
+ try:
+ with urllib.request.urlopen(request, timeout=timeout) as response:
+ if int(response.getcode()) != 200:
+ return None
+ body = response.read(65536).decode("utf-8", errors="replace")
+ except (urllib.error.HTTPError, urllib.error.URLError, TimeoutError) as exc:
+ logger.info(
+ "agent99_outcome_readback_pending",
+ run_id=str(run_id),
+ error_type=type(exc).__name__,
+ )
+ return None
+ try:
+ payload = json.loads(body)
+ except json.JSONDecodeError:
+ return None
+ if not isinstance(payload, dict) or payload.get("found") is not True:
+ return None
+ if str(payload.get("automationRunId") or "") != str(run_id):
+ return None
+ raw_identity = (
+ payload.get("identity")
+ if isinstance(payload.get("identity"), dict)
+ else {}
+ )
+ outcome = payload.get("outcome") if isinstance(payload.get("outcome"), dict) else {}
+ nested_identity = (
+ outcome.get("identity")
+ if isinstance(outcome.get("identity"), dict)
+ else {}
+ )
+ try:
+ identity = parse_agent99_dispatch_identity(raw_identity)
+ nested = parse_agent99_dispatch_identity(nested_identity)
+ except ValueError:
+ return None
+ if (
+ identity != nested
+ or str(identity.run_id) != str(run_id)
+ or identity.public_dict() != nested.public_dict()
+ ):
+ return None
+ safe_evidence_refs = extract_agent99_outcome_evidence_refs(payload)
+ return {
+ "schemaVersion": "agent99_outcome_readback_v1",
+ "automationRunId": str(payload.get("automationRunId") or ""),
+ "traceId": str(payload.get("traceId") or ""),
+ "workItemId": str(payload.get("workItemId") or ""),
+ "idempotencyKey": str(payload.get("idempotencyKey") or ""),
+ "executionGeneration": str(payload.get("executionGeneration") or "1"),
+ "incidentId": str(payload.get("incidentId") or ""),
+ "approvalId": str(payload.get("approvalId") or ""),
+ "identity": identity.public_dict(),
+ "controlledApply": bool(payload.get("controlledApply") is True),
+ "mode": str(payload.get("mode") or ""),
+ "outcome": {
+ "schemaVersion": str(outcome.get("schemaVersion") or ""),
+ "state": str(outcome.get("state") or "unknown"),
+ "resolved": bool(outcome.get("resolved") is True),
+ "transportOk": bool(outcome.get("transportOk") is True),
+ "verifierName": str(outcome.get("verifierName") or ""),
+ "verifierPassed": bool(outcome.get("verifierPassed") is True),
+ "sourceEventResolved": bool(
+ outcome.get("sourceEventResolved") is True
+ ),
+ "failedChecks": [
+ str(value)[:120]
+ for value in (outcome.get("failedChecks") or [])[:32]
+ ],
+ "checks": [
+ {
+ "name": str(check.get("name") or "")[:120],
+ "required": bool(check.get("required") is True),
+ "passed": bool(check.get("passed") is True),
+ }
+ for check in (outcome.get("checks") or [])[:32]
+ if isinstance(check, dict)
+ ],
+ "identity": identity.public_dict(),
+ "evidenceRefs": safe_evidence_refs,
+ "verifiedAt": str(outcome.get("verifiedAt") or "")[:64],
+ },
+ "storesRawEvidence": False,
+ }
+
+
def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str:
receipt = dispatch_agent99_sre_alert_with_receipt(payload)
return str(receipt["transport"])
@@ -534,6 +779,13 @@ def dispatch_agent99_sre_alert_with_receipt(
"http_status": http_status,
"accepted": accepted,
"inbox_triggered": inbox_triggered,
+ "delivery_certainty": (
+ "delivered"
+ if accepted and inbox_triggered
+ else "unknown"
+ if accepted
+ else "not_delivered"
+ ),
"stores_raw_response": False,
}
@@ -548,6 +800,8 @@ def dispatch_agent99_sre_alert_with_receipt(
"kind": kind,
"accepted": True,
"inbox_triggered": False,
+ # A file write is not proof that the Agent99 inbox consumed it.
+ "delivery_certainty": "unknown",
"stores_raw_response": False,
}
@@ -565,6 +819,7 @@ def dispatch_agent99_sre_alert_with_receipt(
"kind": kind,
"accepted": False,
"inbox_triggered": False,
+ "delivery_certainty": "not_delivered",
"stores_raw_response": False,
}
@@ -583,9 +838,20 @@ async def bridge_alertmanager_to_agent99(
alert_category: str = "",
notification_type: str = "",
source_url: str | None = None,
+ project_id: str = "awoooi",
+ incident_id: str = "",
+ approval_id: str = "",
+ route_id: str = "",
+ work_item_id: str = "",
+ execution_generation: str = "1",
) -> dict[str, Any]:
- single_flight_fingerprint = ""
+ single_flight_identity = ""
+ single_flight_owner = alert_id
single_flight_acquired = False
+ delivery_attempt_started = False
+ dispatch_identity = None
+ claim_token = ""
+ ledger = get_agent99_dispatch_ledger()
try:
payload = build_agent99_sre_alert(
alert_id=alert_id,
@@ -604,56 +870,348 @@ async def bridge_alertmanager_to_agent99(
routing = (
payload.get("routing") if isinstance(payload.get("routing"), dict) else {}
)
- single_flight_fingerprint = fingerprint or str(
- routing.get("correlationKey") or ""
+ suggested_mode = str(payload.get("suggestedMode") or "Status")
+ mutating = bool(payload.get("controlledApply") is True)
+ resolved_route_id = (
+ str(route_id or "").strip()
+ or f"agent99:{str(payload.get('kind') or 'monitoring_alert')}:{suggested_mode}"
)
- single_flight_acquired = await try_acquire_agent99_sre_single_flight(
- single_flight_fingerprint,
- alert_id,
+ durable_route = bool(
+ str(incident_id or "").strip()
+ or str(route_id or "").strip()
+ or str(work_item_id or "").strip()
)
+ if mutating and not durable_route:
+ logger.warning(
+ "agent99_mutating_dispatch_identity_missing_fail_closed",
+ alert_id=alert_id,
+ incident_id=incident_id,
+ reason="agent99_dispatch_incident_id_required",
+ )
+ return {
+ "status": "failed",
+ "reason": "agent99_dispatch_incident_id_required",
+ "dispatchPerformed": False,
+ "runtimeClosureVerified": False,
+ }
+ if durable_route:
+ try:
+ generation = max(1, int(str(execution_generation or "1")))
+ except ValueError:
+ generation = 1
+ generation = min(generation, 3)
+ base_payload = payload
+ while True:
+ try:
+ dispatch_identity = build_agent99_dispatch_identity(
+ project_id=project_id,
+ incident_id=incident_id,
+ source_fingerprint=fingerprint,
+ route_id=resolved_route_id,
+ execution_generation=str(generation),
+ approval_id=approval_id,
+ work_item_id=work_item_id,
+ )
+ except ValueError as exc:
+ logger.warning(
+ "agent99_durable_dispatch_identity_missing_fail_closed",
+ alert_id=alert_id,
+ incident_id=incident_id,
+ reason=str(exc),
+ )
+ return {
+ "status": "failed",
+ "reason": str(exc),
+ "dispatchPerformed": False,
+ "runtimeClosureVerified": False,
+ }
+ payload = attach_agent99_dispatch_identity(
+ base_payload,
+ dispatch_identity,
+ )
+ # Durable structured identity makes the inbox actionable.
+ # Mutating routes must not bypass routing checks; read-only
+ # BackupCheck retains controlledApply=false/no-write policy.
+ if mutating:
+ payload["force"] = False
+ reservation = await ledger.reserve(
+ identity=dispatch_identity,
+ payload=payload,
+ )
+ if reservation.get("claimed") is True:
+ claim_token = str(reservation.get("claim_token") or "")
+ if not claim_token:
+ return {
+ "status": "failed",
+ "reason": "agent99_dispatch_claim_token_missing",
+ "dispatchPerformed": False,
+ "identity": dispatch_identity.public_dict(),
+ "runtimeClosureVerified": False,
+ }
+ break
+
+ existing_receipt = reservation.get("receipt")
+ retry_policy = (
+ existing_receipt.get("retry_policy")
+ if isinstance(existing_receipt, dict)
+ and isinstance(existing_receipt.get("retry_policy"), dict)
+ else {}
+ )
+ max_generation = min(
+ 3,
+ int(retry_policy.get("max_generation") or 3),
+ )
+ if reservation.get("status") == "retry_not_before":
+ return {
+ "status": "retryable",
+ "reason": "durable_retry_not_before",
+ "dispatchPerformed": False,
+ "nextAttemptAt": reservation.get("next_attempt_at"),
+ "correlatedReceipt": existing_receipt,
+ "identity": dispatch_identity.public_dict(),
+ "runtimeClosureVerified": False,
+ }
+ if (
+ retry_policy.get("safe_to_retry") is True
+ and generation < max_generation
+ ):
+ generation += 1
+ continue
+
+ dispatch_receipt = (
+ existing_receipt.get("dispatch_receipt")
+ if isinstance(existing_receipt, dict)
+ and isinstance(existing_receipt.get("dispatch_receipt"), dict)
+ else {}
+ )
+ accepted = bool(dispatch_receipt.get("accepted") is True)
+ inbox_triggered = bool(
+ dispatch_receipt.get("inbox_triggered") is True
+ )
+ dispatch_promoted = bool(accepted and inbox_triggered)
+ reconcile_only = bool(
+ retry_policy.get("reconcile_only") is True
+ or reservation.get("status") == "idempotent_running"
+ or (
+ isinstance(existing_receipt, dict)
+ and existing_receipt.get("status")
+ == "dispatch_delivery_unknown_reconcile_only"
+ )
+ )
+ return {
+ "status": (
+ "deduplicated"
+ if dispatch_promoted
+ else "delivery_unknown"
+ if reconcile_only
+ else "suppressed"
+ ),
+ "reason": str(
+ reservation.get("status") or "pg_idempotency_hit"
+ ),
+ "dispatchPerformed": False,
+ "dispatchReceipt": dispatch_receipt or None,
+ "correlatedReceipt": existing_receipt,
+ "identity": dispatch_identity.public_dict(),
+ "runtimeClosureVerified": False,
+ }
+ # Generation retries share one Redis flight key; otherwise a new
+ # generation could race an older still-live transport lock.
+ single_flight_identity = dispatch_identity.single_flight_key
+ single_flight_owner = claim_token
+ else:
+ single_flight_identity = fingerprint or str(
+ routing.get("correlationKey") or ""
+ )
+
+ lock_decision = await acquire_agent99_sre_single_flight(
+ single_flight_identity,
+ single_flight_owner,
+ fail_closed=durable_route,
+ )
+ single_flight_acquired = bool(lock_decision.get("acquired") is True)
if not single_flight_acquired:
logger.info(
"agent99_sre_dispatch_single_flight_suppressed",
alert_id=alert_id,
alertname=alertname,
- fingerprint=single_flight_fingerprint,
+ fingerprint=single_flight_identity,
group_key=routing.get("groupKey"),
+ reason=lock_decision.get("reason"),
)
+ correlated_receipt = None
+ if dispatch_identity is not None:
+ correlated_receipt = await ledger.defer_claim_retryable(
+ identity=dispatch_identity,
+ claim_token=claim_token,
+ reason=str(
+ lock_decision.get("reason") or "single_flight_duplicate"
+ ),
+ retry_after_seconds=AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
+ )
return {
- "status": "suppressed",
- "reason": "single_flight_duplicate",
- "fingerprint": single_flight_fingerprint,
+ "status": "retryable" if dispatch_identity is not None else "suppressed",
+ "reason": str(lock_decision.get("reason") or "single_flight_duplicate"),
+ "fingerprint": single_flight_identity,
"groupKey": routing.get("groupKey"),
+ "dispatchPerformed": False,
+ "correlatedReceipt": correlated_receipt,
+ "identity": (
+ dispatch_identity.public_dict()
+ if dispatch_identity is not None
+ else None
+ ),
+ "runtimeClosureVerified": False,
}
+ if dispatch_identity is not None:
+ delivery_fence = await ledger.mark_delivery_attempt_started(
+ identity=dispatch_identity,
+ claim_token=claim_token,
+ )
+ if delivery_fence.get("receipt_persisted") is not True:
+ await release_agent99_sre_single_flight(
+ single_flight_identity,
+ single_flight_owner,
+ )
+ single_flight_acquired = False
+ return {
+ "status": "failed",
+ "reason": "delivery_attempt_fence_persistence_failed",
+ "dispatchPerformed": False,
+ "correlatedReceipt": delivery_fence,
+ "identity": dispatch_identity.public_dict(),
+ "runtimeClosureVerified": False,
+ }
+ delivery_attempt_started = True
dispatch_receipt = await asyncio.to_thread(
dispatch_agent99_sre_alert_with_receipt, payload
)
- if not dispatch_receipt.get("accepted"):
- await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id)
+ correlated_receipt = None
+ if dispatch_identity is not None:
+ correlated_receipt = await ledger.complete(
+ identity=dispatch_identity,
+ claim_token=claim_token,
+ dispatch_receipt=dispatch_receipt,
+ controlled_apply_authorized=mutating,
+ )
+ accepted = bool(dispatch_receipt.get("accepted") is True)
+ inbox_triggered = bool(dispatch_receipt.get("inbox_triggered") is True)
+ dispatch_promoted = bool(accepted and inbox_triggered)
+ delivery_certainty = str(
+ dispatch_receipt.get("delivery_certainty") or "unknown"
+ )
+ if not accepted and delivery_certainty == "not_delivered":
+ await release_agent99_sre_single_flight(
+ single_flight_identity,
+ single_flight_owner,
+ )
single_flight_acquired = False
return {
"status": "failed",
"reason": dispatch_receipt.get("status") or "dispatch_not_accepted",
"dispatchReceipt": dispatch_receipt,
- "fingerprint": single_flight_fingerprint,
+ "correlatedReceipt": correlated_receipt,
+ "fingerprint": single_flight_identity,
"groupKey": routing.get("groupKey"),
+ "dispatchPerformed": True,
+ "identity": (
+ dispatch_identity.public_dict()
+ if dispatch_identity is not None
+ else None
+ ),
+ "runtimeClosureVerified": False,
+ }
+ if (
+ dispatch_identity is not None
+ and not isinstance(correlated_receipt, dict)
+ ) or (
+ isinstance(correlated_receipt, dict)
+ and correlated_receipt.get("receipt_persisted") is not True
+ ):
+ return {
+ "status": "failed",
+ "reason": "dispatch_receipt_persistence_failed",
+ "dispatchReceipt": dispatch_receipt,
+ "correlatedReceipt": correlated_receipt,
+ "fingerprint": single_flight_identity,
+ "groupKey": routing.get("groupKey"),
+ "dispatchPerformed": True,
+ "identity": dispatch_identity.public_dict(),
+ "runtimeClosureVerified": False,
+ }
+ if not dispatch_promoted:
+ return {
+ "status": "delivery_pending",
+ "reason": "accepted_without_inbox_triggered",
+ "dispatch": dispatch_receipt.get("transport"),
+ "dispatchReceipt": dispatch_receipt,
+ "correlatedReceipt": correlated_receipt,
+ "fingerprint": single_flight_identity,
+ "groupKey": routing.get("groupKey"),
+ "dispatchPerformed": True,
+ "identity": (
+ dispatch_identity.public_dict()
+ if dispatch_identity is not None
+ else None
+ ),
+ "runtimeClosureVerified": False,
}
return {
"status": "dispatched",
"dispatch": dispatch_receipt.get("transport"),
"dispatchReceipt": dispatch_receipt,
- "fingerprint": single_flight_fingerprint,
+ "correlatedReceipt": correlated_receipt,
+ "fingerprint": single_flight_identity,
"groupKey": routing.get("groupKey"),
"kind": payload.get("kind"),
"suggestedMode": payload.get("suggestedMode"),
+ "dispatchPerformed": True,
+ "identity": (
+ dispatch_identity.public_dict()
+ if dispatch_identity is not None
+ else None
+ ),
+ "runtimeClosureVerified": False,
}
except Exception as exc:
- if single_flight_acquired:
- await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id)
+ if single_flight_acquired and not delivery_attempt_started:
+ await release_agent99_sre_single_flight(
+ single_flight_identity,
+ single_flight_owner,
+ )
+ correlated_receipt = None
+ if dispatch_identity is not None:
+ correlated_receipt = await ledger.complete(
+ identity=dispatch_identity,
+ claim_token=claim_token,
+ dispatch_receipt={
+ "schema_version": "agent99_sre_dispatch_receipt_v1",
+ "status": "transport_exception",
+ "transport": "unknown",
+ "alert_id": str(dispatch_identity.run_id),
+ "kind": "host_recovery",
+ "accepted": False,
+ "inbox_triggered": False,
+ "delivery_certainty": "unknown",
+ "stores_raw_response": False,
+ },
+ controlled_apply_authorized=mutating,
+ )
logger.warning(
- "agent99_sre_bridge_failed_fail_open",
+ "agent99_sre_bridge_failed_fail_closed",
alert_id=alert_id,
alertname=alertname,
error=str(exc),
)
- return {"status": "failed", "reason": type(exc).__name__}
+ return {
+ "status": "failed",
+ "reason": type(exc).__name__,
+ "dispatchPerformed": False,
+ "correlatedReceipt": correlated_receipt,
+ "identity": (
+ dispatch_identity.public_dict()
+ if dispatch_identity is not None
+ else None
+ ),
+ "runtimeClosureVerified": False,
+ }
diff --git a/apps/api/src/services/alert_approval_guard.py b/apps/api/src/services/alert_approval_guard.py
index e96d8a967..c21ee226c 100644
--- a/apps/api/src/services/alert_approval_guard.py
+++ b/apps/api/src/services/alert_approval_guard.py
@@ -13,6 +13,9 @@ from dataclasses import dataclass, field
import structlog
from src.services.action_parser import ActionKind, parse_kubectl_action
+from src.services.controlled_alert_target_router import (
+ build_controlled_recovery_promotion_contract,
+)
logger = structlog.get_logger(__name__)
@@ -35,6 +38,7 @@ async def guard_alert_approval_action(
alert_namespace: str | None,
alertname: str,
alert_category: str,
+ target_resource: str | None = None,
) -> ApprovalActionGuardResult:
"""Validate an AI/rule action before it is persisted as an approval.
@@ -55,11 +59,18 @@ async def guard_alert_approval_action(
requested_namespace = parsed.namespace
expected_namespace = (alert_namespace or "awoooi-prod").strip() or "awoooi-prod"
if requested_namespace and requested_namespace not in _ALLOWED_K8S_NAMESPACES:
+ controlled_reroute = _controlled_recovery_reroute(
+ alertname=alertname,
+ target_resource=target_resource or "",
+ namespace=expected_namespace,
+ )
return _blocked(
raw_action,
f"namespace_not_allowed:{requested_namespace}",
alertname,
expected_namespace=expected_namespace,
+ replacement_action=(controlled_reroute or {}).get("replacement_action"),
+ extra_metadata=(controlled_reroute or {}).get("metadata"),
)
if (
@@ -68,11 +79,33 @@ async def guard_alert_approval_action(
and requested_namespace != expected_namespace
and requested_namespace != "observability"
):
+ controlled_reroute = _controlled_recovery_reroute(
+ alertname=alertname,
+ target_resource=target_resource or "",
+ namespace=expected_namespace,
+ )
return _blocked(
raw_action,
f"namespace_mismatch:{requested_namespace}!={expected_namespace}",
alertname,
expected_namespace=expected_namespace,
+ replacement_action=(controlled_reroute or {}).get("replacement_action"),
+ extra_metadata=(controlled_reroute or {}).get("metadata"),
+ )
+
+ controlled_reroute = _controlled_recovery_reroute(
+ alertname=alertname,
+ target_resource=target_resource or "",
+ namespace=expected_namespace,
+ )
+ if controlled_reroute is not None:
+ return _blocked(
+ raw_action,
+ "kubernetes_action_not_applicable:control_plane_recovery",
+ alertname,
+ expected_namespace=expected_namespace,
+ replacement_action=controlled_reroute.get("replacement_action"),
+ extra_metadata=controlled_reroute.get("metadata"),
)
# Read-only commands are safe enough to display once the namespace is sane.
@@ -121,6 +154,47 @@ async def guard_alert_approval_action(
return ApprovalActionGuardResult(action=action)
+def _controlled_recovery_reroute(
+ *,
+ alertname: str,
+ target_resource: str,
+ namespace: str,
+) -> dict[str, object] | None:
+ """Replace a blocked fake K8s action with a fail-closed recovery candidate."""
+
+ contract = build_controlled_recovery_promotion_contract(
+ alertname=alertname,
+ target_resource=target_resource,
+ namespace=namespace,
+ )
+ if contract is None:
+ return None
+ route_id = str(contract.get("route_id") or "agent99_recover_after_owner_review")
+ ready = int(contract.get("ready_count") or 0)
+ total = int(contract.get("total_count") or 0)
+ blocked = int(contract.get("blocked_count") or 0)
+ summary = (
+ f"route={route_id}; promotion={ready}/{total}; blocked={blocked}; "
+ f"status={contract.get('status')}; runtime=false"
+ )
+ return {
+ "replacement_action": (
+ "DRAFT_READY - CONTROLLED_AGENT99_RECOVERY: "
+ "agent99-mode Recover controlledApply=true"
+ ),
+ "metadata": {
+ "controlled_target_reroute": True,
+ "controlled_target_reroute_reason": "kubernetes_action_invalid_for_control_plane",
+ "controlled_target_executor": "Agent99",
+ "controlled_target_route_id": route_id,
+ "repair_candidate_promotion_contract": contract,
+ "repair_candidate_promotion_summary": summary,
+ "runtime_execution_authorized": False,
+ "runtime_write_allowed": False,
+ },
+ }
+
+
def _blocked(
raw_action: str,
reason: str,
@@ -128,6 +202,8 @@ def _blocked(
*,
expected_namespace: str | None = None,
candidates: list[str] | None = None,
+ replacement_action: object | None = None,
+ extra_metadata: object | None = None,
) -> ApprovalActionGuardResult:
logger.warning(
"approval_action_blocked_before_persist",
@@ -137,15 +213,22 @@ def _blocked(
expected_namespace=expected_namespace,
candidates=candidates or [],
)
+ metadata = {
+ "action_guard": "blocked_before_persist",
+ "blocked_action": raw_action[:300],
+ "blocked_reason": reason,
+ "expected_namespace": expected_namespace,
+ "candidates": candidates or [],
+ }
+ if isinstance(extra_metadata, dict):
+ metadata.update(extra_metadata)
return ApprovalActionGuardResult(
- action=f"NO_ACTION - INVALID_TARGET: {reason}; original={raw_action[:180]}",
+ action=(
+ str(replacement_action)
+ if replacement_action
+ else f"NO_ACTION - INVALID_TARGET: {reason}; original={raw_action[:180]}"
+ ),
blocked=True,
reason=reason,
- metadata={
- "action_guard": "blocked_before_persist",
- "blocked_action": raw_action[:300],
- "blocked_reason": reason,
- "expected_namespace": expected_namespace,
- "candidates": candidates or [],
- },
+ metadata=metadata,
)
diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py
index 4c68a9aa4..308dda8b4 100644
--- a/apps/api/src/services/awooop_ansible_audit_service.py
+++ b/apps/api/src/services/awooop_ansible_audit_service.py
@@ -17,6 +17,9 @@ import structlog
from sqlalchemy import text
from src.db.base import get_db_context
+from src.services.controlled_alert_target_router import (
+ resolve_controlled_alert_target,
+)
logger = structlog.get_logger(__name__)
@@ -633,7 +636,63 @@ def _source_haystack(incident: dict[str, Any] | None, drift: dict[str, Any] | No
return " ".join(pieces)
+def _domain_controlled_executor_route(
+ incident: dict[str, Any] | None,
+) -> dict[str, Any] | None:
+ """Keep host control-plane alerts out of the generic Ansible catalog."""
+
+ if not isinstance(incident, dict):
+ return None
+ signals = [
+ signal
+ for signal in incident.get("signals") or []
+ if isinstance(signal, dict)
+ ]
+ first_signal = signals[0] if signals else {}
+ labels = (
+ first_signal.get("labels")
+ if isinstance(first_signal.get("labels"), dict)
+ else {}
+ )
+ alertname = str(
+ incident.get("alertname")
+ or first_signal.get("alert_name")
+ or labels.get("alertname")
+ or ""
+ )
+ affected_services = [
+ str(value)
+ for value in incident.get("affected_services") or []
+ if str(value).strip()
+ ]
+ target_resource = str(
+ affected_services[0]
+ if affected_services
+ else labels.get("component")
+ or labels.get("target_resource")
+ or ""
+ )
+ namespace = str(labels.get("namespace") or "")
+ return resolve_controlled_alert_target(
+ alertname=alertname,
+ target_resource=target_resource,
+ namespace=namespace,
+ )
+
+
def _catalog_hints(incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> dict[str, Any]:
+ controlled_route = _domain_controlled_executor_route(incident)
+ if controlled_route is not None:
+ return {
+ "match_mode": "domain_controlled_executor_route_v1",
+ "decision_effect": "delegate_to_domain_executor",
+ "available_count": len(_CATALOG),
+ "candidates": [],
+ "unmatched_catalog_ids": [str(item["catalog_id"]) for item in _CATALOG],
+ "controlled_executor": controlled_route,
+ "not_used_reason": "non_kubernetes_control_plane_route",
+ }
+
haystack = _source_haystack(incident, drift)
candidates: list[dict[str, Any]] = []
unmatched: list[str] = []
@@ -691,6 +750,7 @@ def build_ansible_truth(
records = [_ansible_record(row) for row in automation_ops if _is_ansible_operation(row)]
summary = summarize_ansible_execution(records)
+ candidate_catalog = _catalog_hints(incident, drift)
return {
"considered": bool(records),
"records": records,
@@ -715,11 +775,12 @@ def build_ansible_truth(
],
"default_execution_mode": "catalog/dry-run audit only until approval execution is explicitly wired",
},
- "candidate_catalog": _catalog_hints(incident, drift),
+ "candidate_catalog": candidate_catalog,
"not_used_reason": (
None
if records
- else "no automation_operation_log row with Ansible operation type, tag, or executor backend for this source"
+ else candidate_catalog.get("not_used_reason")
+ or "no automation_operation_log row with Ansible operation type, tag, or executor backend for this source"
),
}
diff --git a/apps/api/src/services/backup_restore_alertmanager_ingress.py b/apps/api/src/services/backup_restore_alertmanager_ingress.py
new file mode 100644
index 000000000..9a657838d
--- /dev/null
+++ b/apps/api/src/services/backup_restore_alertmanager_ingress.py
@@ -0,0 +1,498 @@
+"""Single-owner Alertmanager ingress for read-only BackupCheck automation.
+
+The raw Alertmanager signal owns dispatch. Telegram is only a projection of
+the durable Agent99 receipt. This lane may collect backup/restore evidence,
+but it must never run backup/restore, delete remote data, change retention,
+write escrow markers, or read secrets.
+"""
+
+from __future__ import annotations
+
+import hashlib
+from typing import Any
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+from src.core.logging import get_logger
+from src.models.approval import (
+ ApprovalRequestCreate,
+ BlastRadius,
+ DataImpact,
+ DryRunCheck,
+ RiskLevel,
+)
+from src.services.agent99_controlled_dispatch_ledger import (
+ read_agent99_dispatch_receipt,
+)
+from src.services.agent99_sre_bridge import (
+ bridge_alertmanager_to_agent99,
+ resolve_agent99_durable_route,
+)
+from src.services.approval_db import get_approval_service
+from src.services.channel_hub import record_alertmanager_event
+from src.services.incident_service import (
+ create_incident_for_approval,
+ get_incident_service,
+)
+
+logger = get_logger("awoooi.backup_restore_alertmanager_ingress")
+
+BACKUP_RESTORE_AGENT99_ROUTE_ID = "agent99:backup_health:BackupCheck"
+BACKUP_RESTORE_DISPATCH_OWNER = "alertmanager_raw_signal"
+BACKUP_RESTORE_OWNER_SCHEMA_VERSION = "backup_restore_ingress_owner_v1"
+BACKUP_RESTORE_PROHIBITED_ACTIONS = (
+ "run_backup",
+ "run_restore",
+ "remote_delete",
+ "prune",
+ "change_retention",
+ "write_escrow_marker",
+ "read_secret",
+)
+
+
+def _stable_digest(*parts: object) -> str:
+ value = "\x1f".join(str(part or "").strip() for part in parts)
+ return hashlib.sha256(value.encode("utf-8")).hexdigest()
+
+
+def build_backup_restore_ingress_owner(
+ *,
+ project_id: str,
+ source_fingerprint: str,
+ source_event_fingerprint: str = "",
+ source_started_at: str = "",
+) -> dict[str, Any]:
+ """Return the deterministic durable owner identity for one firing episode."""
+
+ project = str(project_id or "awoooi").strip() or "awoooi"
+ fingerprint = str(source_fingerprint or "").strip()
+ if not fingerprint:
+ raise ValueError("backup_restore_source_fingerprint_required")
+ source_occurrence = "|".join(
+ value
+ for value in (
+ str(source_event_fingerprint or "").strip(),
+ str(source_started_at or "").strip(),
+ )
+ if value
+ ) or fingerprint
+ owner_key = (
+ f"{BACKUP_RESTORE_OWNER_SCHEMA_VERSION}:{project}:"
+ f"{fingerprint}:{source_occurrence}"
+ )
+ digest = _stable_digest(owner_key)
+ approval_id = uuid5(NAMESPACE_URL, owner_key)
+ return {
+ "schema_version": BACKUP_RESTORE_OWNER_SCHEMA_VERSION,
+ "project_id": project,
+ "source_fingerprint": fingerprint,
+ "source_occurrence": source_occurrence,
+ "source_occurrence_hash": digest,
+ "approval_id": str(approval_id),
+ "incident_id": f"INC-BRR-{digest[:16].upper()}",
+ "work_item_id": f"backupcheck:{project}:{digest[:32]}",
+ "route_id": BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ "kind": "backup_health",
+ "mode": "BackupCheck",
+ "single_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
+ "telegram_role": "receipt_projection_only",
+ "controlled_apply": False,
+ "read_only": True,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ }
+
+
+def _backup_restore_approval_request(
+ *,
+ owner: dict[str, Any],
+ target_resource: str,
+ message: str,
+ incident_id: str = "",
+) -> ApprovalRequestCreate:
+ return ApprovalRequestCreate(
+ action=(
+ "READ_ONLY_BACKUPCHECK - collect backup status, freshness, offsite, "
+ "escrow count, restore-drill and source-resolution receipts"
+ ),
+ description=(
+ "Alertmanager raw backup/restore signal selected the non-mutating "
+ f"BackupCheck evidence lane. {str(message or '')[:500]}"
+ ),
+ risk_level=RiskLevel.LOW,
+ blast_radius=BlastRadius(
+ affected_pods=0,
+ estimated_downtime="0",
+ related_services=[target_resource] if target_resource else [],
+ data_impact=DataImpact.READ_ONLY,
+ ),
+ dry_run_checks=[
+ DryRunCheck(
+ name="single dispatch owner",
+ passed=True,
+ message=BACKUP_RESTORE_DISPATCH_OWNER,
+ ),
+ DryRunCheck(
+ name="non-mutating BackupCheck boundary",
+ passed=True,
+ message="backup/restore/delete/retention/escrow writes prohibited",
+ ),
+ ],
+ requested_by="Alertmanager BackupCheck durable owner",
+ incident_id=incident_id or None,
+ metadata={
+ **owner,
+ "preallocated_approval_id": owner["approval_id"],
+ "owner_policy": "global_product_governance_v2",
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ },
+ )
+
+
+def _value(value: Any) -> str:
+ return str(getattr(value, "value", value) or "").strip()
+
+
+def _incident_risk_from_source_severity(severity: str) -> str:
+ normalized = str(severity or "warning").strip().lower()
+ if normalized in {"critical", "error"}:
+ return "high"
+ if normalized == "warning":
+ return "medium"
+ return "low"
+
+
+def _dispatch_receipt_from_readback(
+ readback: dict[str, Any] | None,
+) -> dict[str, Any]:
+ if not isinstance(readback, dict):
+ return {}
+ receipt = readback.get("dispatch_receipt")
+ return dict(receipt) if isinstance(receipt, dict) else {}
+
+
+def _identity_from(value: dict[str, Any] | None) -> dict[str, Any]:
+ if not isinstance(value, dict):
+ return {}
+ identity = value.get("identity")
+ return dict(identity) if isinstance(identity, dict) else {}
+
+
+async def process_backup_restore_alertmanager_signal(
+ *,
+ project_id: str,
+ alert_id: str,
+ alertname: str,
+ severity: str,
+ namespace: str,
+ target_resource: str,
+ message: str,
+ labels: dict[str, Any] | None,
+ annotations: dict[str, Any] | None,
+ source_fingerprint: str,
+ source_event_fingerprint: str = "",
+ source_started_at: str = "",
+ alert_category: str = "backup",
+ notification_type: str = "TYPE-1",
+ source_url: str | None = None,
+) -> dict[str, Any]:
+ """Create/bind a canonical Incident, dispatch once, and read the receipt.
+
+ The function fails closed before transport whenever owner persistence or
+ canonical Incident readback is missing. A repeated invocation reuses the
+ same approval, Incident, work item, and PostgreSQL dispatch identity.
+ """
+
+ safe_labels = dict(labels or {})
+ safe_annotations = dict(annotations or {})
+ route = resolve_agent99_durable_route(
+ alertname=alertname,
+ severity=severity,
+ target_resource=target_resource,
+ namespace=namespace,
+ message=message,
+ labels=safe_labels,
+ annotations=safe_annotations,
+ )
+ if not isinstance(route, dict) or (
+ route.get("kind") != "backup_health"
+ or route.get("suggested_mode") != "BackupCheck"
+ or route.get("route_id") != BACKUP_RESTORE_AGENT99_ROUTE_ID
+ ):
+ return {
+ "schema_version": "backup_restore_ingress_result_v1",
+ "status": "not_backup_restore_route_no_dispatch",
+ "dispatch_performed": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ }
+
+ owner = build_backup_restore_ingress_owner(
+ project_id=project_id,
+ source_fingerprint=source_fingerprint,
+ source_event_fingerprint=source_event_fingerprint,
+ source_started_at=source_started_at,
+ )
+ approval_service = get_approval_service()
+ owner_approval_id = UUID(str(owner["approval_id"]))
+ approval = await approval_service.get_approval(owner_approval_id)
+ created_owner = approval is None
+ legacy_incident_id = ""
+ if approval is None:
+ recent = await approval_service.find_by_fingerprint(
+ fingerprint=source_fingerprint,
+ debounce_minutes=30,
+ )
+ legacy_incident_id = _value(getattr(recent, "incident_id", ""))
+ request = _backup_restore_approval_request(
+ owner=owner,
+ target_resource=target_resource,
+ message=message,
+ incident_id=legacy_incident_id,
+ )
+ try:
+ approval = await approval_service.create_approval_with_fingerprint(
+ request=request,
+ fingerprint=source_fingerprint,
+ )
+ except Exception:
+ # A concurrent producer may have inserted the deterministic owner.
+ # Read it back; never create another random owner or dispatch twice.
+ approval = await approval_service.get_approval(owner_approval_id)
+ if approval is None:
+ raise
+ else:
+ incremented = await approval_service.increment_hit_count(owner_approval_id)
+ if incremented is not None:
+ approval = incremented
+
+ if _value(getattr(approval, "id", "")) != str(owner_approval_id):
+ return {
+ "schema_version": "backup_restore_ingress_result_v1",
+ "status": "durable_owner_identity_mismatch_no_dispatch",
+ "owner": owner,
+ "dispatch_performed": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ }
+
+ incident_id = _value(getattr(approval, "incident_id", ""))
+ if not incident_id:
+ incident_id = str(owner["incident_id"])
+ incident_id = await create_incident_for_approval(
+ approval_id=str(owner_approval_id),
+ risk_level=_incident_risk_from_source_severity(severity),
+ target_resource=target_resource,
+ namespace=namespace,
+ alert_type="custom",
+ message=message,
+ source="alertmanager",
+ alertname=alertname,
+ alert_labels={
+ **safe_labels,
+ "fingerprint": source_fingerprint,
+ "alert_id": alert_id,
+ "backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
+ },
+ notification_type=notification_type,
+ alert_category=alert_category,
+ canonical_incident_id=incident_id,
+ )
+ await approval_service.update_incident_id(owner_approval_id, incident_id)
+
+ bound_approval = await approval_service.get_approval(owner_approval_id)
+ if (
+ bound_approval is None
+ or _value(getattr(bound_approval, "incident_id", "")) != incident_id
+ ):
+ return {
+ "schema_version": "backup_restore_ingress_result_v1",
+ "status": "canonical_incident_binding_missing_no_dispatch",
+ "owner": owner,
+ "incident_id": incident_id or None,
+ "dispatch_performed": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "safe_next_action": (
+ "persist_owner_to_canonical_incident_binding_then_retry"
+ ),
+ }
+ approval = bound_approval
+
+ canonical_incident = await get_incident_service().get_for_readback(
+ incident_id,
+ project_id=project_id,
+ )
+ canonical_status = _value(getattr(canonical_incident, "status", ""))
+ canonical_durable = bool(
+ canonical_incident is not None
+ and getattr(canonical_incident, "persisted_to_pg", False) is True
+ )
+ if not canonical_durable or canonical_status not in {
+ "investigating",
+ "mitigating",
+ }:
+ return {
+ "schema_version": "backup_restore_ingress_result_v1",
+ "status": "canonical_active_incident_missing_no_dispatch",
+ "owner": owner,
+ "incident_id": incident_id or None,
+ "canonical_incident_status": canonical_status or "missing",
+ "canonical_incident_durable": canonical_durable,
+ "dispatch_performed": False,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "safe_next_action": (
+ "repair_or_create_active_canonical_incident_then_retry_same_source_owner"
+ ),
+ }
+
+ await record_alertmanager_event(
+ project_id=project_id,
+ alert_id=alert_id,
+ alertname=alertname,
+ severity=severity,
+ namespace=namespace,
+ target_resource=target_resource,
+ fingerprint=source_fingerprint,
+ stage="backup_restore_incident_owner_bound",
+ notification_type=notification_type,
+ alert_category=alert_category,
+ incident_id=incident_id,
+ approval_id=str(owner_approval_id),
+ repeat_count=int(getattr(approval, "hit_count", 1) or 1),
+ source_url=source_url,
+ labels={
+ **safe_labels,
+ "backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
+ },
+ annotations=safe_annotations,
+ )
+
+ dispatch = await bridge_alertmanager_to_agent99(
+ alert_id=alert_id,
+ alertname=alertname,
+ severity=severity,
+ namespace=namespace,
+ target_resource=target_resource,
+ message=message,
+ labels=safe_labels,
+ annotations=safe_annotations,
+ fingerprint=source_fingerprint,
+ alert_category=alert_category,
+ notification_type=notification_type,
+ source_url=source_url,
+ project_id=project_id,
+ incident_id=incident_id,
+ approval_id=str(owner_approval_id),
+ route_id=BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ work_item_id=str(owner["work_item_id"]),
+ )
+ readback = await read_agent99_dispatch_receipt(
+ project_id=project_id,
+ incident_id=incident_id,
+ )
+ dispatch_identity = _identity_from(dispatch)
+ readback_identity = _identity_from(readback)
+ readback_receipt = _dispatch_receipt_from_readback(readback)
+ identity_matches = bool(
+ dispatch_identity
+ and readback_identity
+ and all(
+ _value(dispatch_identity.get(field))
+ == _value(readback_identity.get(field))
+ for field in ("incident_id", "run_id", "trace_id", "work_item_id")
+ )
+ )
+ accepted = bool(
+ readback_receipt.get("accepted") is True
+ and readback_receipt.get("inbox_triggered") is True
+ )
+ receipt_persisted = bool(readback and identity_matches)
+ verifier = (
+ dict(readback.get("verifier"))
+ if isinstance(readback, dict)
+ and isinstance(readback.get("verifier"), dict)
+ else {}
+ )
+ verifier_status = _value(verifier.get("status")) or "not_started"
+ durable = bool(accepted and receipt_persisted)
+ result_status = (
+ "backupcheck_dispatched_verifier_pending"
+ if durable and dispatch.get("dispatchPerformed") is True
+ else "backupcheck_deduplicated_verifier_pending"
+ if durable and dispatch.get("status") == "deduplicated"
+ else "backupcheck_dispatch_or_readback_failed"
+ )
+
+ await record_alertmanager_event(
+ project_id=project_id,
+ alert_id=alert_id,
+ alertname=alertname,
+ severity=severity,
+ namespace=namespace,
+ target_resource=target_resource,
+ fingerprint=source_fingerprint,
+ stage=(
+ "backup_restore_dispatch_receipt_persisted"
+ if durable
+ else "backup_restore_dispatch_receipt_missing"
+ ),
+ notification_type=notification_type,
+ alert_category=alert_category,
+ incident_id=incident_id,
+ approval_id=str(owner_approval_id),
+ repeat_count=int(getattr(approval, "hit_count", 1) or 1),
+ source_url=source_url,
+ labels={
+ **safe_labels,
+ "backup_restore_dispatch_owner": BACKUP_RESTORE_DISPATCH_OWNER,
+ },
+ annotations=safe_annotations,
+ )
+ logger.info(
+ "backup_restore_alertmanager_dispatch_result",
+ status=result_status,
+ incident_id=incident_id,
+ approval_id=str(owner_approval_id),
+ automation_run_id=readback_identity.get("run_id"),
+ dispatch_performed=bool(dispatch.get("dispatchPerformed") is True),
+ receipt_persisted=receipt_persisted,
+ accepted=accepted,
+ verifier_status=verifier_status,
+ runtime_execution_authorized=False,
+ runtime_closure_verified=False,
+ )
+ return {
+ "schema_version": "backup_restore_ingress_result_v1",
+ "status": result_status,
+ "owner": owner,
+ "owner_created": created_owner,
+ "approval_id": str(owner_approval_id),
+ "incident_id": incident_id,
+ "canonical_incident_status": canonical_status,
+ "canonical_incident_durable": canonical_durable,
+ "dispatch_performed": bool(dispatch.get("dispatchPerformed") is True),
+ "dispatch_status": _value(dispatch.get("status")),
+ "receipt_persisted": receipt_persisted,
+ "accepted": accepted,
+ "identity": readback_identity or dispatch_identity,
+ "verifier": {
+ "status": verifier_status,
+ "terminal": verifier_status in {"success", "failed"},
+ "receipt": verifier,
+ "outcome_source": "agent99_outcome_contract_v1",
+ "safe_next_action": (
+ "ingest_same_identity_agent99_outcome_then_run_independent_dr_verifier"
+ if verifier_status in {"pending", "pending_dispatch", "not_started"}
+ else "complete_learning_writeback_only_after_verifier_success"
+ ),
+ },
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": bool(
+ isinstance(readback, dict)
+ and readback.get("runtime_closure_verified") is True
+ ),
+ "production_write_performed": False,
+ "prohibited_actions": list(BACKUP_RESTORE_PROHIBITED_ACTIONS),
+ }
diff --git a/apps/api/src/services/backup_restore_signal_automation.py b/apps/api/src/services/backup_restore_signal_automation.py
new file mode 100644
index 000000000..bcd0523f9
--- /dev/null
+++ b/apps/api/src/services/backup_restore_signal_automation.py
@@ -0,0 +1,1020 @@
+"""Fail-closed automation contract for backup / restore / escrow signals.
+
+The Telegram card is only a sensor receipt. This module turns that receipt
+into deterministic work-item, candidate, durable-receipt and verifier
+identities without running backup, restore, prune, remote delete, retention
+changes, or escrow writes.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import html
+import re
+from collections.abc import Mapping, Sequence
+from typing import Any
+
+BACKUP_RESTORE_EVENT_TYPE = "backup_restore_escrow_signal"
+BACKUP_RESTORE_LANE = "backup_restore_escrow_triage"
+BACKUP_RESTORE_POLICY_VERSION = "backup_restore_readback_policy_v2"
+BACKUP_RESTORE_POLICY_EFFECTIVE_AT = "2026-07-11T19:30:58+08:00"
+BACKUP_RESTORE_POLICY_SOURCE_REF = (
+ "apps/api/src/services/backup_restore_signal_automation.py:"
+ "backup_restore_readback_policy_v2"
+)
+_BACKUP_RESTORE_POLICY_CANONICAL = "|".join(
+ [
+ BACKUP_RESTORE_POLICY_VERSION,
+ BACKUP_RESTORE_POLICY_EFFECTIVE_AT,
+ BACKUP_RESTORE_EVENT_TYPE,
+ BACKUP_RESTORE_LANE,
+ "controlled_playbook_queue=readback_only",
+ "runtime_write_gate=0_until_verifier",
+ "backup_status,freshness,offsite_verify,escrow,restore_drill,source_resolution_receipt",
+ "no_backup_restore_delete_retention_escrow_write_or_secret_read",
+ ]
+)
+BACKUP_RESTORE_POLICY_SOURCE_HASH = (
+ "sha256:"
+ + hashlib.sha256(_BACKUP_RESTORE_POLICY_CANONICAL.encode("utf-8")).hexdigest()
+)
+
+_TOP_EVIDENCE_RE = re.compile(
+ r"\s*Top evidence\s*(?P.*?)(?:\s*建議下一步\s*|$)",
+ re.IGNORECASE | re.DOTALL,
+)
+_TAG_RE = re.compile(r"<[^>]+>")
+_SPACE_RE = re.compile(r"\s+")
+_SAFE_ID_RE = re.compile(r"[^a-zA-Z0-9_.:-]+")
+
+_REQUIRED_EVIDENCE = (
+ "backup_status",
+ "freshness",
+ "offsite_verify",
+ "escrow",
+ "restore_drill",
+ "source_resolution_receipt",
+)
+_BACKUP_VERIFIER_EVIDENCE_REF_KEYS = {
+ "agent99_outcome_receipt_id",
+ "post_verifier_evidence_ref",
+ "source_event_evidence_ref",
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+}
+
+_FIELD_ALIASES = {
+ "backup_status": (
+ "backup_status",
+ "backup-status",
+ "backup status",
+ "backupaggregaterun",
+ "backup aggregate",
+ "last_backup_success",
+ "last backup success",
+ ),
+ "freshness": (
+ "freshness",
+ "backup_age",
+ "backup age",
+ "last_success_age",
+ "last success age",
+ ),
+ "offsite_verify": (
+ "offsite_verify",
+ "offsite verify",
+ "offsite_status",
+ "offsite status",
+ "rclone_check",
+ ),
+ "escrow": (
+ "escrow_missing",
+ "escrow missing",
+ "escrow_status",
+ "escrow status",
+ "credential_escrow",
+ ),
+ "restore_drill": (
+ "restore_drill",
+ "restore drill",
+ "restore_test",
+ "restore test",
+ "restore_status",
+ "restore status",
+ ),
+ "source_resolution_receipt": (
+ "source_resolution_receipt",
+ "source resolution receipt",
+ "alert_resolved_receipt",
+ "alert resolved receipt",
+ ),
+}
+
+_BAD_TOKENS = (
+ "failed",
+ "failure",
+ "missing",
+ "blocked",
+ "stale",
+ "expired",
+ "timeout",
+ "degraded",
+ "unhealthy",
+ "false",
+ "error",
+)
+_NEGATIVE_PHRASES = (
+ "not ok",
+ "not verified",
+ "not healthy",
+ "not fresh",
+ "not current",
+ "not complete",
+ "not resolved",
+ "not passed",
+ "never verified",
+ "unverified",
+)
+_GOOD_TOKENS = (
+ "success",
+ "succeeded",
+ "healthy",
+ "verified",
+ "passed",
+ "fresh",
+ "current",
+ "complete",
+ "resolved",
+ "true",
+ "ok",
+)
+
+_PROHIBITED_ACTIONS = (
+ "run_backup",
+ "run_restore",
+ "restore_over_production",
+ "remote_delete",
+ "retention_change",
+ "restic_prune",
+ "escrow_marker_write",
+ "read_backup_contents",
+ "read_secret_values",
+)
+
+
+def backup_restore_readback_policy() -> dict[str, Any]:
+ """Return the immutable source policy used for effective projections."""
+
+ return {
+ "schema_version": "backup_restore_readback_policy_v2",
+ "policy_version": BACKUP_RESTORE_POLICY_VERSION,
+ "effective_at": BACKUP_RESTORE_POLICY_EFFECTIVE_AT,
+ "source_ref": BACKUP_RESTORE_POLICY_SOURCE_REF,
+ "source_hash": BACKUP_RESTORE_POLICY_SOURCE_HASH,
+ "historical_projection_supported": True,
+ "send_time_contract_supported": True,
+ "runtime_write_requires_terminal_verifier": True,
+ "metadata_refs_are_learning_writeback": False,
+ "agent99_dispatch_is_runtime_closure": False,
+ }
+
+
+def _normalized_agent99_dispatch_receipt(
+ receipt: Mapping[str, Any] | None,
+) -> dict[str, Any]:
+ """Validate the persisted dispatch receipt without treating it as outcome."""
+
+ if not isinstance(receipt, Mapping) or not receipt:
+ return {
+ "schema_version": "backup_restore_agent99_dispatch_readback_v1",
+ "status": "dispatch_receipt_missing",
+ "receipt_present": False,
+ "dispatched": False,
+ "accepted": False,
+ "inbox_triggered": False,
+ "backupcheck_route": False,
+ "receipt_persisted": False,
+ "identity_complete": False,
+ "identity_binding_valid": False,
+ "agent99_run_id": None,
+ "trace_id": None,
+ "agent99_work_item_id": None,
+ "alert_id": None,
+ "runtime_closure_verified": False,
+ "post_verifier_passed": False,
+ "post_verifier_failed": False,
+ "backup_evidence_refs_complete": False,
+ "missing_backup_evidence_refs": sorted(
+ _BACKUP_VERIFIER_EVIDENCE_REF_KEYS
+ ),
+ "learning_writeback_complete": False,
+ "current_run_state": None,
+ "current_verifier_status": None,
+ "current_learning_writeback_status": None,
+ "next_action": "persist_agent99_dispatch_receipt_before_verifier",
+ }
+
+ source_schema = str(receipt.get("schema_version") or "")
+ accepted = receipt.get("accepted") is True
+ inbox_triggered = (
+ receipt.get("inbox_triggered") is True
+ or receipt.get("inboxTriggered") is True
+ )
+ kind = str(receipt.get("kind") or "")
+ suggested_mode = str(
+ receipt.get("suggested_mode") or receipt.get("suggestedMode") or ""
+ )
+ identity = receipt.get("identity")
+ if not isinstance(identity, Mapping):
+ identity = receipt.get("agent99_dispatch_identity")
+ if not isinstance(identity, Mapping):
+ identity = {}
+ agent99_run_id = str(
+ receipt.get("run_id") or identity.get("run_id") or ""
+ ).strip()
+ trace_id = str(receipt.get("trace_id") or identity.get("trace_id") or "").strip()
+ agent99_work_item_id = str(
+ receipt.get("work_item_id") or identity.get("work_item_id") or ""
+ ).strip()
+ legacy_schema = source_schema == "telegram_agent99_dispatch_receipt_v1"
+ projection_schema = (
+ source_schema == "telegram_agent99_dispatch_receipt_projection_v1"
+ )
+ valid_schema = legacy_schema or projection_schema
+ receipt_persisted = receipt.get("receipt_persisted") is True
+ identity_binding_valid = receipt.get("identity_binding_valid") is not False
+ identity_complete = bool(
+ agent99_run_id and trace_id and agent99_work_item_id
+ )
+ backupcheck_route = kind == "backup_health" and suggested_mode == "BackupCheck"
+ current_verifier = receipt.get("current_verifier")
+ if not isinstance(current_verifier, Mapping):
+ current_verifier = {}
+ current_learning = receipt.get("current_learning_writeback")
+ if not isinstance(current_learning, Mapping):
+ current_learning = {}
+
+ def same_stage_identity(stage: Mapping[str, Any]) -> bool:
+ return bool(
+ identity_complete
+ and str(stage.get("run_id") or "").strip() == agent99_run_id
+ and str(stage.get("trace_id") or "").strip() == trace_id
+ and str(stage.get("work_item_id") or "").strip()
+ == agent99_work_item_id
+ )
+
+ verifier_evidence_refs = current_verifier.get("evidence_refs")
+ if not isinstance(verifier_evidence_refs, Mapping):
+ verifier_evidence_refs = {}
+ missing_backup_evidence_refs = sorted(
+ key
+ for key in _BACKUP_VERIFIER_EVIDENCE_REF_KEYS
+ if not str(verifier_evidence_refs.get(key) or "").strip()
+ )
+ backup_evidence_refs_complete = not missing_backup_evidence_refs
+ post_verifier_passed = bool(
+ receipt.get("post_verifier_passed") is True
+ and current_verifier.get("schema_version")
+ == "agent99_independent_verifier_receipt_v1"
+ and current_verifier.get("status") == "success"
+ and current_verifier.get("verifier_passed") is True
+ and current_verifier.get("source_event_resolved") is True
+ and current_verifier.get("transport_ok") is True
+ and current_verifier.get("outcome_state") == "resolved"
+ and bool(str(current_verifier.get("verified_at") or "").strip())
+ and backup_evidence_refs_complete
+ and same_stage_identity(current_verifier)
+ )
+ post_verifier_failed = bool(
+ current_verifier.get("schema_version")
+ == "agent99_independent_verifier_receipt_v1"
+ and current_verifier.get("status") == "failed"
+ and same_stage_identity(current_verifier)
+ )
+ learning_refs = current_learning.get("receipt_refs")
+ if not isinstance(learning_refs, Mapping):
+ learning_refs = {}
+ learning_writeback_complete = bool(
+ current_learning.get("schema_version")
+ == "agent99_learning_writeback_receipt_v1"
+ and current_learning.get("status") == "success"
+ and same_stage_identity(current_learning)
+ and {
+ "incident_closure_receipt_id",
+ "telegram_lifecycle_receipt_id",
+ "km_writeback_ack_id",
+ "playbook_trust_writeback_ack_id",
+ "dr_scorecard_writeback_ack_id",
+ }.issubset(learning_refs)
+ )
+ runtime_closure_verified = bool(
+ receipt.get("runtime_closure_verified") is True
+ and receipt.get("closure_complete") is True
+ and str(receipt.get("current_run_state") or "") == "completed"
+ and post_verifier_passed
+ and learning_writeback_complete
+ )
+ dispatched = bool(
+ projection_schema
+ and receipt_persisted
+ and identity_complete
+ and identity_binding_valid
+ and accepted
+ and inbox_triggered
+ and backupcheck_route
+ )
+ if not valid_schema:
+ status = "dispatch_receipt_schema_invalid"
+ next_action = "repair_agent99_dispatch_receipt_schema"
+ elif legacy_schema:
+ # The old Telegram mirror dispatched independently and lacked the
+ # durable run/trace/work-item binding. Keep it as declared history,
+ # never as evidence that this candidate was dispatched.
+ status = "legacy_dispatch_receipt_unbound"
+ next_action = "project_durable_agent99_dispatch_receipt_with_identity"
+ elif not receipt_persisted:
+ status = "dispatch_receipt_not_persisted_fail_closed"
+ next_action = "persist_agent99_dispatch_receipt_before_verifier"
+ elif not identity_complete:
+ status = "dispatch_identity_incomplete_fail_closed"
+ next_action = "repair_agent99_run_trace_work_item_binding"
+ elif not identity_binding_valid:
+ status = "current_ledger_identity_mismatch_fail_closed"
+ next_action = "repair_agent99_run_trace_work_item_binding"
+ elif not accepted or not inbox_triggered:
+ status = "dispatch_not_accepted_or_inbox_not_triggered"
+ next_action = "retry_idempotent_agent99_backupcheck_dispatch"
+ elif not backupcheck_route:
+ status = "dispatch_route_mismatch_fail_closed"
+ next_action = "repair_backupcheck_route_before_dispatch_replay"
+ elif runtime_closure_verified:
+ status = "closed_verified_learning_written"
+ next_action = "monitor_backup_restore_recurrence"
+ elif post_verifier_passed:
+ status = "verifier_passed_learning_writeback_pending"
+ next_action = "complete_km_playbook_and_lifecycle_writeback"
+ elif post_verifier_failed:
+ status = "verifier_failed_no_runtime_closure"
+ next_action = "collect_failed_verifier_evidence_then_retry_bounded_readback"
+ else:
+ status = "accepted_inbox_triggered"
+ next_action = "ingest_agent99_backupcheck_outcome_and_independent_dr_verifier"
+
+ return {
+ "schema_version": "backup_restore_agent99_dispatch_readback_v1",
+ "source_schema_version": source_schema,
+ "status": status,
+ "receipt_present": True,
+ "dispatched": dispatched,
+ "accepted": accepted,
+ "inbox_triggered": inbox_triggered,
+ "backupcheck_route": backupcheck_route,
+ "receipt_persisted": receipt_persisted,
+ "identity_complete": identity_complete,
+ "identity_binding_valid": identity_binding_valid,
+ "kind": kind,
+ "suggested_mode": suggested_mode,
+ "transport": str(receipt.get("transport") or ""),
+ "alert_id": str(receipt.get("alert_id") or "") or None,
+ "agent99_run_id": agent99_run_id or None,
+ "trace_id": trace_id or None,
+ "agent99_work_item_id": agent99_work_item_id or None,
+ "stores_raw_response": False,
+ "runtime_closure_verified": bool(
+ runtime_closure_verified
+ ),
+ "post_verifier_passed": post_verifier_passed,
+ "post_verifier_failed": post_verifier_failed,
+ "backup_evidence_refs_complete": backup_evidence_refs_complete,
+ "missing_backup_evidence_refs": missing_backup_evidence_refs,
+ "learning_writeback_complete": learning_writeback_complete,
+ "current_run_state": str(receipt.get("current_run_state") or "") or None,
+ "current_verifier_status": str(current_verifier.get("status") or "") or None,
+ "current_learning_writeback_status": (
+ str(current_learning.get("status") or "") or None
+ ),
+ "next_action": next_action,
+ }
+
+
+def _safe_id(value: object, *, fallback: str) -> str:
+ cleaned = _SAFE_ID_RE.sub("-", str(value or "")).strip("-._:")
+ return (cleaned or fallback)[:96]
+
+
+def _stable_digest(*values: object) -> str:
+ material = "|".join(str(value or "").strip().lower() for value in values)
+ return hashlib.sha256(material.encode("utf-8")).hexdigest()[:16]
+
+
+def _evidence_lines(text: str) -> list[str]:
+ """Return only the normalized card's public-safe Top evidence section."""
+
+ match = _TOP_EVIDENCE_RE.search(text or "")
+ body = match.group("body") if match else text or ""
+ lines: list[str] = []
+ for raw_line in body.splitlines():
+ plain = html.unescape(_TAG_RE.sub(" ", raw_line))
+ plain = _SPACE_RE.sub(" ", plain).strip(" ├└\t")
+ if plain:
+ lines.append(plain[:240])
+ if len(lines) >= 8:
+ break
+ return lines
+
+
+def _numeric_assignment(text: str, aliases: Sequence[str]) -> float | None:
+ for alias in aliases:
+ match = re.search(
+ rf"{re.escape(alias)}\s*[:=]\s*(?P-?\d+(?:\.\d+)?)",
+ text,
+ re.IGNORECASE,
+ )
+ if match:
+ try:
+ return float(match.group("value"))
+ except ValueError:
+ return None
+ return None
+
+
+def _field_observation(field: str, lines: Sequence[str]) -> dict[str, Any]:
+ aliases = _FIELD_ALIASES[field]
+ matching = [line for line in lines if any(alias in line.lower() for alias in aliases)]
+ if not matching:
+ return {
+ "field": field,
+ "status": "missing",
+ "observed": False,
+ "safe_value": None,
+ }
+
+ joined = " ".join(matching).lower()
+ number = _numeric_assignment(joined, aliases)
+
+ # Explicit negation must win over both a numeric token and positive
+ # substrings (for example, "not verified" contains "verified").
+ if any(phrase in joined for phrase in _NEGATIVE_PHRASES):
+ status = "failed"
+ safe_value: object = "explicit_negative_result"
+ elif field == "escrow" and number is not None:
+ status = "healthy" if number == 0 else "failed"
+ safe_value = int(number) if number.is_integer() else number
+ elif field in {"backup_status", "offsite_verify", "restore_drill"} and number is not None:
+ status = "healthy" if number > 0 else "failed"
+ safe_value = int(number) if number.is_integer() else number
+ elif any(token in joined for token in _BAD_TOKENS):
+ status = "failed"
+ safe_value = "explicit_failure_or_missing"
+ elif any(token in joined for token in _GOOD_TOKENS):
+ status = "healthy"
+ safe_value = "explicit_success"
+ else:
+ status = "unverified"
+ safe_value = "observed_without_verifiable_result"
+
+ return {
+ "field": field,
+ "status": status,
+ "observed": True,
+ "safe_value": safe_value,
+ }
+
+
+def extract_backup_restore_evidence(text: str) -> list[dict[str, Any]]:
+ """Extract only allowlisted, non-secret backup readback facts."""
+
+ lines = _evidence_lines(text)
+ return [_field_observation(field, lines) for field in _REQUIRED_EVIDENCE]
+
+
+def _normalize_observations(
+ observations: Sequence[Mapping[str, Any]] | None,
+ *,
+ evidence_text: str,
+) -> list[dict[str, Any]]:
+ if not observations:
+ return extract_backup_restore_evidence(evidence_text)
+
+ by_field = {
+ str(item.get("field") or ""): dict(item)
+ for item in observations
+ if isinstance(item, Mapping)
+ }
+ normalized: list[dict[str, Any]] = []
+ for field in _REQUIRED_EVIDENCE:
+ item = by_field.get(field) or {}
+ status = str(item.get("status") or "missing")
+ if status not in {"healthy", "failed", "unverified", "missing"}:
+ status = "unverified"
+ normalized.append({
+ "field": field,
+ "status": status,
+ "observed": bool(item.get("observed")) if item else False,
+ "safe_value": item.get("safe_value") if item else None,
+ })
+ return normalized
+
+
+def build_backup_restore_signal_automation_contract(
+ *,
+ project_id: str,
+ event_type: str = BACKUP_RESTORE_EVENT_TYPE,
+ lane: str = BACKUP_RESTORE_LANE,
+ target: str = "backup_restore",
+ evidence_text: str = "",
+ evidence_observations: Sequence[Mapping[str, Any]] | None = None,
+ source_refs: Mapping[str, Any] | None = None,
+ receipt_id: object = "",
+ run_id: object = "",
+ source_row_exists: bool = False,
+ contract_origin: str = "generated_for_send",
+ contract_persisted_at_send: bool = False,
+ recorded_contract_origin: str = "",
+ recorded_policy: Mapping[str, Any] | None = None,
+ recorded_immutable_declared: Mapping[str, Any] | None = None,
+ agent99_dispatch_receipt: Mapping[str, Any] | None = None,
+ declared_gates: Sequence[str] | None = None,
+ declared_runtime_write_gate_count: int = 0,
+ declared_runtime_write_gate_state: str = "unknown",
+ declared_controlled_playbook_queue: bool = False,
+ declared_candidate_only: bool = False,
+ declared_learning_writeback_status: str = "not_evaluated",
+) -> dict[str, Any]:
+ """Build a deterministic, no-write backup readback work package."""
+
+ if contract_origin not in {
+ "generated_for_send",
+ "persisted_at_send",
+ "synthesized_at_read",
+ }:
+ contract_origin = "synthesized_at_read"
+ synthesized_at_read = contract_origin == "synthesized_at_read"
+ policy = backup_restore_readback_policy()
+ refs = source_refs or {}
+ fingerprints = refs.get("fingerprints")
+ source_fingerprint = (
+ str(fingerprints[0])
+ if isinstance(fingerprints, list) and fingerprints
+ else f"{event_type}:{lane}:{target}"
+ )
+ cluster_digest = _stable_digest(project_id, event_type, lane, target, source_fingerprint)
+ receipt_token = _safe_id(receipt_id, fallback=f"pending-{cluster_digest}")
+ run_token = _safe_id(run_id, fallback=f"unbound-{cluster_digest}")
+ work_item_id = f"BRR-WI-{cluster_digest.upper()}"
+ candidate_id = f"BRR-CAND-{_stable_digest(work_item_id, receipt_token).upper()}"
+ durable_receipt_id = f"BRR-RCPT-{_stable_digest(project_id, receipt_token).upper()}"
+ verifier_id = f"BRR-VER-{_stable_digest(work_item_id, run_token).upper()}"
+ dispatch = _normalized_agent99_dispatch_receipt(agent99_dispatch_receipt)
+ if dispatch["runtime_closure_verified"]:
+ identity_binding_status = "bound_closed_verified_learning_written"
+ elif dispatch["post_verifier_passed"]:
+ identity_binding_status = "bound_verifier_passed_learning_pending"
+ elif dispatch["post_verifier_failed"]:
+ identity_binding_status = "bound_verifier_failed_retry_required"
+ elif dispatch["dispatched"]:
+ identity_binding_status = "bound_dispatch_verifier_pending"
+ elif dispatch["receipt_present"]:
+ identity_binding_status = "binding_incomplete_fail_closed"
+ else:
+ identity_binding_status = "dispatch_receipt_missing"
+ identity_binding = {
+ "schema_version": "backup_restore_dispatch_identity_binding_v1",
+ "status": identity_binding_status,
+ "complete": bool(dispatch["dispatched"]),
+ "source_message_receipt_id": str(receipt_id or "") or None,
+ "source_message_run_id": str(run_id or "") or None,
+ "brr_work_item_id": work_item_id,
+ "brr_candidate_id": candidate_id,
+ "brr_verifier_id": verifier_id,
+ "agent99_run_id": dispatch.get("agent99_run_id"),
+ "agent99_trace_id": dispatch.get("trace_id"),
+ "agent99_work_item_id": dispatch.get("agent99_work_item_id"),
+ "same_agent99_identity_required_for_outcome": True,
+ "runtime_closure_verified": bool(
+ dispatch["runtime_closure_verified"]
+ ),
+ }
+
+ checks = _normalize_observations(
+ evidence_observations,
+ evidence_text=evidence_text,
+ )
+ failed_fields = [item["field"] for item in checks if item["status"] == "failed"]
+ missing_fields = [
+ item["field"]
+ for item in checks
+ if item["status"] in {"missing", "unverified"}
+ ]
+ if failed_fields:
+ evidence_result = "failed"
+ domain_work_item_status = "open_evidence_failed"
+ evidence_next_action = (
+ "collect_failed_domain_readback_then_rerun_no_false_green_verifier"
+ )
+ elif missing_fields:
+ evidence_result = "degraded"
+ domain_work_item_status = "open_evidence_incomplete"
+ evidence_next_action = (
+ "collect_missing_backup_readback_then_rerun_no_false_green_verifier"
+ )
+ else:
+ # A firing signal cannot be closed only because all point-in-time checks
+ # look green. The source-resolution receipt remains mandatory.
+ evidence_result = "degraded"
+ domain_work_item_status = "open_source_resolution_receipt_required"
+ evidence_next_action = (
+ "persist_source_resolution_receipt_before_incident_closure"
+ )
+
+ if dispatch["runtime_closure_verified"]:
+ domain_work_item_status = "closed_verified_learning_written"
+ candidate_domain_status = "completed_verified_readback"
+ verifier_domain_status = "success_terminal"
+ next_action = "monitor_backup_restore_recurrence"
+ elif dispatch["post_verifier_passed"]:
+ candidate_domain_status = "verifier_passed_learning_writeback_pending"
+ verifier_domain_status = "success_learning_writeback_pending"
+ next_action = "complete_km_playbook_and_lifecycle_writeback"
+ elif dispatch["post_verifier_failed"]:
+ domain_work_item_status = "open_verifier_failed"
+ candidate_domain_status = "verifier_failed_retry_required"
+ verifier_domain_status = "failed_terminal"
+ next_action = (
+ "collect_failed_verifier_evidence_then_retry_bounded_readback"
+ )
+ elif dispatch["dispatched"]:
+ candidate_domain_status = "dispatched_verifier_pending"
+ verifier_domain_status = "dispatched_verifier_pending"
+ next_action = (
+ "ingest_agent99_backupcheck_outcome_then_run_independent_dr_verifier"
+ )
+ elif dispatch["receipt_present"]:
+ candidate_domain_status = "dispatch_blocked_fail_closed"
+ verifier_domain_status = "dispatch_blocked_verifier_not_started"
+ next_action = str(dispatch["next_action"])
+ elif source_row_exists:
+ candidate_domain_status = "source_row_observed_dispatch_receipt_missing"
+ verifier_domain_status = "dispatch_receipt_missing_verifier_not_started"
+ next_action = "persist_agent99_dispatch_receipt_before_verifier"
+ else:
+ candidate_domain_status = "pending_source_message_persistence"
+ verifier_domain_status = "source_message_pending_verifier_not_started"
+ next_action = "persist_source_message_then_dispatch_backupcheck_readonly"
+
+ def asset_status(value: str) -> str:
+ return f"projected_{value}" if synthesized_at_read else value
+
+ work_item_status = asset_status(domain_work_item_status)
+ candidate_status = asset_status(candidate_domain_status)
+ verifier_status = asset_status(verifier_domain_status)
+ if synthesized_at_read and source_row_exists:
+ source_receipt_status = "projected_from_historical_source_row"
+ elif synthesized_at_read:
+ source_receipt_status = "projected_source_row_missing"
+ elif not source_row_exists:
+ source_receipt_status = "pending_source_message_persistence"
+ elif contract_persisted_at_send:
+ source_receipt_status = "source_row_observed_contract_persisted_at_send"
+ else:
+ source_receipt_status = "source_row_observed_contract_projected_at_read"
+
+ if dispatch["runtime_closure_verified"]:
+ outcome_status = "terminal_verified_learning_written"
+ outcome_next_action = "monitor_backup_restore_recurrence"
+ elif dispatch["post_verifier_passed"]:
+ outcome_status = "verifier_passed_learning_writeback_pending"
+ outcome_next_action = "complete_km_playbook_and_lifecycle_writeback"
+ elif dispatch["post_verifier_failed"]:
+ outcome_status = "verifier_failed_no_runtime_closure"
+ outcome_next_action = (
+ "collect_failed_verifier_evidence_then_retry_bounded_readback"
+ )
+ elif dispatch["dispatched"]:
+ outcome_status = "dispatch_accepted_outcome_pending"
+ outcome_next_action = (
+ "persist_agent99_outcome_contract_v1_with_dispatch_identity_then_"
+ "run_independent_backup_dr_evidence_verifier"
+ )
+ else:
+ outcome_status = "outcome_source_unavailable"
+ outcome_next_action = str(dispatch["next_action"])
+
+ outcome_readback = {
+ "schema_version": "backup_restore_agent99_outcome_readback_v1",
+ "required_source_schema": "agent99_outcome_contract_v1",
+ "status": outcome_status,
+ "terminal": bool(
+ dispatch["runtime_closure_verified"]
+ or dispatch["post_verifier_failed"]
+ ),
+ "same_identity_required": True,
+ "agent99_run_id": dispatch.get("agent99_run_id"),
+ "agent99_trace_id": dispatch.get("trace_id"),
+ "agent99_work_item_id": dispatch.get("agent99_work_item_id"),
+ "brr_work_item_id": work_item_id,
+ "brr_verifier_id": verifier_id,
+ "dispatch_alert_id": dispatch.get("alert_id"),
+ "backup_evidence_refs_complete": dispatch.get(
+ "backup_evidence_refs_complete"
+ ),
+ "missing_backup_evidence_refs": dispatch.get(
+ "missing_backup_evidence_refs"
+ ),
+ "required_mode": "BackupCheck",
+ "required_fields": [
+ "outcome.state",
+ "outcome.resolved",
+ "transportOk",
+ "verifierName",
+ "verifierPassed",
+ "evidenceFresh",
+ "evidenceModeMatched",
+ "sourceEventResolutionRequired",
+ "sourceEventResolutionPolicy",
+ "sourceEventResolved",
+ "sourceEventResolutionEvidence",
+ "failedChecks",
+ "checks",
+ "verifiedAt",
+ ],
+ "next_action": outcome_next_action,
+ }
+
+ immutable_declared = {
+ "gates": list(declared_gates or []),
+ "runtime_write_gate_count": int(declared_runtime_write_gate_count or 0),
+ "runtime_write_gate_state": str(
+ declared_runtime_write_gate_state or "unknown"
+ ),
+ "runtime_write_allowed": bool(declared_runtime_write_gate_count),
+ "controlled_playbook_queue": bool(declared_controlled_playbook_queue),
+ "candidate_only": bool(declared_candidate_only),
+ "learning_writeback_status": str(
+ declared_learning_writeback_status or "not_evaluated"
+ ),
+ }
+ if recorded_policy is None and contract_persisted_at_send:
+ recorded_policy_snapshot = dict(policy)
+ else:
+ policy_source = recorded_policy or {}
+ recorded_policy_snapshot = {
+ key: policy_source[key]
+ for key in (
+ "schema_version",
+ "policy_version",
+ "effective_at",
+ "source_ref",
+ "source_hash",
+ "historical_projection_only",
+ "runtime_write_requires_terminal_verifier",
+ "metadata_refs_are_learning_writeback",
+ "agent99_dispatch_is_runtime_closure",
+ )
+ if key in policy_source
+ }
+ if recorded_immutable_declared is None and contract_persisted_at_send:
+ recorded_immutable_snapshot = dict(immutable_declared)
+ else:
+ immutable_source = recorded_immutable_declared or {}
+ recorded_immutable_snapshot = {
+ key: immutable_source[key]
+ for key in (
+ "gates",
+ "runtime_write_gate_count",
+ "runtime_write_gate_state",
+ "runtime_write_allowed",
+ "controlled_playbook_queue",
+ "candidate_only",
+ "learning_writeback_status",
+ )
+ if key in immutable_source
+ }
+ contract_provenance = {
+ "schema_version": "backup_restore_contract_provenance_v1",
+ "source_contract_present_at_send": bool(contract_persisted_at_send),
+ "recorded_contract_origin": (
+ str(recorded_contract_origin or "").strip()
+ or (
+ contract_origin
+ if contract_persisted_at_send
+ else "source_contract_missing"
+ )
+ ),
+ "recorded_policy_present": bool(recorded_policy_snapshot),
+ "recorded_policy_effective_at": recorded_policy_snapshot.get(
+ "effective_at"
+ ),
+ "recorded_policy_source_hash": recorded_policy_snapshot.get(
+ "source_hash"
+ ),
+ "recorded_immutable_declared_present": bool(
+ recorded_immutable_snapshot
+ ),
+ "recorded_immutable_declared": recorded_immutable_snapshot,
+ "effective_projection_policy_effective_at": policy["effective_at"],
+ "effective_projection_policy_source_hash": policy["source_hash"],
+ "synthesized_at_read": synthesized_at_read,
+ }
+ effective_policy_projection = {
+ "schema_version": "backup_restore_effective_policy_projection_v1",
+ "policy": policy,
+ "contract_origin": contract_origin,
+ "effective_gates": [
+ "controlled_playbook_queue=readback_only",
+ "runtime_write_gate=0_until_verifier",
+ ],
+ "runtime_write_gate_count": 0,
+ "runtime_write_allowed": False,
+ "controlled_playbook_queue": False,
+ "candidate_only": True,
+ "learning_writeback_ready": bool(
+ dispatch["learning_writeback_complete"]
+ ),
+ "learning_writeback_status": (
+ "closed_verified_learning_written"
+ if dispatch["runtime_closure_verified"]
+ else "verifier_passed_learning_writeback_pending"
+ if dispatch["post_verifier_passed"]
+ else "blocked_backup_restore_outcome_and_independent_verifier"
+ ),
+ "candidate_status": candidate_status,
+ "verifier_status": verifier_status,
+ "production_write_performed": False,
+ }
+
+ asset_ledger = [
+ {
+ "asset_type": "WorkItem",
+ "asset_id": work_item_id,
+ "owner": "backup_dr",
+ "status": work_item_status,
+ "domain_status": domain_work_item_status,
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ {
+ "asset_type": "ReadbackCandidate",
+ "asset_id": candidate_id,
+ "owner": "Agent99",
+ "status": candidate_status,
+ "domain_status": candidate_domain_status,
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ {
+ "asset_type": "DurableReceipt",
+ "asset_id": durable_receipt_id,
+ "owner": "AwoooP",
+ "status": source_receipt_status,
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ {
+ "asset_type": "Verifier",
+ "asset_id": verifier_id,
+ "owner": "PostExecutionVerifier",
+ "status": verifier_status,
+ "domain_status": verifier_domain_status,
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ {
+ "asset_type": "Agent99DispatchBinding",
+ "asset_id": (
+ str(dispatch.get("agent99_run_id") or "")
+ or f"AGENT99-BIND-{cluster_digest.upper()}"
+ ),
+ "owner": "Agent99",
+ "status": asset_status(identity_binding_status),
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ {
+ "asset_type": "DRScorecard",
+ "asset_id": f"DR-SCORECARD-{cluster_digest.upper()}",
+ "owner": "backup_dr",
+ "status": asset_status(
+ "closed_verified_learning_written"
+ if dispatch["runtime_closure_verified"]
+ else "writeback_blocked_until_verifier_terminal"
+ ),
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ ]
+
+ return {
+ "schema_version": "backup_restore_readback_automation_v1",
+ "event_type": event_type,
+ "lane": lane,
+ "target": target,
+ "source_fingerprint": source_fingerprint,
+ "contract_origin": contract_origin,
+ "contract_persisted_at_send": bool(contract_persisted_at_send),
+ "policy": policy,
+ "contract_provenance": contract_provenance,
+ "immutable_declared": immutable_declared,
+ "effective_policy_projection": effective_policy_projection,
+ "work_item": {
+ "work_item_id": work_item_id,
+ "kind": "backup_restore_readback",
+ "owner": "backup_dr",
+ "status": work_item_status,
+ "domain_status": domain_work_item_status,
+ "contract_origin": contract_origin,
+ "next_action": next_action,
+ },
+ "candidate": {
+ "candidate_id": candidate_id,
+ "action": "collect_readonly_backup_restore_readback",
+ "mode": "BackupCheck",
+ "risk": "low",
+ "status": candidate_status,
+ "domain_status": candidate_domain_status,
+ "contract_origin": contract_origin,
+ "execution_state": (
+ "completed_verified"
+ if dispatch["runtime_closure_verified"]
+ else "dispatched"
+ if dispatch["dispatched"]
+ else "not_started"
+ ),
+ "work_item_id": work_item_id,
+ "source_message_run_id": str(run_id or "") or None,
+ "agent99_dispatch": dispatch,
+ "identity_binding": identity_binding,
+ "allowed_operations": [
+ "read_backup_status_metadata",
+ "read_freshness_metrics",
+ "read_offsite_verifier_receipt",
+ "read_escrow_missing_count",
+ "read_restore_drill_receipt",
+ ],
+ },
+ "receipt": {
+ "receipt_id": durable_receipt_id,
+ "source": "awooop_outbound_message",
+ "source_receipt_id": str(receipt_id or ""),
+ "source_message_run_id": str(run_id or ""),
+ "status": source_receipt_status,
+ "source_row_exists": bool(source_row_exists),
+ "contract_origin": contract_origin,
+ "contract_persisted_at_send": bool(contract_persisted_at_send),
+ "is_agent99_execution_receipt": False,
+ },
+ "verifier": {
+ "verifier_id": verifier_id,
+ "status": verifier_status,
+ "domain_status": verifier_domain_status,
+ "evidence_result": evidence_result,
+ "result": (
+ "success"
+ if dispatch["runtime_closure_verified"]
+ else "failed"
+ if dispatch["post_verifier_failed"]
+ else "degraded"
+ if dispatch["post_verifier_passed"]
+ else evidence_result
+ ),
+ "terminal": bool(
+ dispatch["runtime_closure_verified"]
+ or dispatch["post_verifier_failed"]
+ ),
+ "no_false_green": True,
+ "checks": checks,
+ "failed_fields": failed_fields,
+ "missing_fields": missing_fields,
+ "required_fields": list(_REQUIRED_EVIDENCE),
+ "agent99_dispatch": dispatch,
+ "identity_binding": identity_binding,
+ "agent99_outcome": outcome_readback,
+ "backup_evidence_refs_complete": dispatch.get(
+ "backup_evidence_refs_complete"
+ ),
+ "missing_backup_evidence_refs": dispatch.get(
+ "missing_backup_evidence_refs"
+ ),
+ "evidence_next_action": evidence_next_action,
+ },
+ "asset_ledger": asset_ledger,
+ "declared_gates": list(declared_gates or []),
+ "effective_gates": list(effective_policy_projection["effective_gates"]),
+ "prohibited_actions": list(_PROHIBITED_ACTIONS),
+ "controlled_repair_queue": False,
+ "read_only_candidate_allowed": True,
+ "runtime_write_allowed": False,
+ "runtime_execution_authorized": False,
+ "production_write_performed": False,
+ "human_intervention_required": False,
+ "closure_state": (
+ "closed_verified_learning_written"
+ if dispatch["runtime_closure_verified"]
+ else "partial_degraded_safe_next_action"
+ ),
+ "next_action": next_action,
+ }
diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py
new file mode 100644
index 000000000..0b2265cbf
--- /dev/null
+++ b/apps/api/src/services/controlled_alert_target_router.py
@@ -0,0 +1,281 @@
+"""Domain-aware routing for non-Kubernetes controlled alert targets.
+
+The Alertmanager namespace label is a source attribute, not proof that every
+alert is a Kubernetes workload. Cold-start/reboot recovery is a host control
+plane lane. It must never fall through to a guessed ``kubectl`` action or the
+generic Ansible keyword catalog merely because the source payload mentions a
+host, backup, or container.
+
+This module is deliberately pure. It only builds machine-readable routing
+and receipt contracts; it does not dispatch Agent99, run a command, or write
+runtime state.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+_RECOVERY_MARKERS = (
+ "cold-start",
+ "cold_start",
+ "cold start",
+ "full-stack-cold-start",
+ "full_stack_cold_start",
+ "full stack cold start",
+ "reboot-auto-recovery",
+ "reboot_auto_recovery",
+ "reboot auto recovery",
+)
+
+
+def _text_token(value: str | None) -> str:
+ return " ".join(str(value or "").strip().lower().split())
+
+
+def _asset_token(value: str) -> str:
+ normalized = re.sub(r"[^a-z0-9_.-]+", "-", value.lower()).strip("-.")
+ return normalized[:96] or "unbound"
+
+
+def resolve_controlled_alert_target(
+ *,
+ alertname: str,
+ target_resource: str,
+ namespace: str,
+) -> dict[str, Any] | None:
+ """Return the dedicated executor route for a known non-K8s target."""
+
+ text = " ".join(
+ (
+ _text_token(alertname),
+ _text_token(target_resource),
+ )
+ )
+ if not any(marker in text for marker in _RECOVERY_MARKERS):
+ return None
+
+ return {
+ "schema_version": "controlled_alert_target_route_v1",
+ "target_kind": "control_plane_recovery",
+ "target_resource": target_resource or "cold-start-gate",
+ "source_namespace": namespace or None,
+ "normalized_execution_namespace": "host_recovery",
+ "kubernetes_namespace_applicable": False,
+ "executor": "Agent99",
+ "route_id": "agent99_recover_after_owner_review",
+ "check_route": "agent99-mode Status controlledApply=false",
+ "controlled_apply_route": "agent99-mode Recover controlledApply=true",
+ "rollback_route": "agent99-mode Status controlledApply=false",
+ "runtime_execution_authorized": False,
+ "runtime_write_allowed": False,
+ }
+
+
+def build_controlled_recovery_promotion_contract(
+ *,
+ alertname: str,
+ target_resource: str,
+ namespace: str,
+ incident_id: str = "",
+) -> dict[str, Any] | None:
+ """Build the no-false-green check/apply/verifier/learning contract."""
+
+ route = resolve_controlled_alert_target(
+ alertname=alertname,
+ target_resource=target_resource,
+ namespace=namespace,
+ )
+ if route is None:
+ return None
+
+ source_id = _asset_token(incident_id or f"{alertname}:{target_resource}")
+ fields = [
+ {
+ "field": "target_selector",
+ "status": "ready",
+ "source": "domain_target_normalizer",
+ "value": {
+ "target_kind": route["target_kind"],
+ "target_resource": route["target_resource"],
+ "normalized_execution_namespace": route[
+ "normalized_execution_namespace"
+ ],
+ },
+ },
+ {
+ "field": "source_sensor_receipt",
+ "status": "pending",
+ "source": "alertmanager+cold_start_scorecard",
+ "value": "require_current_firing_or_recurrence_and_scorecard_ref",
+ },
+ {
+ "field": "check_mode_receipt",
+ "status": "pending",
+ "source": "Agent99 Status + read-only cold-start sensor",
+ "value": route["check_route"],
+ },
+ {
+ "field": "controlled_apply_route",
+ "status": "ready",
+ "source": "domain_target_normalizer",
+ "value": route["controlled_apply_route"],
+ },
+ {
+ "field": "dispatch_receipt",
+ "status": "pending",
+ "source": "agent99_sre_dispatch_receipt_v1",
+ "value": "accepted_and_inbox_triggered_required",
+ },
+ {
+ "field": "rollback_route",
+ "status": "ready",
+ "source": "domain_target_normalizer",
+ "value": route["rollback_route"],
+ },
+ {
+ "field": "post_apply_verifier",
+ "status": "pending",
+ "source": "Agent99 outcome + cold-start scorecard",
+ "value": [
+ "agent99_recover_outcome_terminal_readback",
+ "full_stack_cold_start_scorecard_rerun",
+ "required_hosts_harbor_k3s_routes_green",
+ "original_fingerprint_resolved_or_recurrence_decreased",
+ ],
+ },
+ {
+ "field": "incident_closure_receipt",
+ "status": "pending",
+ "source": "incident_timeline",
+ "value": "terminal_only_after_independent_verifier",
+ },
+ {
+ "field": "telegram_receipt",
+ "status": "pending",
+ "source": "telegram_outbound_mirror",
+ "value": "lifecycle_result_delivery_ack_required",
+ },
+ {
+ "field": "km_writeback",
+ "status": "pending",
+ "source": "Hermes",
+ "value": f"km:cold-start-recovery:{source_id}",
+ },
+ {
+ "field": "playbook_trust_writeback",
+ "status": "pending",
+ "source": "OpenClaw/NemoTron",
+ "value": f"playbook-trust:cold-start-recovery:{source_id}",
+ },
+ ]
+ ready_fields = [row["field"] for row in fields if row["status"] == "ready"]
+ blocked_fields = [row["field"] for row in fields if row["status"] != "ready"]
+ return {
+ "schema_version": "repair_candidate_promotion_contract_v1",
+ "status": "controlled_route_candidate_pending_receipts",
+ "route_id": route["route_id"],
+ "executor": route["executor"],
+ "incident_id": incident_id or None,
+ "target_route": route,
+ "ready_count": len(ready_fields),
+ "total_count": len(fields),
+ "blocked_count": len(blocked_fields),
+ "ready_fields": ready_fields,
+ "blocked_fields": blocked_fields,
+ "fields": fields,
+ "runtime_execution_authorized": False,
+ "runtime_write_allowed": False,
+ "controlled_playbook_queue": True,
+ "owner_review_required": False,
+ "owner_review_gate": "auto_waived_for_low_medium_high",
+ "needs_human": False,
+ "completion_status": "partial",
+ "completion_blocker": "dispatch_verifier_and_learning_receipts_missing",
+ "forbidden_operations": [
+ "kubectl_for_host_control_plane_target",
+ "generic_ansible_keyword_fallback",
+ "host_reboot",
+ "node_drain",
+ "backup_restore",
+ "secret_read",
+ "mark_resolved_without_post_verifier",
+ ],
+ "controlled_automation_closure": {
+ "schema_version": "controlled_recovery_closure_contract_v1",
+ "trace_id_required": True,
+ "status": "pending_source_check_dispatch_verify_writeback",
+ "stages": [
+ {"stage": "sensor", "status": "pending", "writes_runtime_state": False},
+ {"stage": "normalize", "status": "passed", "writes_runtime_state": False},
+ {"stage": "check", "status": "pending", "writes_runtime_state": False},
+ {"stage": "controlled_apply", "status": "pending", "writes_runtime_state": False},
+ {"stage": "verify", "status": "pending", "writes_runtime_state": False},
+ {"stage": "learn_writeback", "status": "pending", "writes_runtime_state": False},
+ ],
+ "required_receipts": [row["field"] for row in fields],
+ "automation_asset_ledger": [
+ {
+ "asset_type": "Sensor",
+ "asset_id": "scripts/reboot-recovery/full-stack-cold-start-check.sh",
+ "owner": "ColdStartMonitor",
+ "status": "read_only_receipt_pending",
+ },
+ {
+ "asset_type": "PlayBook",
+ "asset_id": "agent99:Recover",
+ "owner": "Agent99",
+ "status": "dispatch_receipt_pending",
+ },
+ {
+ "asset_type": "Verifier",
+ "asset_id": f"verifier:cold-start-recovery:{source_id}",
+ "owner": "PostExecutionVerifier",
+ "status": "pending",
+ },
+ {
+ "asset_type": "KM",
+ "asset_id": f"km:cold-start-recovery:{source_id}",
+ "owner": "Hermes",
+ "status": "pending_after_verifier",
+ },
+ ],
+ },
+ }
+
+
+def build_controlled_recovery_handoff(
+ *,
+ alertname: str,
+ target_resource: str,
+ namespace: str,
+ incident_id: str = "",
+) -> dict[str, Any] | None:
+ contract = build_controlled_recovery_promotion_contract(
+ alertname=alertname,
+ target_resource=target_resource,
+ namespace=namespace,
+ incident_id=incident_id,
+ )
+ if contract is None:
+ return None
+ return {
+ "schema_version": "ai_decision_controlled_executor_handoff_v1",
+ "status": "agent99_dispatch_receipt_readback_pending",
+ "queued": False,
+ "side_effect_performed": False,
+ "single_writer_executor": "Agent99",
+ "route_id": contract["route_id"],
+ "active_blockers": [
+ "agent99_dispatch_receipt_readback_pending",
+ "post_apply_verifier_missing",
+ "km_playbook_writeback_missing",
+ ],
+ "safe_next_action": "read_agent99_dispatch_then_run_independent_cold_start_verifier",
+ "repair_candidate_promotion_contract": contract,
+ "runtime_execution_authorized": False,
+ "runtime_write_allowed": False,
+ "owner_review_required": False,
+ "owner_review_gate": "auto_waived_for_low_medium_high",
+ "needs_human": False,
+ }
diff --git a/apps/api/src/services/incident_engine.py b/apps/api/src/services/incident_engine.py
index 7d0b26628..a98336297 100644
--- a/apps/api/src/services/incident_engine.py
+++ b/apps/api/src/services/incident_engine.py
@@ -25,7 +25,7 @@ from typing import Any, Protocol, runtime_checkable
import structlog
from src.core.redis_client import get_redis
-from src.models.incident import Incident
+from src.models.incident import Incident, parse_signal_source
from src.services.graph_rag import BlastRadiusResult, topology_graph
from src.services.incident_memory import get_incident_memory
from src.utils.incident_converter import brain_to_local
@@ -215,7 +215,11 @@ class IncidentEngineAdapter:
async def process_signal(self, signal_data: dict[str, Any]) -> Incident | None:
"""處理 Signal，返回 LocalIncident (brain 輸出轉換)"""
- brain_incident = await self._engine.process_signal(signal_data)
+ normalized_signal = dict(signal_data)
+ normalized_signal["source"] = parse_signal_source(
+ normalized_signal.get("source")
+ )
+ brain_incident = await self._engine.process_signal(normalized_signal)
if brain_incident is None:
return None
return brain_to_local(brain_incident)
diff --git a/apps/api/src/services/incident_service.py b/apps/api/src/services/incident_service.py
index 91a6c86ee..f1f27b64a 100644
--- a/apps/api/src/services/incident_service.py
+++ b/apps/api/src/services/incident_service.py
@@ -26,6 +26,7 @@ from typing import Any, Literal
from uuid import UUID
import structlog
+from pydantic import ValidationError
from sqlalchemy import select, text
from src.core.redis_client import get_redis
@@ -37,6 +38,8 @@ from src.models.incident import (
IncidentStatus,
Severity,
Signal,
+ SignalSource,
+ parse_signal_source,
)
from src.utils.timezone import now_taipei
@@ -47,6 +50,29 @@ class IncidentDurableReadError(RuntimeError):
"""The durable incident source could not be read safely."""
+def _durable_read_reason_code(exc: Exception, *, default: str) -> str:
+ """Map schema failures to public-safe reason codes without leaking values."""
+
+ if isinstance(exc, ValidationError):
+ errors = exc.errors(include_input=False)
+ source_errors = [
+ error
+ for error in errors
+ if tuple(error.get("loc") or ())[-1:] == ("source",)
+ ]
+ if source_errors and all(
+ error.get("type") != "missing" for error in source_errors
+ ):
+ return "incident_signal_source_unsupported"
+ if any(
+ tuple(error.get("loc") or ())[-1:]
+ in {("source",), ("severity",), ("fired_at",)}
+ for error in errors
+ ):
+ return "incident_signal_schema_invalid"
+ return default
+
+
# =============================================================================
# C2 修正: 從 webhooks.py 遷入的業務邏輯
# 2026-04-10 Claude Sonnet 4.6 Asia/Taipei
@@ -320,6 +346,7 @@ async def create_incident_for_approval(
alert_labels: dict | None = None,
notification_type: str | None = None, # ADR-073 Phase 2-2
alert_category: str | None = None, # ADR-073 Phase 2-2
+ canonical_incident_id: str | None = None,
) -> str:
"""
為 Approval 創建對應的 Incident (活躍事件同步)。
@@ -388,7 +415,11 @@ async def create_incident_for_approval(
except Exception:
pass
+ incident_fields: dict[str, Any] = {}
+ if canonical_incident_id:
+ incident_fields["incident_id"] = canonical_incident_id
incident = Incident(
+ **incident_fields,
status=IncidentStatus.INVESTIGATING,
severity=severity,
signals=[signal],
@@ -850,15 +881,18 @@ class IncidentService:
project_id=project_id,
)
except Exception as exc:
+ reason_code = _durable_read_reason_code(
+ exc,
+ default="incident_durable_readback_unavailable",
+ )
logger.warning(
"incident_durable_readback_unavailable",
incident_id=incident_id,
project_id=project_id,
error_type=type(exc).__name__,
+ reason_code=reason_code,
)
- raise IncidentDurableReadError(
- "incident_durable_readback_unavailable"
- ) from exc
+ raise IncidentDurableReadError(reason_code) from exc
async def get_active_incidents(
self,
@@ -874,26 +908,51 @@ class IncidentService:
Returns:
list[Incident]: 活躍事件列表 (investigating 或 mitigating)
"""
+ readback = await self.get_active_incidents_readback(project_id=project_id)
+ return readback.incidents
+
+ async def get_active_incidents_readback(
+ self,
+ *,
+ project_id: str | None = None,
+ ) -> Any:
+ """Return valid rows and bounded degradation metadata from PostgreSQL."""
+
try:
from src.repositories.incident_repository import get_incident_repository
- incidents = await get_incident_repository().get_active(
- project_id=project_id,
- )
+ repository = get_incident_repository()
+ detailed_reader = getattr(repository, "get_active_readback", None)
+ if callable(detailed_reader):
+ readback = await detailed_reader(project_id=project_id)
+ else:
+ # Compatibility for injected repository implementations. The
+ # production PostgreSQL repository always exposes diagnostics.
+ from src.repositories.incident_repository import (
+ ActiveIncidentReadResult,
+ )
+
+ readback = ActiveIncidentReadResult(
+ incidents=await repository.get_active(project_id=project_id)
+ )
logger.info(
"get_active_incidents_from_episodic_memory",
- count=len(incidents),
+ count=len(readback.incidents),
+ degraded_count=len(readback.degraded_records),
)
- return incidents
+ return readback
except Exception as e:
+ reason_code = _durable_read_reason_code(
+ e,
+ default="active_incidents_durable_read_unavailable",
+ )
logger.warning(
"active_incidents_episodic_read_failed",
- error=str(e),
+ error_type=type(e).__name__,
+ reason_code=reason_code,
fallback="disabled_fail_closed",
)
- raise IncidentDurableReadError(
- "active_incidents_durable_read_unavailable"
- ) from e
+ raise IncidentDurableReadError(reason_code) from e
# =========================================================================
# Episodic Memory (PostgreSQL)
@@ -1221,16 +1280,13 @@ class IncidentService:
def _parse_source(
self,
source_str: str,
- ) -> Literal["prometheus", "signoz", "alertmanager", "manual", "telegram"]:
+ ) -> SignalSource:
"""
- 解析來源字串，映射到 Signal 允許的 Literal 值
+ 解析來源字串，映射到 Signal 的 durable contract。
- 不在白名單中的來源一律映射為 'manual'
+ 未登錄的來源明確拒絕，不得靜默改寫為 manual。
"""
- valid_sources = {"prometheus", "signoz", "alertmanager", "manual", "telegram"}
- if source_str.lower() in valid_sources:
- return source_str.lower() # type: ignore
- return "manual"
+ return parse_signal_source(source_str)
def _parse_severity(self, severity_str: str) -> Severity:
"""解析嚴重度字串"""
diff --git a/apps/api/src/services/knowledge_service.py b/apps/api/src/services/knowledge_service.py
index 189a8d07f..96a499664 100644
--- a/apps/api/src/services/knowledge_service.py
+++ b/apps/api/src/services/knowledge_service.py
@@ -660,9 +660,20 @@ class KnowledgeService:
# I1: 持有背景 Task 引用，防止 GC 提前回收
self._pending_tasks: set[asyncio.Task] = set() # type: ignore[type-arg]
- async def create_entry(self, data: KnowledgeEntryCreate) -> KnowledgeEntry:
+ async def create_entry(
+ self,
+ data: KnowledgeEntryCreate,
+ *,
+ project_id: str | None = None,
+ schedule_embedding: bool = True,
+ ) -> KnowledgeEntry:
"""建立知識條目，建立後背景自動產生 embedding"""
- async with get_db_context() as db:
+ context = (
+ get_db_context(project_id)
+ if project_id is not None
+ else get_db_context()
+ )
+ async with context as db:
repo: IKnowledgeRepository = KnowledgeDBRepository(db)
entry = await repo.create(data)
logger.info(
@@ -673,17 +684,33 @@ class KnowledgeService:
)
# 背景產生 embedding (不阻塞回應)；持有引用防 GC 回收
- task = asyncio.create_task(self._embed_entry(entry.id, data.title, data.content))
- self._pending_tasks.add(task)
- task.add_done_callback(self._pending_tasks.discard)
+ if schedule_embedding:
+ task = asyncio.create_task(
+ self._embed_entry(
+ entry.id,
+ data.title,
+ data.content,
+ project_id=project_id,
+ )
+ )
+ self._pending_tasks.add(task)
+ task.add_done_callback(self._pending_tasks.discard)
return entry
- async def _embed_entry(self, entry_id: str, title: str, content: str) -> None:
+ async def _embed_entry(
+ self,
+ entry_id: str,
+ title: str,
+ content: str,
+ *,
+ project_id: str | None = None,
+ ) -> None:
"""背景任務：產生並儲存 embedding"""
await self.ensure_entry_embedding(
entry_id,
title,
content,
+ project_id=project_id,
)
async def ensure_entry_embedding(
diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py
index 2ca07b034..3a332ebb5 100644
--- a/apps/api/src/services/platform_operator_service.py
+++ b/apps/api/src/services/platform_operator_service.py
@@ -53,6 +53,11 @@ from src.services.awooop_truth_chain_service import (
_summarize_mcp,
fetch_truth_chain,
)
+from src.services.backup_restore_signal_automation import (
+ BACKUP_RESTORE_EVENT_TYPE,
+ backup_restore_readback_policy,
+ build_backup_restore_signal_automation_contract,
+)
from src.services.governance_km_stale_review_service import (
query_km_stale_owner_review_completion_queue,
)
@@ -1435,10 +1440,31 @@ async def list_ai_alert_card_delivery_readback(
) AS delivery_receipt_required_total,
COUNT(*) FILTER (
WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) = 'backup_restore_escrow_signal'
+ ) AS backup_restore_total,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,declared_runtime_write_gate_count}}',
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{ai_automation_alert_card,runtime_write_gate_count}}',
'0'
) <> '0'
+ ) AS declared_runtime_write_gate_open_count,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,runtime_write_gate_count}}',
+ '0'
+ ) <> '0'
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> 'backup_restore_escrow_signal'
) AS runtime_write_gate_open_count,
COUNT(*) FILTER (
WHERE COALESCE(
@@ -1456,6 +1482,28 @@ async def list_ai_alert_card_delivery_readback(
'{{ai_automation_alert_card,lane}}',
''
) <> ''
+ ) AS declared_learning_writeback_ready_total,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,delivery_receipt_readback_required}}',
+ ''
+ ) = 'true'
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> ''
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,lane}}',
+ ''
+ ) <> ''
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> 'backup_restore_escrow_signal'
) AS learning_writeback_ready_total,
MAX(m.sent_at) AS latest_sent_at,
MAX(m.queued_at) AS latest_queued_at
@@ -1479,6 +1527,11 @@ async def list_ai_alert_card_delivery_readback(
-> 'ai_automation_alert_card' AS alert_card,
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb)
-> 'source_refs' AS source_refs,
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb)
+ -> 'agent99_dispatch_receipt' AS agent99_dispatch_receipt,
+ ar.error_detail AS agent99_current_receipt,
+ ar.state AS agent99_current_run_state,
+ ar.trace_id AS agent99_current_trace_id,
r.agent_id,
r.state AS run_state,
r.created_at AS run_created_at
@@ -1486,6 +1539,47 @@ async def list_ai_alert_card_delivery_readback(
LEFT JOIN awooop_run_state r
ON r.project_id = m.project_id
AND r.run_id = m.run_id
+ LEFT JOIN LATERAL (
+ SELECT
+ ars.error_detail,
+ ars.state,
+ ars.trace_id
+ FROM awooop_run_state ars
+ WHERE ars.project_id = m.project_id
+ AND ars.agent_id = 'agent99_controlled_dispatch'
+ AND (
+ ars.run_id::text = COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{agent99_dispatch_receipt,run_id}}',
+ ''
+ )
+ OR (
+ COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{agent99_dispatch_receipt,incident_id}}',
+ ''
+ ) <> ''
+ AND ars.trigger_ref LIKE (
+ 'agent99:' ||
+ COALESCE(
+ COALESCE(
+ m.source_envelope::jsonb,
+ '{{}}'::jsonb
+ ) #>> '{{agent99_dispatch_receipt,incident_id}}',
+ ''
+ ) || ':%'
+ )
+ )
+ )
+ ORDER BY (
+ ars.run_id::text = COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{agent99_dispatch_receipt,run_id}}',
+ ''
+ )
+ ) DESC, ars.created_at DESC
+ LIMIT 1
+ ) ar ON TRUE
WHERE {where_sql}
ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC
LIMIT :limit OFFSET :offset
@@ -1713,10 +1807,31 @@ async def _load_ai_alert_card_delivery_readback_direct(
) AS delivery_receipt_required_total,
COUNT(*) FILTER (
WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) = 'backup_restore_escrow_signal'
+ ) AS backup_restore_total,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,declared_runtime_write_gate_count}}',
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
'{{ai_automation_alert_card,runtime_write_gate_count}}',
'0'
) <> '0'
+ ) AS declared_runtime_write_gate_open_count,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,runtime_write_gate_count}}',
+ '0'
+ ) <> '0'
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> 'backup_restore_escrow_signal'
) AS runtime_write_gate_open_count,
COUNT(*) FILTER (
WHERE COALESCE(
@@ -1734,6 +1849,28 @@ async def _load_ai_alert_card_delivery_readback_direct(
'{{ai_automation_alert_card,lane}}',
''
) <> ''
+ ) AS declared_learning_writeback_ready_total,
+ COUNT(*) FILTER (
+ WHERE COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,delivery_receipt_readback_required}}',
+ ''
+ ) = 'true'
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> ''
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,lane}}',
+ ''
+ ) <> ''
+ AND COALESCE(
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb) #>>
+ '{{ai_automation_alert_card,event_type}}',
+ ''
+ ) <> 'backup_restore_escrow_signal'
) AS learning_writeback_ready_total,
MAX(m.sent_at) AS latest_sent_at,
MAX(m.queued_at) AS latest_queued_at
@@ -1764,6 +1901,13 @@ async def _load_ai_alert_card_delivery_readback_direct(
COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb)
-> 'source_refs'
)::text AS source_refs_json,
+ (
+ COALESCE(m.source_envelope::jsonb, '{{}}'::jsonb)
+ -> 'agent99_dispatch_receipt'
+ )::text AS agent99_dispatch_receipt_json,
+ ar.error_detail::text AS agent99_current_receipt_json,
+ ar.state AS agent99_current_run_state,
+ ar.trace_id::text AS agent99_current_trace_id,
r.agent_id,
r.state AS run_state,
r.created_at AS run_created_at
@@ -1771,6 +1915,53 @@ async def _load_ai_alert_card_delivery_readback_direct(
LEFT JOIN awooop_run_state r
ON r.project_id = m.project_id
AND r.run_id = m.run_id
+ LEFT JOIN LATERAL (
+ SELECT
+ ars.error_detail,
+ ars.state,
+ ars.trace_id
+ FROM awooop_run_state ars
+ WHERE ars.project_id = m.project_id
+ AND ars.agent_id = 'agent99_controlled_dispatch'
+ AND (
+ ars.run_id::text = COALESCE(
+ COALESCE(
+ m.source_envelope::jsonb,
+ '{{}}'::jsonb
+ ) #>> '{{agent99_dispatch_receipt,run_id}}',
+ ''
+ )
+ OR (
+ COALESCE(
+ COALESCE(
+ m.source_envelope::jsonb,
+ '{{}}'::jsonb
+ ) #>> '{{agent99_dispatch_receipt,incident_id}}',
+ ''
+ ) <> ''
+ AND ars.trigger_ref LIKE (
+ 'agent99:' ||
+ COALESCE(
+ COALESCE(
+ m.source_envelope::jsonb,
+ '{{}}'::jsonb
+ ) #>> '{{agent99_dispatch_receipt,incident_id}}',
+ ''
+ ) || ':%'
+ )
+ )
+ )
+ ORDER BY (
+ ars.run_id::text = COALESCE(
+ COALESCE(
+ m.source_envelope::jsonb,
+ '{{}}'::jsonb
+ ) #>> '{{agent99_dispatch_receipt,run_id}}',
+ ''
+ )
+ ) DESC, ars.created_at DESC
+ LIMIT 1
+ ) ar ON TRUE
WHERE {where_sql}
ORDER BY COALESCE(m.sent_at, m.queued_at) DESC, m.message_id DESC
LIMIT ${limit_param} OFFSET ${offset_param}
@@ -1810,6 +2001,12 @@ def _ai_alert_card_direct_row(row: Mapping[str, Any]) -> dict[str, Any]:
payload = dict(row)
payload["alert_card"] = _json_object(payload.pop("alert_card_json", None))
payload["source_refs"] = _json_object(payload.pop("source_refs_json", None))
+ payload["agent99_dispatch_receipt"] = _json_object(
+ payload.pop("agent99_dispatch_receipt_json", None)
+ )
+ payload["agent99_current_receipt"] = _json_object(
+ payload.pop("agent99_current_receipt_json", None)
+ )
return payload
@@ -1951,15 +2148,56 @@ def _ai_alert_card_delivery_summary_from_row(
runtime_write_gate_open_count = _safe_int(
row.get("runtime_write_gate_open_count")
)
+ declared_runtime_write_gate_open_count = _safe_int(
+ row.get("declared_runtime_write_gate_open_count")
+ if row.get("declared_runtime_write_gate_open_count") is not None
+ else runtime_write_gate_open_count
+ )
learning_writeback_ready_total = _safe_int(
row.get("learning_writeback_ready_total")
)
+ declared_learning_writeback_ready_total = _safe_int(
+ row.get("declared_learning_writeback_ready_total")
+ if row.get("declared_learning_writeback_ready_total") is not None
+ else learning_writeback_ready_total
+ )
+ backup_restore_filtered = event_type == BACKUP_RESTORE_EVENT_TYPE
+ backup_restore_total = _safe_int(row.get("backup_restore_total"))
+ if backup_restore_filtered:
+ backup_restore_total = total
+ runtime_write_gate_open_count = 0
+ learning_writeback_ready_total = 0
+ backup_runtime_gate_excluded_total = max(
+ declared_runtime_write_gate_open_count - runtime_write_gate_open_count,
+ 0,
+ )
+ backup_learning_writeback_excluded_total = max(
+ declared_learning_writeback_ready_total - learning_writeback_ready_total,
+ 0,
+ )
+ backup_restore_policy_applied = bool(
+ backup_restore_total
+ or backup_runtime_gate_excluded_total
+ or backup_learning_writeback_excluded_total
+ )
learning_writeback_missing_total = max(total - learning_writeback_ready_total, 0)
status_value = "no_delivery_receipt" if total == 0 else "observed"
if failed_total > 0:
status_value = "delivery_failure_observed"
elif pending_total > 0:
status_value = "delivery_pending_observed"
+ elif backup_restore_filtered and total > 0:
+ status_value = "blocked_backup_restore_evidence_or_verifier"
+ elif backup_restore_policy_applied and total > 0:
+ status_value = "observed_effective_backup_policy_projection_applied"
+
+ learning_next_action = (
+ "collect_backup_status_freshness_offsite_escrow_restore_and_resolution_receipts"
+ if backup_restore_policy_applied and total > 0
+ else "none"
+ if total > 0 and learning_writeback_missing_total == 0
+ else "ensure_ai_alert_cards_include_event_type_lane_and_delivery_receipt"
+ )
return {
"schema_version": "awooop_ai_alert_card_delivery_readback_v1",
@@ -1975,8 +2213,21 @@ def _ai_alert_card_delivery_summary_from_row(
"delivery_receipt_required_total": _safe_int(
row.get("delivery_receipt_required_total")
),
+ "backup_restore_total": backup_restore_total,
+ "declared_runtime_write_gate_open_count": (
+ declared_runtime_write_gate_open_count
+ ),
+ "backup_runtime_write_gate_excluded_total": (
+ backup_runtime_gate_excluded_total
+ ),
"runtime_write_gate_open_count": runtime_write_gate_open_count,
"runtime_write_allowed": runtime_write_gate_open_count > 0,
+ "declared_learning_writeback_ready_total": (
+ declared_learning_writeback_ready_total
+ ),
+ "backup_learning_writeback_excluded_total": (
+ backup_learning_writeback_excluded_total
+ ),
"learning_writeback_ready_total": learning_writeback_ready_total,
"learning_writeback_missing_total": learning_writeback_missing_total,
"km_writeback_ref_ready_total": learning_writeback_ready_total,
@@ -1984,11 +2235,40 @@ def _ai_alert_card_delivery_summary_from_row(
"rag_chunk_ref_ready_total": learning_writeback_ready_total,
"mcp_evidence_ref_ready_total": learning_writeback_ready_total,
"post_verifier_ref_ready_total": learning_writeback_ready_total,
- "learning_writeback_next_action": (
- "none"
- if total > 0 and learning_writeback_missing_total == 0
- else "ensure_ai_alert_cards_include_event_type_lane_and_delivery_receipt"
+ "learning_writeback_next_action": learning_next_action,
+ "backup_restore_readback_required": backup_restore_policy_applied,
+ "no_false_green_enforced": backup_restore_policy_applied,
+ "backup_restore_next_action": (
+ learning_next_action if backup_restore_policy_applied else "none"
),
+ "effective_policy_projection": {
+ "schema_version": "ai_alert_card_effective_policy_projection_v1",
+ "policy": backup_restore_readback_policy(),
+ "applied": backup_restore_policy_applied,
+ "scope": (
+ "backup_restore_filtered"
+ if backup_restore_filtered
+ else "mixed_or_unfiltered"
+ ),
+ "declared_runtime_write_gate_open_count": (
+ declared_runtime_write_gate_open_count
+ ),
+ "effective_runtime_write_gate_open_count": (
+ runtime_write_gate_open_count
+ ),
+ "declared_learning_writeback_ready_total": (
+ declared_learning_writeback_ready_total
+ ),
+ "effective_learning_writeback_ready_total": (
+ learning_writeback_ready_total
+ ),
+ "backup_runtime_write_gate_excluded_total": (
+ backup_runtime_gate_excluded_total
+ ),
+ "backup_learning_writeback_excluded_total": (
+ backup_learning_writeback_excluded_total
+ ),
+ },
"learning_registry_target_count": 0,
"learning_registry_ready_target_count": 0,
"learning_registry_missing_target_count": 0,
@@ -1999,12 +2279,133 @@ def _ai_alert_card_delivery_summary_from_row(
}
+def _agent99_current_dispatch_projection(
+ row: Mapping[str, Any],
+ *,
+ frozen_projection: Mapping[str, Any],
+) -> dict[str, Any]:
+ """Project the current durable ledger receipt with strict identity checks."""
+
+ current = _json_object(row.get("agent99_current_receipt"))
+ if current.get("schema_version") != "agent99_controlled_dispatch_receipt_v1":
+ return dict(frozen_projection)
+
+ identity = _as_dict(current.get("identity"))
+ dispatch = _as_dict(current.get("dispatch_receipt"))
+ verifier = _as_dict(current.get("verifier"))
+ learning = _as_dict(current.get("learning_writeback"))
+ run_id = str(identity.get("run_id") or "").strip()
+ trace_id = str(identity.get("trace_id") or "").strip()
+ work_item_id = str(identity.get("work_item_id") or "").strip()
+ incident_id = str(identity.get("incident_id") or "").strip()
+ identity_complete = bool(run_id and trace_id and work_item_id and incident_id)
+
+ expected_pairs = (
+ (str(frozen_projection.get("run_id") or "").strip(), run_id),
+ (str(frozen_projection.get("trace_id") or "").strip(), trace_id),
+ (
+ str(frozen_projection.get("work_item_id") or "").strip(),
+ work_item_id,
+ ),
+ (
+ str(frozen_projection.get("incident_id") or "").strip(),
+ incident_id,
+ ),
+ )
+ identity_matches = identity_complete and all(
+ not expected or expected == observed
+ for expected, observed in expected_pairs
+ )
+ row_trace_id = str(row.get("agent99_current_trace_id") or "").strip()
+ if row_trace_id and row_trace_id != trace_id:
+ identity_matches = False
+
+ kind = str(
+ dispatch.get("kind") or frozen_projection.get("kind") or ""
+ ).strip()
+ suggested_mode = str(
+ frozen_projection.get("suggested_mode")
+ or frozen_projection.get("suggestedMode")
+ or ""
+ ).strip()
+ if not identity_matches:
+ return {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "current_ledger_identity_mismatch_fail_closed",
+ "kind": kind,
+ "suggested_mode": suggested_mode,
+ "accepted": False,
+ "inbox_triggered": False,
+ "incident_id": incident_id or None,
+ "run_id": run_id or None,
+ "trace_id": trace_id or None,
+ "work_item_id": work_item_id or None,
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ "current_ledger_observed": True,
+ "identity_binding_valid": False,
+ "stores_raw_response": False,
+ }
+
+ return {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": str(current.get("status") or "unknown"),
+ "transport": str(dispatch.get("transport") or "unknown"),
+ "alert_id": str(
+ dispatch.get("alert_id")
+ or frozen_projection.get("alert_id")
+ or ""
+ ),
+ "kind": kind,
+ "suggested_mode": suggested_mode,
+ "accepted": bool(
+ current.get("dispatch_accepted") is True
+ and dispatch.get("accepted") is True
+ ),
+ "inbox_triggered": bool(dispatch.get("inbox_triggered") is True),
+ "incident_id": incident_id,
+ "run_id": run_id,
+ "trace_id": trace_id,
+ "work_item_id": work_item_id,
+ "idempotency_key": str(identity.get("idempotency_key") or ""),
+ "receipt_persisted": True,
+ "runtime_execution_attempted": bool(
+ current.get("runtime_execution_attempted") is True
+ ),
+ "post_verifier_passed": bool(
+ current.get("post_verifier_passed") is True
+ ),
+ "runtime_closure_verified": bool(
+ current.get("runtime_closure_verified") is True
+ ),
+ "closure_complete": bool(current.get("closure_complete") is True),
+ "current_run_state": str(
+ row.get("agent99_current_run_state") or "unknown"
+ ),
+ "current_verifier": verifier,
+ "current_learning_writeback": learning,
+ "current_ledger_observed": True,
+ "identity_binding_valid": True,
+ "stores_raw_response": False,
+ }
+
+
def _ai_alert_card_delivery_item(row: Mapping[str, Any]) -> dict[str, Any]:
"""Convert one AI alert-card outbound mirror row into delivery evidence."""
alert_card = _as_dict(row.get("alert_card"))
source_refs = _as_dict(row.get("source_refs"))
+ frozen_agent99_dispatch_receipt = _as_dict(
+ row.get("agent99_dispatch_receipt")
+ )
+ agent99_dispatch_receipt = _agent99_current_dispatch_projection(
+ row,
+ frozen_projection=frozen_agent99_dispatch_receipt,
+ )
run_id = row.get("run_id")
project_id = str(row.get("project_id") or "")
+ event_type = str(alert_card.get("event_type") or "")
+ lane = str(alert_card.get("lane") or "")
+ target = str(alert_card.get("target") or "")
runtime_write_gate_count = _safe_int(
alert_card.get("runtime_write_gate_count")
)
@@ -2016,6 +2417,113 @@ def _ai_alert_card_delivery_item(row: Mapping[str, Any]) -> dict[str, Any]:
run_id=run_id,
project_id=project_id,
)
+ backup_restore_automation: dict[str, Any] | None = None
+ gates = (
+ alert_card.get("gates")
+ if isinstance(alert_card.get("gates"), list)
+ else []
+ )
+ declared_gates = (
+ list(alert_card.get("declared_gates"))
+ if isinstance(alert_card.get("declared_gates"), list)
+ else list(gates)
+ )
+ declared_runtime_write_gate_count = _safe_int(
+ alert_card.get("declared_runtime_write_gate_count")
+ if alert_card.get("declared_runtime_write_gate_count") is not None
+ else 1
+ if "runtime_write_gate=controlled" in declared_gates
+ else runtime_write_gate_count
+ )
+ declared_runtime_write_gate_state = str(
+ alert_card.get("declared_runtime_write_gate_state")
+ or ("controlled" if declared_runtime_write_gate_count else "unknown")
+ )
+ declared_candidate_only = bool(
+ alert_card.get("declared_candidate_only")
+ if alert_card.get("declared_candidate_only") is not None
+ else "candidate_only" in declared_gates
+ )
+ declared_controlled_playbook_queue = bool(
+ alert_card.get("declared_controlled_playbook_queue")
+ if alert_card.get("declared_controlled_playbook_queue") is not None
+ else "controlled_playbook_queue" in declared_gates
+ )
+ declared_learning_writeback_status = str(learning_writeback_refs["status"])
+ candidate_only = bool(alert_card.get("candidate_only"))
+ controlled_playbook_queue = bool(alert_card.get("controlled_playbook_queue"))
+ runtime_write_gate_state = str(
+ alert_card.get("runtime_write_gate_state") or "unknown"
+ )
+ if event_type == BACKUP_RESTORE_EVENT_TYPE:
+ existing_contract = _as_dict(alert_card.get("backup_restore_automation"))
+ contract_persisted_at_send = (
+ existing_contract.get("schema_version")
+ == "backup_restore_readback_automation_v1"
+ )
+ contract_origin = (
+ "persisted_at_send"
+ if contract_persisted_at_send
+ else "synthesized_at_read"
+ )
+ existing_verifier = _as_dict(existing_contract.get("verifier"))
+ existing_policy = _as_dict(existing_contract.get("policy"))
+ existing_immutable_declared = _as_dict(
+ existing_contract.get("immutable_declared")
+ )
+ existing_checks = existing_verifier.get("checks")
+ backup_restore_automation = build_backup_restore_signal_automation_contract(
+ project_id=project_id or "awoooi",
+ event_type=event_type,
+ lane=lane or "backup_restore_escrow_triage",
+ target=target or "backup_restore",
+ evidence_observations=(
+ existing_checks if isinstance(existing_checks, list) else None
+ ),
+ source_refs=source_refs,
+ receipt_id=row.get("message_id"),
+ run_id=run_id,
+ source_row_exists=bool(row.get("message_id")),
+ contract_origin=contract_origin,
+ contract_persisted_at_send=contract_persisted_at_send,
+ recorded_contract_origin=(
+ str(
+ existing_contract.get("contract_origin")
+ or "legacy_unspecified"
+ )
+ if contract_persisted_at_send
+ else ""
+ ),
+ recorded_policy=existing_policy,
+ recorded_immutable_declared=existing_immutable_declared,
+ agent99_dispatch_receipt=agent99_dispatch_receipt,
+ declared_gates=declared_gates,
+ declared_runtime_write_gate_count=(
+ declared_runtime_write_gate_count
+ ),
+ declared_runtime_write_gate_state=(
+ declared_runtime_write_gate_state
+ ),
+ declared_controlled_playbook_queue=(
+ declared_controlled_playbook_queue
+ ),
+ declared_candidate_only=declared_candidate_only,
+ declared_learning_writeback_status=(
+ declared_learning_writeback_status
+ ),
+ )
+ learning_writeback_refs.update({
+ "status": "blocked_backup_restore_evidence_or_verifier",
+ "work_item_id": backup_restore_automation["work_item"]["work_item_id"],
+ "backup_restore_automation": backup_restore_automation,
+ "production_write_performed": False,
+ "next_action": backup_restore_automation["next_action"],
+ })
+ gates = list(backup_restore_automation["effective_gates"])
+ runtime_write_gate_count = 0
+ candidate_only = True
+ controlled_playbook_queue = False
+ runtime_write_gate_state = "evidence_only"
return {
"message_id": row.get("message_id"),
"run_id": run_id,
@@ -2027,17 +2535,37 @@ def _ai_alert_card_delivery_item(row: Mapping[str, Any]) -> dict[str, Any]:
"send_error": row.get("send_error"),
"provider_message_id": row.get("provider_message_id"),
"triggered_by_state": row.get("triggered_by_state"),
- "event_type": str(alert_card.get("event_type") or ""),
- "lane": str(alert_card.get("lane") or ""),
- "target": str(alert_card.get("target") or ""),
- "gates": alert_card.get("gates") if isinstance(alert_card.get("gates"), list) else [],
+ "event_type": event_type,
+ "lane": lane,
+ "target": target,
+ "gates": gates,
+ "declared_gates": declared_gates,
+ "declared_runtime_write_gate_count": (
+ declared_runtime_write_gate_count
+ ),
+ "declared_runtime_write_allowed": (
+ declared_runtime_write_gate_count > 0
+ ),
+ "declared_runtime_write_gate_state": (
+ declared_runtime_write_gate_state
+ ),
+ "declared_candidate_only": declared_candidate_only,
+ "declared_controlled_playbook_queue": (
+ declared_controlled_playbook_queue
+ ),
+ "declared_learning_writeback_status": (
+ declared_learning_writeback_status
+ ),
+ "effective_policy_projection": (
+ backup_restore_automation.get("effective_policy_projection")
+ if backup_restore_automation
+ else None
+ ),
"runtime_write_gate_count": runtime_write_gate_count,
"runtime_write_allowed": runtime_write_gate_count > 0,
- "candidate_only": bool(alert_card.get("candidate_only")),
- "controlled_playbook_queue": bool(alert_card.get("controlled_playbook_queue")),
- "runtime_write_gate_state": str(
- alert_card.get("runtime_write_gate_state") or "unknown"
- ),
+ "candidate_only": candidate_only,
+ "controlled_playbook_queue": controlled_playbook_queue,
+ "runtime_write_gate_state": runtime_write_gate_state,
"delivery_receipt_readback_required": bool(
alert_card.get("delivery_receipt_readback_required")
),
@@ -2049,6 +2577,7 @@ def _ai_alert_card_delivery_item(row: Mapping[str, Any]) -> dict[str, Any]:
"learning_registry_bindings": _ai_alert_card_learning_registry_bindings(
learning_writeback_refs
),
+ "backup_restore_automation": backup_restore_automation,
"source_refs": source_refs,
"run_state": row.get("run_state"),
"agent_id": row.get("agent_id"),
@@ -2175,7 +2704,9 @@ def _ai_alert_card_learning_registry_readback(
binding.get("ref")
for item in items
for binding in item.get("learning_registry_bindings", [])
- if binding.get("target") == target["target"] and binding.get("ref")
+ if binding.get("target") == target["target"]
+ and binding.get("status") == "ready_for_consumer_context"
+ and binding.get("ref")
]
ready_count = len(refs)
ready_receipt_ref_count += ready_count
diff --git a/apps/api/src/services/signal_producer.py b/apps/api/src/services/signal_producer.py
index 1a4e3ec46..5ec3bb0d9 100644
--- a/apps/api/src/services/signal_producer.py
+++ b/apps/api/src/services/signal_producer.py
@@ -20,6 +20,7 @@ import structlog
from src.core.redis_client import get_redis
from src.core.telemetry import get_trace_context
+from src.models.incident import SignalSource
from src.utils.timezone import now_taipei
logger = structlog.get_logger(__name__)
@@ -32,7 +33,7 @@ SIGNAL_STREAM_MAXLEN = 10000
@dataclass
class SignalData:
"""Signal 資料結構"""
- source: str
+ source: SignalSource
alert_name: str
severity: str
namespace: str
diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py
index 3d0e5d24f..246be013e 100644
--- a/apps/api/src/services/telegram_gateway.py
+++ b/apps/api/src/services/telegram_gateway.py
@@ -38,6 +38,14 @@ from opentelemetry import trace
from src.core.config import settings
from src.core.redis_client import get_redis
+from src.services.agent99_controlled_dispatch_ledger import (
+ read_agent99_dispatch_receipt,
+)
+from src.services.agent99_sre_bridge import build_agent99_sre_alert
+from src.services.approval_action_classifier import (
+ is_executable_repair_approval_action,
+ is_no_action_approval_action,
+)
from src.services.awooop_ansible_audit_service import summarize_ansible_execution
from src.services.awooop_deeplinks import (
AWOOOP_WEB_BASE_URL,
@@ -47,16 +55,15 @@ from src.services.awooop_deeplinks import (
incident_truth_chain_reply_markup,
work_item_url,
)
-from src.services.approval_action_classifier import (
- is_executable_repair_approval_action,
- is_no_action_approval_action,
-)
-from src.services.agent99_sre_bridge import (
- build_agent99_sre_alert,
- dispatch_agent99_sre_alert_with_receipt,
+from src.services.backup_restore_signal_automation import (
+ BACKUP_RESTORE_EVENT_TYPE,
+ build_backup_restore_signal_automation_contract,
)
from src.services.chat_manager import get_chat_manager
-from src.services.operator_outcome import build_operator_outcome, normalize_operator_outcome
+from src.services.operator_outcome import (
+ build_operator_outcome,
+ normalize_operator_outcome,
+)
from src.services.security_interceptor import (
NonceReplayError,
UserNotWhitelistedError,
@@ -486,6 +493,13 @@ def format_aiops_signal_alert_card(text: str) -> str:
target = _aiops_signal_target(text, spec)
severity = _aiops_signal_severity(text, spec)
evidence_lines = _aiops_signal_evidence_lines(text)
+ gate_line = (
+ "├ Gate：controlled_playbook_queue=readback_only / "
+ "runtime_write_gate=0_until_verifier"
+ if spec.event_type == BACKUP_RESTORE_EVENT_TYPE
+ else "├ Gate：controlled_playbook_queue / "
+ "runtime_write_gate=controlled"
+ )
return "\n".join(
[
@@ -500,7 +514,7 @@ def format_aiops_signal_alert_card(text: str) -> str:
"",
"AI 自動化判讀",
f"├ Lane：{html.escape(spec.lane)}",
- "├ Gate：controlled_playbook_queue / runtime_write_gate=controlled",
+ gate_line,
f"└ 下一步：{html.escape(spec.next_step)}",
"",
"Top evidence",
@@ -2332,6 +2346,55 @@ def _ai_automation_alert_card_metadata(text: str) -> dict[str, object] | None:
"delivery_receipt_readback_required": True,
"mirror_source": "legacy_telegram_gateway_outbound_message",
}
+ if event_type == BACKUP_RESTORE_EVENT_TYPE:
+ declared_gates = list(gates)
+ declared_runtime_write_gate_count = (
+ 1 if runtime_write_gate_state == "controlled" else 0
+ )
+ declared_candidate_only = "candidate_only" in declared_gates
+ declared_controlled_playbook_queue = (
+ "controlled_playbook_queue" in declared_gates
+ )
+ contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ event_type=event_type,
+ lane=lane,
+ target=values.get("Target", "") or "backup_restore",
+ evidence_text=text,
+ # This envelope is assembled before the outbound row is committed.
+ # The operator readback upgrades the origin to ``persisted_at_send``
+ # only after it observes this contract in the durable source row.
+ contract_origin="generated_for_send",
+ contract_persisted_at_send=False,
+ declared_gates=declared_gates,
+ declared_runtime_write_gate_count=(
+ declared_runtime_write_gate_count
+ ),
+ declared_runtime_write_gate_state=runtime_write_gate_state,
+ declared_controlled_playbook_queue=(
+ declared_controlled_playbook_queue
+ ),
+ declared_candidate_only=declared_candidate_only,
+ )
+ metadata.update({
+ "gates": list(contract["effective_gates"]),
+ "declared_gates": declared_gates,
+ "declared_runtime_write_gate_count": (
+ declared_runtime_write_gate_count
+ ),
+ "declared_runtime_write_gate_state": runtime_write_gate_state,
+ "declared_candidate_only": declared_candidate_only,
+ "declared_controlled_playbook_queue": (
+ declared_controlled_playbook_queue
+ ),
+ "candidate_only": True,
+ "controlled_playbook_queue": False,
+ "backup_readback_candidate": True,
+ "runtime_write_gate_count": 0,
+ "runtime_write_gate_state": "evidence_only",
+ "no_false_green": True,
+ "backup_restore_automation": contract,
+ })
return metadata
@@ -2484,6 +2547,9 @@ def _agent99_alert_from_actionable_outbound(
payload["awoooi"]["incidentId"] = incident_id
payload["awoooi"]["commitSha"] = commit_sha[:40]
payload["awoooi"]["cardSchema"] = card_schema
+ backup_automation = metadata.get("backup_restore_automation")
+ if isinstance(backup_automation, dict):
+ payload["awoooi"]["backupRestoreAutomation"] = backup_automation
return payload
@@ -2504,53 +2570,91 @@ async def _mirror_ai_automation_alert_card_to_agent99(
*,
source: str = "telegram_gateway",
) -> dict[str, object] | None:
+ """Project the durable Agent99 dispatch receipt into Telegram metadata.
+
+ Telegram is deliberately not a dispatch owner. The webhook/incident lane
+ reserves and dispatches exactly once; outbound cards can only read and
+ project the receipt linked to that incident.
+ """
+
payload = _agent99_alert_from_actionable_outbound(text, source=source)
if not payload:
return None
alert_id = str(payload.get("id") or "")
- try:
- dispatch_receipt = await asyncio.to_thread(
- dispatch_agent99_sre_alert_with_receipt,
- payload,
- )
- except Exception as exc:
+ awoooi = payload.get("awoooi") if isinstance(payload.get("awoooi"), dict) else {}
+ incident_id = str(awoooi.get("incidentId") or "").strip()
+ if not incident_id:
logger.warning(
- "telegram_agent99_alert_card_mirror_failed_fail_open",
+ "telegram_agent99_receipt_projection_unbound_fail_closed",
alert_id=alert_id,
kind=payload.get("kind"),
- error=str(exc),
)
return {
- "schema_version": "telegram_agent99_dispatch_receipt_v1",
- "status": "dispatch_failed",
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "receipt_unbound_fail_closed",
"alert_id": alert_id,
"kind": str(payload.get("kind") or "monitoring_alert"),
"suggested_mode": str(payload.get("suggestedMode") or "Status"),
"accepted": False,
"inbox_triggered": False,
- "error_type": type(exc).__name__,
+ "dispatch_performed": False,
+ "runtime_closure_verified": False,
}
- logger.info(
- "telegram_agent99_alert_card_mirrored",
- alert_id=alert_id,
- kind=payload.get("kind"),
- suggested_mode=payload.get("suggestedMode"),
- dispatch_result=dispatch_receipt.get("status"),
+ correlated_receipt = await read_agent99_dispatch_receipt(
+ project_id="awoooi",
+ incident_id=incident_id,
+ )
+ dispatch_receipt = (
+ correlated_receipt.get("dispatch_receipt")
+ if isinstance(correlated_receipt, dict)
+ and isinstance(correlated_receipt.get("dispatch_receipt"), dict)
+ else {}
+ )
+ identity = (
+ correlated_receipt.get("identity")
+ if isinstance(correlated_receipt, dict)
+ and isinstance(correlated_receipt.get("identity"), dict)
+ else {}
+ )
+ receipt_found = bool(correlated_receipt)
+ logger.info(
+ "telegram_agent99_dispatch_receipt_projected",
+ alert_id=alert_id,
+ incident_id=incident_id,
+ kind=payload.get("kind"),
+ receipt_found=receipt_found,
+ run_id=identity.get("run_id"),
)
- awoooi = payload.get("awoooi") if isinstance(payload.get("awoooi"), dict) else {}
return {
- "schema_version": "telegram_agent99_dispatch_receipt_v1",
- "status": str(dispatch_receipt.get("status") or "unknown"),
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": (
+ str(correlated_receipt.get("status") or "receipt_available")
+ if isinstance(correlated_receipt, dict)
+ else "receipt_pending_fail_closed"
+ ),
"transport": str(dispatch_receipt.get("transport") or "unknown"),
"alert_id": alert_id,
"kind": str(payload.get("kind") or "monitoring_alert"),
"suggested_mode": str(payload.get("suggestedMode") or "Status"),
"accepted": bool(dispatch_receipt.get("accepted") is True),
"inbox_triggered": bool(dispatch_receipt.get("inbox_triggered") is True),
- "incident_id": str(awoooi.get("incidentId") or ""),
+ "incident_id": incident_id,
"commit_sha": str(awoooi.get("commitSha") or ""),
+ "run_id": str(identity.get("run_id") or ""),
+ "trace_id": str(identity.get("trace_id") or ""),
+ "work_item_id": str(identity.get("work_item_id") or ""),
+ "idempotency_key": str(identity.get("idempotency_key") or ""),
+ "dispatch_performed": False,
+ "receipt_persisted": bool(
+ isinstance(correlated_receipt, dict)
+ and correlated_receipt.get("receipt_persisted") is True
+ ),
+ "runtime_closure_verified": bool(
+ isinstance(correlated_receipt, dict)
+ and correlated_receipt.get("runtime_closure_verified") is True
+ ),
"stores_raw_response": False,
}
diff --git a/apps/api/src/utils/incident_converter.py b/apps/api/src/utils/incident_converter.py
index fed1aa383..b71fa9e6b 100644
--- a/apps/api/src/utils/incident_converter.py
+++ b/apps/api/src/utils/incident_converter.py
@@ -27,6 +27,7 @@ from src.models.incident import (
IncidentStatus,
Severity,
Signal,
+ parse_signal_source,
)
logger = structlog.get_logger(__name__)
@@ -83,12 +84,10 @@ def brain_to_local(brain_incident: Any) -> Incident:
except ValueError:
sig_severity = Severity.P2
- # BrainSignal.source 是 str，LocalSignal.source 是 Literal
- # 使用 getattr 確保不因 BrainSignal 欄位增減而崩潰
- source = getattr(brain_signal, "source", "alertmanager")
- valid_sources = {"prometheus", "signoz", "alertmanager", "manual", "telegram"}
- if source not in valid_sources:
- source = "alertmanager"
+ # Preserve repo-owned producer provenance across the package
+ # boundary. Unknown values fail closed instead of being mislabeled
+ # as Alertmanager and later poisoning the durable read model.
+ source = parse_signal_source(getattr(brain_signal, "source", ""))
local_signals.append(
Signal(
diff --git a/apps/api/src/workers/signal_worker.py b/apps/api/src/workers/signal_worker.py
index f6846e7bf..bee848313 100644
--- a/apps/api/src/workers/signal_worker.py
+++ b/apps/api/src/workers/signal_worker.py
@@ -620,6 +620,7 @@ async def _main() -> None:
# Initialize and start worker
worker = await init_signal_worker()
ansible_candidate_backfill_task: asyncio.Task[Any] | None = None
+ backup_restore_legacy_backfill_task: asyncio.Task[Any] | None = None
if settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER:
from src.jobs.awooop_ansible_candidate_backfill_job import (
run_awooop_ansible_candidate_backfill_loop,
@@ -634,6 +635,33 @@ async def _main() -> None:
interval_seconds=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_INTERVAL_SECONDS,
batch_limit=settings.AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_BATCH_LIMIT,
)
+ if settings.ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL:
+ from src.jobs.backup_restore_legacy_backfill_job import (
+ build_backup_restore_backfill_relay_preflight,
+ run_backup_restore_legacy_backfill_loop,
+ )
+
+ relay_preflight = build_backup_restore_backfill_relay_preflight()
+ if relay_preflight["ready"] is True:
+ backup_restore_legacy_backfill_task = asyncio.create_task(
+ run_backup_restore_legacy_backfill_loop(),
+ name="run_backup_restore_legacy_backfill_loop",
+ )
+ logger.info(
+ "signal_worker_backup_restore_legacy_backfill_loop_started",
+ batch_limit=settings.BACKUP_RESTORE_LEGACY_BACKFILL_BATCH_LIMIT,
+ max_rows=settings.BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS,
+ auth_mode=relay_preflight["auth_mode"],
+ telegram_dispatch_performed=False,
+ )
+ else:
+ logger.warning(
+ "signal_worker_backup_restore_legacy_backfill_loop_skipped_fail_closed",
+ status=relay_preflight["status"],
+ auth_mode=relay_preflight["auth_mode"],
+ endpoint_class=relay_preflight["endpoint_class"],
+ telegram_dispatch_performed=False,
+ )
# Setup graceful shutdown
shutdown_event = asyncio.Event()
@@ -665,6 +693,12 @@ async def _main() -> None:
await ansible_candidate_backfill_task
except asyncio.CancelledError:
pass
+ if backup_restore_legacy_backfill_task is not None:
+ backup_restore_legacy_backfill_task.cancel()
+ try:
+ await backup_restore_legacy_backfill_task
+ except asyncio.CancelledError:
+ pass
await worker.stop()
await close_worker_redis_pool() # 關閉 Worker 專屬連線
await close_redis_pool()
diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py
index 59881bfe8..f55704fa0 100644
--- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py
+++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py
@@ -3,6 +3,8 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
+CONTROL_PLANE = ROOT / "agent99-control-plane.ps1"
+SRE_RELAY = ROOT / "agent99-sre-alert-relay.ps1"
def test_alertmanager_groups_before_agent99_dispatch() -> None:
@@ -12,22 +14,40 @@ def test_alertmanager_groups_before_agent99_dispatch() -> None:
grouped_return_at = source.index(
"if grouping_result and grouping_result.is_grouped", grouping_at
)
- dispatch_at = source.index("bridge_alertmanager_to_agent99", grouped_return_at)
+ dispatch_at = source.index("_try_auto_repair_background,", grouped_return_at)
assert grouping_at < grouped_return_at < dispatch_at
-def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None:
+def test_agent99_bridge_reserves_and_locks_before_transport() -> None:
source = (ROOT / "apps/api/src/services/agent99_sre_bridge.py").read_text(
encoding="utf-8"
)
bridge_at = source.index("async def bridge_alertmanager_to_agent99")
- lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at)
+ reserve_at = source.index("ledger.reserve", bridge_at)
+ lock_at = source.index("acquire_agent99_sre_single_flight", reserve_at)
dispatch_at = source.index(
"dispatch_agent99_sre_alert_with_receipt, payload", lock_at
)
- assert lock_at < dispatch_at
+ assert reserve_at < lock_at < dispatch_at
+
+
+def test_telegram_is_receipt_projection_not_agent99_dispatch_owner() -> None:
+ source = (ROOT / "apps/api/src/services/telegram_gateway.py").read_text(
+ encoding="utf-8"
+ )
+ projection_at = source.index(
+ "async def _mirror_ai_automation_alert_card_to_agent99"
+ )
+ projection_end = source.index(
+ "def _schedule_agent99_ai_alert_card_mirror", projection_at
+ )
+ projection = source[projection_at:projection_end]
+
+ assert "read_agent99_dispatch_receipt" in projection
+ assert "dispatch_agent99_sre_alert_with_receipt" not in projection
+ assert '"dispatch_performed": False' in projection
def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
@@ -61,6 +81,62 @@ def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None:
assert 'reason = "duplicate_single_flight"' in source
assert "severity_escalation_dispatch" in source
assert "correlationKey = $correlationKey" in source
+ assert "cold[-_ ]start|reboot[-_ ]auto[-_ ]recovery" in source
+ for field in (
+ "automationRunId",
+ "traceId",
+ "workItemId",
+ "idempotencyKey",
+ "executionGeneration",
+ "incidentId",
+ ):
+ assert field in source
+ assert "controlled_dispatch_identity_missing" in source
+
+ control_plane = CONTROL_PLANE.read_text(encoding="utf-8")
+ assert "controlled_dispatch_identity_missing" in control_plane
+ assert "-Name automationRunId" in control_plane
+ assert "-Name traceId" in control_plane
+ assert "-Name workItemId" in control_plane
+ for evidence_ref in (
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+ ):
+ assert evidence_ref in control_plane
+
+
+def test_running_claim_is_reconcile_only_and_pending_retry_is_not_before() -> None:
+ source = (
+ ROOT
+ / "apps/api/src/services/agent99_controlled_dispatch_ledger.py"
+ ).read_text(encoding="utf-8")
+ reserve = source[
+ source.index("async def reserve"):source.index(
+ "async def defer_claim_retryable"
+ )
+ ]
+
+ assert 'AwoooPRunState.state == "pending"' in reserve
+ assert "AwoooPRunState.next_attempt_at.is_(None)" in reserve
+ assert "AwoooPRunState.next_attempt_at <= now" in reserve
+ assert "AwoooPRunState.lease_until < now" not in reserve
+ assert "worker_id=claim_token" in reserve
+ assert "AwoooPRunState.attempt_count < AwoooPRunState.max_attempts" in reserve
+ assert "reconciliation-only" in reserve
+
+
+def test_agent99_relay_exposes_authenticated_public_safe_outcome_readback() -> None:
+ source = SRE_RELAY.read_text(encoding="utf-8")
+
+ assert "function Get-AgentOutcomeReadback" in source
+ assert '$context.Request.HttpMethod -eq "GET"' in source
+ assert "outcome_readback_auth_not_configured" in source
+ assert 'Headers["X-Agent99-Relay-Token"]' in source
+ assert "storesRawEvidence = $false" in source
def test_telegram_ingress_writes_same_structured_route_contract() -> None:
diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py
index 01d07e138..fe9994589 100644
--- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py
+++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py
@@ -5,6 +5,7 @@ from pathlib import Path
_REPO_ROOT = Path(__file__).resolve().parents[3]
_CONTROL_PLANE = _REPO_ROOT / "agent99-control-plane.ps1"
_BOOTSTRAP = _REPO_ROOT / "agent99-bootstrap.ps1"
+_SRE_INBOX = _REPO_ROOT / "agent99-sre-alert-inbox.ps1"
def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None:
@@ -64,3 +65,68 @@ def test_agent99_outcome_contract_has_runtime_self_test() -> None:
assert '$failedPostCondition.state -eq "degraded"' in source
assert '$transportFailed.state -eq "failed"' in source
assert '$securityCandidate.state -eq "candidate_ready"' in source
+
+
+def test_agent99_posts_fenced_outcome_to_production_ingestion() -> None:
+ source = _CONTROL_PLANE.read_text(encoding="utf-8")
+ inbox = _SRE_INBOX.read_text(encoding="utf-8")
+
+ assert "function Send-AgentOutcomeWriteback" in source
+ assert "/api/v1/webhooks/agent99/outcomes" in source
+ assert '"X-Agent99-Relay-Token" = $token' in source
+ assert "$attempt -le 3" in source
+ assert "agent99_outcome_receipt_id" in source
+ assert "post_verifier_evidence_ref" in source
+ assert "source_event_evidence_ref" in source
+ for field in (
+ "projectId",
+ "sourceFingerprint",
+ "dispatchRouteId",
+ "singleFlightKey",
+ "lockOwner",
+ "canonicalDigest",
+ "automationRunId",
+ "traceId",
+ "workItemId",
+ "idempotencyKey",
+ "executionGeneration",
+ "incidentId",
+ ):
+ assert field in inbox
+ assert field in source
+
+
+def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None:
+ source = _CONTROL_PLANE.read_text(encoding="utf-8")
+ readback = source[
+ source.index("function Test-AgentBackupProtectionReceipts") :
+ source.index("function Invoke-AgentBackupReadbackContractSelfTest")
+ ]
+
+ assert "[switch]$SelfTestBackupReadbackContract" in source
+ assert "function Convert-AgentBackupProtectionReadback" in source
+ for metric in (
+ "awoooi_backup_offsite_configured",
+ "awoooi_backup_offsite_fresh",
+ "awoooi_backup_offsite_remote_verify_ok",
+ "awoooi_backup_credential_escrow_fresh",
+ "awoooi_backup_dr_credential_escrow_missing_count",
+ ):
+ assert metric in source
+ assert "stat -c '__PROTECTION_MTIME__=%Y'" in readback
+ assert "grep -E" in readback
+ assert "backupRunPerformed = $false" in source
+ assert "restoreRunPerformed = $false" in source
+ assert "markerWritePerformed = $false" in source
+ assert '"offsite_verify_evidence_ref"' in source
+ assert '"escrow_evidence_ref"' in source
+ assert "protectionMaxAgeMinutes" in source
+ assert "$evidenceAgeMinutes -ge -5" in source
+ assert "$evidenceAgeMinutes -le" in source
+ assert "-eq $configured.Count" in source
+ assert "-eq $offsiteFresh.Count" in source
+ assert "-eq $remoteVerify.Count" in source
+ assert "-not $staleResult.offsiteVerify.ok" in source
+ assert "-not $partialResult.offsiteVerify.ok" in source
+ for mutation in ("rclone ", "rsync ", " rm ", "restore ", "Set-Content"):
+ assert mutation not in readback
diff --git a/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py
new file mode 100644
index 000000000..bbca57c5e
--- /dev/null
+++ b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py
@@ -0,0 +1,119 @@
+from __future__ import annotations
+
+import pytest
+
+from src.services.agent99_controlled_dispatch_ledger import (
+ PostgresAgent99DispatchLedger,
+ build_agent99_dispatch_identity,
+ build_agent99_dispatch_receipt_envelope,
+ parse_agent99_dispatch_identity,
+)
+
+
+def test_dispatch_identity_parser_rejects_cross_run_projection() -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="fp-cold-start",
+ route_id="agent99:host_recovery:Recover",
+ )
+ public = identity.public_dict()
+
+ assert parse_agent99_dispatch_identity(public) == identity
+ public["trace_id"] = "00-00000000000000000000000000000000-0000000000000000-01"
+ with pytest.raises(ValueError, match="identity_mismatch:trace_id"):
+ parse_agent99_dispatch_identity(public)
+
+
+def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="fp-cold-start",
+ route_id="agent99:host_recovery:Recover",
+ )
+
+ envelope = build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt={
+ "status": "accepted_inbox_pending",
+ "accepted": True,
+ "inbox_triggered": False,
+ "delivery_certainty": "unknown",
+ },
+ controlled_apply_authorized=True,
+ )
+
+ assert envelope["status"] == "dispatch_delivery_unknown_reconcile_only"
+ assert envelope["dispatch_promoted"] is False
+ assert envelope["controlled_apply_authorized"] is False
+ assert envelope["retry_policy"]["reconcile_only"] is True
+
+
+@pytest.mark.asyncio
+async def test_backup_verifier_requires_all_six_dr_evidence_refs() -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-BACKUP",
+ source_fingerprint="fp-backup",
+ route_id="agent99:backup_health:BackupCheck",
+ )
+ ledger = PostgresAgent99DispatchLedger()
+
+ result = await ledger.record_verifier(
+ identity=identity,
+ outcome_receipt={
+ "identity": identity.public_dict(),
+ "automationRunId": str(identity.run_id),
+ "traceId": identity.trace_id,
+ "workItemId": identity.work_item_id,
+ "mode": "BackupCheck",
+ "outcome": {
+ "identity": identity.public_dict(),
+ "schemaVersion": "agent99_outcome_contract_v1",
+ "state": "resolved",
+ "transportOk": True,
+ "verifierPassed": True,
+ "sourceEventResolved": True,
+ },
+ },
+ evidence_refs={
+ "agent99_outcome_receipt_id": "agent99:outcome:1",
+ "post_verifier_evidence_ref": "agent99:BackupCheck:verified",
+ "source_event_evidence_ref": "incident:source-resolved",
+ },
+ )
+
+ assert result["status"] == "verifier_evidence_missing_fail_closed"
+ assert set(result["missing_evidence_refs"]) == {
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+ }
+
+
+@pytest.mark.asyncio
+async def test_backup_learning_requires_dr_scorecard_ack() -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-BACKUP",
+ source_fingerprint="fp-backup",
+ route_id="agent99:backup_health:BackupCheck",
+ )
+ ledger = PostgresAgent99DispatchLedger()
+
+ result = await ledger.record_learning_writeback(
+ identity=identity,
+ receipt_refs={
+ "incident_closure_receipt_id": "incident:1",
+ "telegram_lifecycle_receipt_id": "telegram:1",
+ "km_writeback_ack_id": "km:1",
+ "playbook_trust_writeback_ack_id": "playbook:1",
+ },
+ )
+
+ assert result["status"] == "learning_writeback_incomplete_fail_closed"
+ assert result["missing_receipts"] == ["dr_scorecard_writeback_ack_id"]
diff --git a/apps/api/tests/test_agent99_controlled_dispatch_p1.py b/apps/api/tests/test_agent99_controlled_dispatch_p1.py
new file mode 100644
index 000000000..f7bfc4b73
--- /dev/null
+++ b/apps/api/tests/test_agent99_controlled_dispatch_p1.py
@@ -0,0 +1,899 @@
+from __future__ import annotations
+
+import json
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from fastapi import HTTPException
+
+from src.api.v1 import webhooks
+from src.db.models import IncidentRecord
+from src.models.incident import IncidentStatus
+from src.services import agent99_controlled_dispatch_ledger as ledger_module
+from src.services import agent99_outcome_ingestion as ingestion_module
+from src.services.agent99_controlled_dispatch_ledger import (
+ PostgresAgent99DispatchLedger,
+ build_agent99_dispatch_identity,
+ build_agent99_dispatch_receipt_envelope,
+ parse_agent99_dispatch_identity,
+ record_agent99_learning_writeback,
+)
+from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
+
+
+def _identity():
+ return build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="cold-start-source-fingerprint",
+ route_id="agent99_cold_start_recovery",
+ execution_generation="1",
+ work_item_id="agent99-dispatch:awoooi:INC-20260711-11C751:recovery",
+ )
+
+
+def _evidence_refs() -> dict[str, str]:
+ return {
+ "agent99_outcome_receipt_id": "agent99-outcome:queue-1",
+ "post_verifier_evidence_ref": "evidence:cold-start-scorecard:1",
+ "source_event_evidence_ref": "alert:fingerprint:resolved:1",
+ }
+
+
+def _outcome(identity, *, transport_ok: bool = True) -> dict:
+ return {
+ "identity": identity.public_dict(),
+ "controlledApply": True,
+ "outcome": {
+ "identity": identity.public_dict(),
+ "schemaVersion": "agent99_outcome_contract_v1",
+ "state": "resolved",
+ "transportOk": transport_ok,
+ "verifierPassed": True,
+ "sourceEventResolved": True,
+ "verifiedAt": "2026-07-11T20:00:00+08:00",
+ },
+ }
+
+
+def test_pending_inbox_never_promotes_or_authorizes() -> None:
+ receipt = build_agent99_dispatch_receipt_envelope(
+ identity=_identity(),
+ dispatch_receipt={
+ "accepted": True,
+ "inbox_triggered": False,
+ "delivery_certainty": "unknown",
+ },
+ controlled_apply_authorized=True,
+ )
+
+ assert receipt["dispatch_accepted"] is True
+ assert receipt["inbox_triggered"] is False
+ assert receipt["dispatch_promoted"] is False
+ assert receipt["controlled_apply_authorized"] is False
+ assert receipt["runtime_execution_authorized"] is False
+ assert receipt["verifier"]["status"] == "blocked_dispatch_not_promoted"
+ assert receipt["runtime_closure_verified"] is False
+
+
+def test_complete_identity_is_required_and_recomputed() -> None:
+ identity = _identity()
+
+ assert parse_agent99_dispatch_identity(identity.public_dict()) == identity
+ missing = identity.public_dict()
+ missing["trace_id"] = ""
+ with pytest.raises(ValueError, match="identity_fields_required:trace_id"):
+ parse_agent99_dispatch_identity(missing)
+ mismatched = identity.public_dict()
+ mismatched["run_id"] = "00000000-0000-0000-0000-000000000000"
+ with pytest.raises(ValueError, match="identity_mismatch:run_id"):
+ parse_agent99_dispatch_identity(mismatched)
+
+
+def test_approval_identity_cannot_alias_run_or_idempotency() -> None:
+ common = {
+ "project_id": "awoooi",
+ "incident_id": "INC-20260711-11C751",
+ "source_fingerprint": "cold-start-source-fingerprint",
+ "route_id": "agent99_cold_start_recovery",
+ "execution_generation": "1",
+ "work_item_id": "agent99-dispatch:awoooi:11c751",
+ }
+ first = build_agent99_dispatch_identity(
+ **common,
+ approval_id="00000000-0000-0000-0000-000000000751",
+ )
+ second = build_agent99_dispatch_identity(
+ **common,
+ approval_id="00000000-0000-0000-0000-000000000752",
+ )
+
+ assert first.run_id != second.run_id
+ assert first.idempotency_key != second.idempotency_key
+ assert first.public_dict()["canonical_digest"] != (
+ second.public_dict()["canonical_digest"]
+ )
+ assert first.single_flight_key == second.single_flight_key
+
+
+def test_terminal_incident_status_uses_database_enum_member() -> None:
+ status_type = IncidentRecord.__table__.c.status.type
+
+ assert status_type.enum_class is IncidentStatus
+ assert IncidentStatus.RESOLVED.name in status_type.enums
+ assert IncidentStatus.RESOLVED.value == "resolved"
+
+
+class _ScalarResult:
+ def __init__(self, value=None, row=None) -> None:
+ self.value = value
+ self.row = row
+
+ def scalar_one_or_none(self):
+ return self.value
+
+ def one_or_none(self):
+ return self.row
+
+
+class _Context:
+ def __init__(self, db) -> None:
+ self.db = db
+
+ async def __aenter__(self):
+ return self.db
+
+ async def __aexit__(self, *_args):
+ return False
+
+
+@pytest.mark.asyncio
+async def test_reservation_claim_tokens_are_unique_and_running_is_reconcile_only(
+ monkeypatch,
+) -> None:
+ identity = _identity()
+ claim_statements = []
+
+ class ReserveDB:
+ def __init__(self) -> None:
+ self.call = 0
+
+ async def execute(self, statement):
+ self.call += 1
+ if self.call == 3:
+ claim_statements.append(statement)
+ return _ScalarResult(identity.run_id)
+ if self.call == 5:
+ return _ScalarResult(
+ row=SimpleNamespace(
+ state="running",
+ error_detail=json.dumps({"identity": identity.public_dict()}),
+ )
+ )
+ return _ScalarResult()
+
+ databases = [ReserveDB(), ReserveDB()]
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(databases.pop(0)),
+ )
+ ledger = PostgresAgent99DispatchLedger()
+ payload = {
+ "kind": "host_recovery",
+ "suggestedMode": "Recover",
+ "controlledApply": True,
+ "awoooi": {"targetResource": "cold-start-gate"},
+ }
+
+ first = await ledger.reserve(identity=identity, payload=payload)
+ second = await ledger.reserve(identity=identity, payload=payload)
+
+ assert first["claimed"] is True
+ assert second["claimed"] is True
+ assert first["claim_token"] != second["claim_token"]
+ claim_sql = str(claim_statements[0])
+ assert "awooop_run_state.state = :state_1" in claim_sql
+ assert "awooop_run_state.next_attempt_at IS NULL" in claim_sql
+ assert "awooop_run_state.next_attempt_at <=" in claim_sql
+ assert "awooop_run_state.lease_until <" not in claim_sql
+
+
+@pytest.mark.asyncio
+async def test_stale_worker_cannot_complete_newer_claim(monkeypatch) -> None:
+ identity = _identity()
+
+ class FencedDB:
+ async def execute(self, _statement):
+ return _ScalarResult(None)
+
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(FencedDB()),
+ )
+ result = await PostgresAgent99DispatchLedger().complete(
+ identity=identity,
+ claim_token="stale-claim-token",
+ dispatch_receipt={
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ controlled_apply_authorized=True,
+ )
+
+ assert result["status"] == "claim_fenced_no_write"
+ assert result["receipt_persisted"] is False
+ assert result["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_redis_lock_collision_returns_run_to_retryable_pending(
+ monkeypatch,
+) -> None:
+ identity = _identity()
+ initial = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "identity": identity.public_dict(),
+ "runtime_closure_verified": False,
+ }
+
+ class RetryDB:
+ def __init__(self) -> None:
+ self.call = 0
+
+ async def execute(self, _statement):
+ self.call += 1
+ if self.call == 1:
+ return _ScalarResult(
+ row=SimpleNamespace(
+ state="running",
+ worker_id="claim-2",
+ attempt_count=1,
+ max_attempts=3,
+ error_detail=json.dumps(initial),
+ )
+ )
+ if self.call == 2:
+ return _ScalarResult(identity.run_id)
+ return _ScalarResult()
+
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(RetryDB()),
+ )
+ result = await PostgresAgent99DispatchLedger().defer_claim_retryable(
+ identity=identity,
+ claim_token="claim-2",
+ reason="single_flight_duplicate",
+ retry_after_seconds=600,
+ )
+
+ assert result["status"] == "dispatch_lock_retryable"
+ assert result["receipt_persisted"] is True
+ assert result["retry_policy"]["retry_same_generation"] is True
+ assert result["controlled_apply_authorized"] is False
+ assert result["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_verifier_requires_transport_and_all_evidence(monkeypatch) -> None:
+ identity = _identity()
+ ledger = PostgresAgent99DispatchLedger()
+
+ missing = await ledger.record_verifier(
+ identity=identity,
+ outcome_receipt=_outcome(identity),
+ evidence_refs={"agent99_outcome_receipt_id": "one-ref-only"},
+ )
+ assert missing["status"] == "verifier_evidence_missing_fail_closed"
+ assert missing["receipt_persisted"] is False
+
+ promoted = build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt={
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ controlled_apply_authorized=True,
+ )
+
+ class VerifierDB:
+ def __init__(self) -> None:
+ self.call = 0
+
+ async def execute(self, _statement):
+ self.call += 1
+ if self.call == 1:
+ return _ScalarResult(
+ row=SimpleNamespace(
+ state="waiting_tool",
+ error_detail=json.dumps(promoted),
+ )
+ )
+ if self.call == 2:
+ return _ScalarResult(identity.run_id)
+ if self.call == 4:
+ return _ScalarResult(identity.incident_id)
+ if self.call in {5, 6}:
+ return _ScalarResult(identity.run_id)
+ return _ScalarResult()
+
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(VerifierDB()),
+ )
+ failed = await ledger.record_verifier(
+ identity=identity,
+ outcome_receipt=_outcome(identity, transport_ok=False),
+ evidence_refs=_evidence_refs(),
+ )
+
+ assert failed["post_verifier_passed"] is False
+ assert failed["verifier"]["transport_ok"] is False
+ assert failed["runtime_closure_verified"] is False
+
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(VerifierDB()),
+ )
+ passed = await ledger.record_verifier(
+ identity=identity,
+ outcome_receipt=_outcome(identity, transport_ok=True),
+ evidence_refs=_evidence_refs(),
+ )
+ assert passed["post_verifier_passed"] is True
+ assert passed["verifier"]["transport_ok"] is True
+ assert passed["status"] == "verifier_passed_learning_writeback_pending"
+ assert passed["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_learning_writeback_is_the_only_same_run_terminal_step(
+ monkeypatch,
+) -> None:
+ identity = _identity()
+ ledger = PostgresAgent99DispatchLedger()
+ verifier_passed = build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt={
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ controlled_apply_authorized=True,
+ )
+ verifier_passed.update({
+ "status": "verifier_passed_learning_writeback_pending",
+ "post_verifier_passed": True,
+ "runtime_closure_verified": False,
+ "closure_complete": False,
+ "learning_writeback": {
+ "status": "pending_writeback",
+ "receipt_refs": {"km_writeback_ack_id": "km-only"},
+ },
+ })
+
+ class IncompleteCheckpointDB:
+ def __init__(self) -> None:
+ self.call = 0
+
+ async def execute(self, _statement):
+ self.call += 1
+ if self.call == 1:
+ return _ScalarResult(
+ row=SimpleNamespace(
+ state="waiting_tool",
+ error_detail=json.dumps(verifier_passed),
+ )
+ )
+ raise AssertionError("terminal tables must not be touched")
+
+ incomplete_db = IncompleteCheckpointDB()
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(incomplete_db),
+ )
+ incomplete = await ledger.record_learning_writeback(
+ identity=identity,
+ receipt_refs={
+ "telegram_lifecycle_receipt_id": "telegram:fake",
+ "km_writeback_ack_id": "knowledge:fake",
+ "playbook_trust_writeback_ack_id": "playbook:fake",
+ },
+ )
+ assert incomplete["status"] == (
+ "learning_writeback_checkpoint_incomplete_fail_closed"
+ )
+ assert incomplete["runtime_closure_verified"] is False
+ assert incomplete_db.call == 1
+
+ checkpoint_refs = {
+ "telegram_lifecycle_receipt_id": "telegram-1",
+ "km_writeback_ack_id": "km-1",
+ "playbook_trust_writeback_ack_id": "playbook-1",
+ }
+ verifier_passed["learning_writeback"]["receipt_refs"] = checkpoint_refs
+
+ class LearningDB:
+ def __init__(self) -> None:
+ self.call = 0
+
+ async def execute(self, _statement):
+ self.call += 1
+ if self.call == 1:
+ return _ScalarResult(
+ row=SimpleNamespace(
+ state="waiting_tool",
+ error_detail=json.dumps(verifier_passed),
+ )
+ )
+ if self.call == 2:
+ return _ScalarResult(SimpleNamespace(outcome={}))
+ if self.call == 4:
+ return _ScalarResult(identity.incident_id)
+ if self.call in {5, 6}:
+ return _ScalarResult(identity.run_id)
+ return _ScalarResult()
+
+ monkeypatch.setattr(
+ ledger_module,
+ "get_db_context",
+ lambda _project_id: _Context(LearningDB()),
+ )
+ closed = await ledger.record_learning_writeback(
+ identity=identity,
+ receipt_refs=checkpoint_refs,
+ )
+
+ assert closed["status"] == "closed_verified_learning_written"
+ assert closed["runtime_closure_verified"] is True
+ assert closed["closure_complete"] is True
+
+
+@pytest.mark.asyncio
+async def test_terminal_post_commit_projection_is_project_scoped(
+ monkeypatch,
+) -> None:
+ identity = _identity()
+ incident = SimpleNamespace(incident_id=identity.incident_id)
+ episodic = AsyncMock(return_value=incident)
+ working = AsyncMock(return_value=True)
+
+ monkeypatch.setattr(
+ ledger_module._ledger,
+ "record_learning_writeback",
+ AsyncMock(
+ return_value={
+ "status": "closed_verified_learning_written",
+ "postgres_terminal_bundle_committed": True,
+ "redis_projection_pending": True,
+ "runtime_closure_verified": True,
+ }
+ ),
+ )
+ monkeypatch.setattr(
+ "src.services.incident_service.get_incident_service",
+ lambda: SimpleNamespace(
+ get_from_episodic_memory=episodic,
+ save_to_working_memory=working,
+ ),
+ )
+
+ result = await record_agent99_learning_writeback(
+ identity=identity,
+ receipt_refs={
+ "telegram_lifecycle_receipt_id": "telegram:1",
+ "km_writeback_ack_id": "knowledge:1",
+ "playbook_trust_writeback_ack_id": "playbook:1",
+ },
+ )
+
+ episodic.assert_awaited_once_with(
+ identity.incident_id,
+ project_id=identity.project_id,
+ )
+ working.assert_awaited_once_with(incident)
+ assert result["redis_projection_acknowledged"] is True
+ assert result["redis_projection_pending"] is False
+
+
+@pytest.mark.asyncio
+async def test_outcome_ingestion_ignores_untrusted_refs_and_cannot_terminalize(
+ monkeypatch,
+) -> None:
+ identity = _identity()
+ calls: list[tuple[str, str]] = []
+
+ async def verifier(**kwargs):
+ calls.append(("verifier", str(kwargs["identity"].run_id)))
+ return {
+ "status": "verifier_passed_learning_writeback_pending",
+ "receipt_persisted": True,
+ "post_verifier_passed": True,
+ "runtime_closure_verified": False,
+ }
+
+ terminal = AsyncMock(side_effect=AssertionError("terminal bypass invoked"))
+ latest = AsyncMock(side_effect=AssertionError("terminal readback invoked"))
+ monkeypatch.setattr(ingestion_module, "record_agent99_verifier_receipt", verifier)
+ monkeypatch.setattr(
+ ingestion_module,
+ "record_agent99_learning_writeback",
+ terminal,
+ raising=False,
+ )
+ monkeypatch.setattr(
+ ingestion_module,
+ "read_agent99_dispatch_receipt",
+ latest,
+ raising=False,
+ )
+ result = await ingestion_module.ingest_agent99_outcome_receipt({
+ "identity": identity.public_dict(),
+ "outcome_receipt": _outcome(identity),
+ "evidence_refs": _evidence_refs(),
+ "learning_receipt_refs": {
+ "telegram_lifecycle_receipt_id": "telegram:fake",
+ "km_writeback_ack_id": "knowledge:fake",
+ "playbook_trust_writeback_ack_id": "playbook:fake",
+ },
+ })
+
+ assert calls == [("verifier", str(identity.run_id))]
+ terminal.assert_not_awaited()
+ latest.assert_not_awaited()
+ assert result["status"] == "verifier_recorded_learning_pending"
+ assert result["learning_writeback"]["status"] == (
+ "untrusted_external_receipts_ignored_reconciler_pending"
+ )
+ assert result["untrusted_learning_receipt_refs_ignored"] is True
+ assert result["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_authenticated_backup_outcome_merges_all_nine_safe_refs(
+ monkeypatch,
+) -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-BRR-20260711-001",
+ source_fingerprint="backup-source-fingerprint",
+ route_id="agent99:backup_health:BackupCheck",
+ work_item_id="agent99-dispatch:awoooi:backup-readback",
+ )
+ captured: dict[str, object] = {}
+
+ async def verifier(**kwargs):
+ captured.update(kwargs)
+ return {
+ "status": "verifier_passed_learning_writeback_pending",
+ "receipt_persisted": True,
+ "post_verifier_passed": True,
+ "runtime_closure_verified": False,
+ }
+
+ monkeypatch.setattr(ingestion_module, "record_agent99_verifier_receipt", verifier)
+ result = await ingestion_module.ingest_agent99_outcome_receipt({
+ "identity": identity.public_dict(),
+ "evidence_refs": {
+ "agent99_outcome_receipt_id": "agent99-outcome:backup-1",
+ "post_verifier_evidence_ref": "backupcheck:verified:1",
+ "source_event_evidence_ref": "backup-alert:resolved:1",
+ "untrusted_extra": "must-not-persist",
+ },
+ "outcome_receipt": {
+ "identity": identity.public_dict(),
+ "mode": "BackupCheck",
+ "outcome": {
+ "identity": identity.public_dict(),
+ "schemaVersion": "agent99_outcome_contract_v1",
+ "state": "resolved",
+ "transportOk": True,
+ "verifierPassed": True,
+ "sourceEventResolved": True,
+ "evidenceRefs": {
+ "backup_status_evidence_ref": "backup-status:run:1",
+ "freshness_evidence_ref": "backup-freshness:slo:1",
+ "offsite_verify_evidence_ref": "offsite-verify:run:1",
+ "escrow_evidence_ref": "escrow-metadata:run:1",
+ "restore_drill_evidence_ref": "restore-drill:run:1",
+ "source_resolution_receipt_ref": "source-resolution:run:1",
+ "token": "must-not-persist",
+ },
+ },
+ },
+ })
+
+ refs = captured["evidence_refs"]
+ assert set(refs) == {
+ "agent99_outcome_receipt_id",
+ "post_verifier_evidence_ref",
+ "source_event_evidence_ref",
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+ }
+ assert "must-not-persist" not in str(refs)
+ assert captured["identity"].run_id == identity.run_id
+ assert result["status"] == "verifier_recorded_learning_pending"
+ assert result["runtime_closure_verified"] is False
+
+
+def test_outcome_evidence_ref_filter_rejects_secret_shaped_values() -> None:
+ refs = ingestion_module._public_evidence_refs(
+ {
+ "agent99_outcome_receipt_id": "https://unsafe.invalid/outcome",
+ "post_verifier_evidence_ref": "verifier:public-ref",
+ "source_event_evidence_ref": "source:public-ref",
+ },
+ {
+ "outcome": {
+ "evidenceRefs": {
+ "backup_status_evidence_ref": "token=must-not-persist",
+ "freshness_evidence_ref": "freshness:public-ref",
+ "offsite_verify_evidence_ref": "receipt?token=unsafe",
+ "escrow_evidence_ref": "../runtime-secret",
+ "restore_drill_evidence_ref": "Bearer must-not-persist",
+ }
+ }
+ },
+ )
+
+ assert refs == {
+ "post_verifier_evidence_ref": "verifier:public-ref",
+ "source_event_evidence_ref": "source:public-ref",
+ "freshness_evidence_ref": "freshness:public-ref",
+ }
+
+
+@pytest.mark.asyncio
+async def test_outcome_webhook_requires_shared_token(monkeypatch) -> None:
+ identity = _identity()
+ request = webhooks.Agent99OutcomeWebhookPayload(
+ identity=identity.public_dict(),
+ outcome_receipt=_outcome(identity),
+ evidence_refs=_evidence_refs(),
+ )
+ monkeypatch.setattr(
+ webhooks.settings,
+ "AGENT99_SRE_ALERT_RELAY_TOKEN",
+ "relay-token",
+ )
+ with pytest.raises(HTTPException) as exc_info:
+ await webhooks.ingest_agent99_outcome_webhook(request, None)
+ assert exc_info.value.status_code == 401
+
+ async def accepted(_payload):
+ return {
+ "status": "verifier_recorded_learning_pending",
+ "accepted": True,
+ "runtime_closure_verified": False,
+ }
+
+ monkeypatch.setattr(webhooks, "ingest_agent99_outcome_receipt", accepted)
+ response = await webhooks.ingest_agent99_outcome_webhook(
+ request,
+ "relay-token",
+ )
+ assert response["accepted"] is True
+ assert response["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_safe_failed_generation_advances_bounded_and_keeps_one_flight_key(
+ monkeypatch,
+) -> None:
+ identities = []
+ flight_calls: list[tuple[str, str]] = []
+
+ class GenerationLedger:
+ async def reserve(self, *, identity, payload):
+ identities.append(identity)
+ if identity.execution_generation == "1":
+ return {
+ "status": "idempotent_failed",
+ "claimed": False,
+ "receipt": {
+ "status": "dispatch_not_delivered_retry_safe",
+ "retry_policy": {
+ "safe_to_retry": True,
+ "requires_generation_advance": True,
+ "max_generation": 3,
+ },
+ },
+ }
+ return {
+ "status": "reserved",
+ "claimed": True,
+ "claim_token": "generation-2-claim",
+ "receipt": None,
+ }
+
+ async def complete(
+ self,
+ *,
+ identity,
+ claim_token,
+ dispatch_receipt,
+ **_kwargs,
+ ):
+ return {
+ **build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt=dispatch_receipt,
+ controlled_apply_authorized=True,
+ ),
+ "receipt_persisted": True,
+ }
+
+ async def mark_delivery_attempt_started(self, *, identity, claim_token):
+ return {
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "identity": identity.public_dict(),
+ "receipt_persisted": True,
+ }
+
+ async def acquire(key, owner, **_kwargs):
+ flight_calls.append((key, owner))
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: GenerationLedger(),
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ acquire,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda _payload: {
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ )
+
+ result = await bridge_alertmanager_to_agent99(
+ alert_id="recurrence-2",
+ alertname="ColdStartGateBlocked",
+ severity="critical",
+ namespace="host_recovery",
+ target_resource="cold-start-gate",
+ message="cold-start gate blocked",
+ fingerprint="same-source-fingerprint",
+ incident_id="INC-20260711-11C751",
+ route_id="agent99_cold_start_recovery",
+ )
+
+ assert result["status"] == "dispatched"
+ assert [item.execution_generation for item in identities] == ["1", "2"]
+ assert identities[0].idempotency_key != identities[1].idempotency_key
+ assert identities[0].single_flight_key == identities[1].single_flight_key
+ assert flight_calls == [
+ (identities[1].single_flight_key, "generation-2-claim")
+ ]
+
+
+@pytest.mark.asyncio
+async def test_retry_not_before_does_not_burn_generation_or_transport(
+ monkeypatch,
+) -> None:
+ identities = []
+
+ class CooldownLedger:
+ async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
+ identities.append(identity)
+ return {
+ "status": "retry_not_before",
+ "claimed": False,
+ "next_attempt_at": "2026-07-11T20:10:00",
+ "receipt": {
+ "status": "dispatch_not_delivered_retry_safe",
+ "retry_policy": {
+ "safe_to_retry": True,
+ "requires_generation_advance": True,
+ "max_generation": 3,
+ },
+ },
+ }
+
+ async def unexpected_flight(*_args, **_kwargs): # type: ignore[no-untyped-def]
+ raise AssertionError("cooldown must return before Redis or transport")
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: CooldownLedger(),
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ unexpected_flight,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda _payload: (_ for _ in ()).throw(
+ AssertionError("cooldown must not dispatch")
+ ),
+ )
+
+ result = await bridge_alertmanager_to_agent99(
+ alert_id="cooldown-recurrence",
+ alertname="ColdStartGateBlocked",
+ severity="critical",
+ namespace="host_recovery",
+ target_resource="cold-start-gate",
+ message="cold-start gate blocked",
+ fingerprint="same-source-fingerprint",
+ incident_id="INC-20260711-11C751",
+ route_id="agent99_cold_start_recovery",
+ )
+
+ assert result["status"] == "retryable"
+ assert result["reason"] == "durable_retry_not_before"
+ assert result["nextAttemptAt"] == "2026-07-11T20:10:00"
+ assert [item.execution_generation for item in identities] == ["1"]
+
+
+@pytest.mark.asyncio
+async def test_webhook_pending_inbox_is_not_queued_or_authorized(monkeypatch) -> None:
+ append = AsyncMock()
+ monkeypatch.setattr(
+ "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
+ lambda: SimpleNamespace(append=append),
+ )
+ monkeypatch.setattr(
+ webhooks,
+ "bridge_alertmanager_to_agent99",
+ AsyncMock(
+ return_value={
+ "status": "delivery_pending",
+ "reason": "accepted_without_inbox_triggered",
+ "dispatchPerformed": True,
+ "identity": _identity().public_dict(),
+ "dispatchReceipt": {
+ "accepted": True,
+ "inbox_triggered": False,
+ "status": "accepted_inbox_pending",
+ },
+ "correlatedReceipt": {
+ "receipt_persisted": True,
+ "dispatch_promoted": False,
+ "controlled_apply_authorized": False,
+ "runtime_closure_verified": False,
+ },
+ }
+ ),
+ )
+
+ handoff = await webhooks._try_auto_repair_background(
+ incident_id="INC-20260711-11C751",
+ approval_id="00000000-0000-0000-0000-000000000751",
+ alert_type="ColdStartGateBlocked",
+ target_resource="cold-start-gate",
+ namespace="host_recovery",
+ risk_level="low",
+ source_alert_id="pending-inbox",
+ source_alertname="ColdStartGateBlocked",
+ source_fingerprint="same-source-fingerprint",
+ )
+
+ assert handoff["queued"] is False
+ assert handoff["dispatch_accepted"] is True
+ assert handoff["inbox_triggered"] is False
+ assert handoff["dispatch_promoted"] is False
+ assert handoff["runtime_execution_authorized"] is False
+ assert handoff["runtime_closure_verified"] is False
+ assert append.await_args.kwargs["success"] is False
diff --git a/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py b/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py
new file mode 100644
index 000000000..4b6a4ab95
--- /dev/null
+++ b/apps/api/tests/test_agent99_controlled_dispatch_reconciler_job.py
@@ -0,0 +1,313 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+import pytest
+
+from src.jobs import agent99_controlled_dispatch_reconciler_job as job
+from src.services.agent99_controlled_dispatch_ledger import (
+ build_agent99_dispatch_identity,
+)
+
+
+class _Ledger:
+ def __init__(self, receipt: dict[str, object]) -> None:
+ self.receipt = receipt
+ self.verifier_calls: list[dict[str, object]] = []
+
+ async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
+ return [{"identity": IDENTITY, "receipt": self.receipt}]
+
+ async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def]
+ self.verifier_calls.append(kwargs)
+ return {
+ **self.receipt,
+ "post_verifier_passed": True,
+ "receipt_persisted": True,
+ "verifier": {
+ "status": "success",
+ "evidence_refs": kwargs["evidence_refs"],
+ },
+ }
+
+
+IDENTITY = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="fp-cold-start",
+ route_id="agent99:host_recovery:Recover",
+)
+
+
+def test_agent99_playbook_identity_is_project_scoped() -> None:
+ other = build_agent99_dispatch_identity(
+ project_id="other-project",
+ incident_id=IDENTITY.incident_id,
+ source_fingerprint=IDENTITY.source_fingerprint,
+ route_id=IDENTITY.route_id,
+ )
+
+ assert job._playbook_id(IDENTITY) != job._playbook_id(other)
+
+
+@pytest.mark.asyncio
+async def test_reconciler_consumes_authenticated_outcome_and_closes_same_run(
+ monkeypatch,
+) -> None:
+ ledger = _Ledger({
+ "status": "dispatch_accepted_verifier_pending",
+ "post_verifier_passed": False,
+ "dispatch_receipt": {"kind": "host_recovery"},
+ })
+ finalized: list[dict[str, object]] = []
+
+ monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
+ monkeypatch.setattr(
+ job,
+ "read_agent99_sre_outcome",
+ lambda _run_id: {
+ "automationRunId": str(IDENTITY.run_id),
+ "traceId": IDENTITY.trace_id,
+ "workItemId": IDENTITY.work_item_id,
+ "mode": "Recover",
+ "controlledApply": True,
+ "outcome": {
+ "schemaVersion": "agent99_outcome_contract_v1",
+ "state": "resolved",
+ "transportOk": True,
+ "verifierName": "agent99-recover-post-condition-v1",
+ "verifierPassed": True,
+ "sourceEventResolved": True,
+ "verifiedAt": "2026-07-11T20:00:00+08:00",
+ },
+ },
+ )
+
+ async def finalize(identity, *, mode, evidence_refs): # type: ignore[no-untyped-def]
+ finalized.append({
+ "identity": identity,
+ "mode": mode,
+ "evidence_refs": evidence_refs,
+ })
+ return {"runtime_closure_verified": True}
+
+ monkeypatch.setattr(job, "_finalize_learning", finalize)
+
+ result = await job.reconcile_agent99_controlled_dispatches_once()
+
+ assert result == {"scanned": 1, "verifier_written": 1, "closed": 1}
+ assert len(ledger.verifier_calls) == 1
+ refs = ledger.verifier_calls[0]["evidence_refs"]
+ assert refs["agent99_outcome_receipt_id"].endswith(str(IDENTITY.run_id))
+ assert "post_verifier_evidence_ref" in refs
+ assert "source_event_evidence_ref" in refs
+ assert finalized[0]["identity"].run_id == IDENTITY.run_id
+ assert finalized[0]["mode"] == "Recover"
+
+
+@pytest.mark.asyncio
+async def test_reconciler_does_not_finalize_without_outcome(monkeypatch) -> None:
+ ledger = _Ledger({
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "post_verifier_passed": False,
+ })
+ monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
+ monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run_id: None)
+
+ result = await job.reconcile_agent99_controlled_dispatches_once()
+
+ assert result == {"scanned": 1, "verifier_written": 0, "closed": 0}
+ assert ledger.verifier_calls == []
+
+
+def test_windows_relay_outcome_readback_is_authenticated_and_public_safe() -> None:
+ source = Path(__file__).resolve().parents[3] / "agent99-sre-alert-relay.ps1"
+ text = source.read_text(encoding="utf-8")
+
+ assert '$context.Request.HttpMethod -eq "GET"' in text
+ assert "outcome_readback_auth_not_configured" in text
+ assert 'Headers["X-Agent99-Relay-Token"]' in text
+ assert "storesRawEvidence = $false" in text
+ assert "stdout" not in text[text.index("function Get-AgentOutcomeReadback"):text.index("function Write-AgentRelayEvent")]
+
+
+@pytest.mark.asyncio
+async def test_exact_11c751_legacy_failure_backfills_once_without_recurrence() -> None:
+ row = {
+ "incident_id": "INC-20260711-11C751",
+ "project_id": "awoooi",
+ "alertname": "ColdStartGateBlocked",
+ "severity": "low",
+ "alert_category": "general",
+ "notification_type": "legacy_backfill",
+ "approval_id": "00000000-0000-0000-0000-000000000751",
+ "approval_fingerprint": "legacy-source-fingerprint-11c751",
+ "approval_status": "execution_failed",
+ }
+ calls: list[dict[str, object]] = []
+ dispatched_keys: set[tuple[str, str, str]] = set()
+ fetch_count = 0
+
+ async def fetcher(**_kwargs): # type: ignore[no-untyped-def]
+ nonlocal fetch_count
+ fetch_count += 1
+ if fetch_count == 1:
+ return [row]
+ return [{
+ **row,
+ "latest_approval_id": "00000000-0000-0000-0000-000000000752",
+ "latest_approval_fingerprint": "newer-approval-fingerprint-11c751",
+ }]
+
+ async def dispatcher(**kwargs): # type: ignore[no-untyped-def]
+ calls.append(kwargs)
+ key = (
+ str(kwargs["incident_id"]),
+ str(kwargs["fingerprint"]),
+ str(kwargs["route_id"]),
+ )
+ if key in dispatched_keys:
+ return {"status": "deduplicated", "dispatchPerformed": False}
+ dispatched_keys.add(key)
+ return {"status": "dispatched", "dispatchPerformed": True}
+
+ first = await job.backfill_legacy_agent99_dispatches_once(
+ fetcher=fetcher,
+ dispatcher=dispatcher,
+ )
+ second = await job.backfill_legacy_agent99_dispatches_once(
+ fetcher=fetcher,
+ dispatcher=dispatcher,
+ )
+
+ assert first == {
+ "scanned": 1,
+ "dispatch_performed": 1,
+ "deduplicated": 0,
+ "retryable": 0,
+ "blocked": 0,
+ }
+ assert second == {
+ "scanned": 1,
+ "dispatch_performed": 0,
+ "deduplicated": 1,
+ "retryable": 0,
+ "blocked": 0,
+ }
+ assert len(dispatched_keys) == 1
+ assert calls[0]["incident_id"] == "INC-20260711-11C751"
+ assert calls[0]["namespace"] == "host_recovery"
+ assert calls[0]["target_resource"] == "cold-start-gate"
+ assert calls[0]["route_id"] == "agent99:host_recovery:Recover"
+ assert calls[0]["fingerprint"].startswith("legacy-cold-start:")
+ assert calls[0]["fingerprint"] == calls[1]["fingerprint"]
+ assert calls[0]["approval_id"] == calls[1]["approval_id"]
+ assert calls[0]["labels"]["legacy_approval_fingerprint"] == (
+ calls[1]["labels"]["legacy_approval_fingerprint"]
+ )
+ assert calls[0]["labels"]["legacy_executor"] == "ansible"
+ assert calls[0]["labels"]["legacy_status"] == "execution_failed"
+
+
+@pytest.mark.asyncio
+async def test_learning_checkpoint_resumes_without_replaying_assets(
+ monkeypatch,
+) -> None:
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-BACKUP",
+ source_fingerprint="backup-source-1",
+ route_id="agent99:backup_health:BackupCheck",
+ approval_id="00000000-0000-0000-0000-000000000751",
+ )
+
+ class CheckpointLedger:
+ def __init__(self) -> None:
+ self.refs: dict[str, str] = {}
+
+ async def record_learning_asset_acks(
+ self,
+ *,
+ identity, # type: ignore[no-untyped-def]
+ receipt_refs,
+ ) -> dict[str, object]:
+ self.refs.update(receipt_refs)
+ return {
+ "status": "learning_asset_acks_recorded",
+ "receipt_persisted": True,
+ "learning_receipt_refs": dict(self.refs),
+ "runtime_closure_verified": False,
+ }
+
+ ledger = CheckpointLedger()
+ effects = {"playbook": 0, "km": 0, "telegram": 0, "dr": 0}
+ terminal_calls = 0
+
+ async def write_playbook(*_args, **_kwargs): # type: ignore[no-untyped-def]
+ effects["playbook"] += 1
+ return "playbooks:PB-A99:version:1"
+
+ async def write_km(*_args, **_kwargs): # type: ignore[no-untyped-def]
+ effects["km"] += 1
+ return "knowledge_entries:km-1:embedding_verified"
+
+ async def write_telegram(*_args, **_kwargs): # type: ignore[no-untyped-def]
+ effects["telegram"] += 1
+ return "telegram_outbound:1:durable_ack"
+
+ async def write_dr(*_args, **_kwargs): # type: ignore[no-untyped-def]
+ effects["dr"] += 1
+ return "knowledge_entries:dr-1:dr_scorecard"
+
+ async def terminal(**_kwargs): # type: ignore[no-untyped-def]
+ nonlocal terminal_calls
+ terminal_calls += 1
+ if terminal_calls == 1:
+ return {
+ "status": "learning_writeback_persistence_failed",
+ "runtime_closure_verified": False,
+ }
+ return {
+ "status": "closed_verified_learning_written",
+ "runtime_closure_verified": True,
+ }
+
+ monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
+ monkeypatch.setattr(job, "_ensure_playbook_trust", write_playbook)
+ monkeypatch.setattr(job, "_ensure_km_writeback", write_km)
+ monkeypatch.setattr(job, "_ensure_telegram_receipt", write_telegram)
+ monkeypatch.setattr(job, "_ensure_dr_scorecard", write_dr)
+ monkeypatch.setattr(job, "record_agent99_learning_writeback", terminal)
+
+ first = await job._finalize_learning(
+ identity,
+ mode="BackupCheck",
+ evidence_refs={},
+ )
+ second = await job._finalize_learning(
+ identity,
+ mode="BackupCheck",
+ evidence_refs={},
+ )
+
+ assert first["runtime_closure_verified"] is False
+ assert second["runtime_closure_verified"] is True
+ assert effects == {"playbook": 1, "km": 1, "telegram": 1, "dr": 1}
+ assert terminal_calls == 2
+
+
+def test_legacy_backfill_query_is_bounded_and_requires_failure_evidence() -> None:
+ source = Path(job.__file__).read_text(encoding="utf-8")
+ fetch = source[
+ source.index("async def _fetch_legacy_cold_start_failures"):source.index(
+ "def _legacy_source_fingerprint"
+ )
+ ]
+
+ assert 'LEGACY_COLD_START_INCIDENT_ID = "INC-20260711-11C751"' in source
+ assert "lower(approval_state.status::text) = 'execution_failed'" in fetch
+ assert "ORDER BY candidate.created_at ASC, candidate.id ASC" in fetch
+ assert "legacy.input ->> 'executor' = 'ansible'" in fetch
+ assert "LIMIT :limit" in fetch
+ assert "statement_timeout = '5000ms'" in fetch
+ assert "backfill_legacy_agent99_dispatches_once()" in source
diff --git a/apps/api/tests/test_agent99_dispatch_receipt_projection.py b/apps/api/tests/test_agent99_dispatch_receipt_projection.py
new file mode 100644
index 000000000..d7b292828
--- /dev/null
+++ b/apps/api/tests/test_agent99_dispatch_receipt_projection.py
@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import pytest
+
+from src.services import telegram_gateway as telegram_gateway_module
+
+
+@pytest.mark.asyncio
+async def test_telegram_projects_correlated_receipt_without_dispatch(
+ monkeypatch,
+) -> None:
+ calls: list[tuple[str, str]] = []
+ correlated_receipt = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "dispatch_accepted_verifier_pending",
+ "identity": {
+ "incident_id": "INC-20260711-11C751",
+ "run_id": "7cf8fdf7-0966-5ac6-95b2-09e32808b248",
+ "trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
+ "work_item_id": "agent99-dispatch:awoooi:INC-20260711-11C751:recovery",
+ "idempotency_key": "agent99-controlled:stable-key",
+ },
+ "dispatch_receipt": {
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "receipt_persisted": True,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "verifier": {"status": "pending"},
+ "learning_writeback": {"status": "pending_verifier"},
+ }
+
+ async def fake_read(*, project_id: str, incident_id: str):
+ calls.append((project_id, incident_id))
+ return correlated_receipt
+
+ monkeypatch.setattr(
+ telegram_gateway_module,
+ "read_agent99_dispatch_receipt",
+ fake_read,
+ )
+
+ projection = (
+ await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
+ "\n".join(
+ [
+ "ℹ️ ACTION REQUIRED | 低風險",
+ "📋 INC-20260711-11C751",
+ "🎯 資源：cold-start-gate",
+ "🏷️ 分類：general",
+ "🤖 AI 自動化鏈路",
+ ]
+ ),
+ source="unit-test",
+ )
+ )
+
+ assert calls == [("awoooi", "INC-20260711-11C751")]
+ assert projection is not None
+ assert projection["schema_version"] == (
+ "telegram_agent99_dispatch_receipt_projection_v1"
+ )
+ assert projection["dispatch_performed"] is False
+ assert projection["accepted"] is True
+ assert projection["incident_id"] == "INC-20260711-11C751"
+ assert projection["run_id"] == correlated_receipt["identity"]["run_id"]
+ assert projection["trace_id"] == correlated_receipt["identity"]["trace_id"]
+ assert projection["idempotency_key"] == (
+ correlated_receipt["identity"]["idempotency_key"]
+ )
+ assert projection["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_telegram_unbound_card_fails_closed_without_receipt_read(
+ monkeypatch,
+) -> None:
+ reads: list[str] = []
+
+ async def unexpected_read(*, project_id: str, incident_id: str):
+ reads.append(f"{project_id}:{incident_id}")
+ return None
+
+ monkeypatch.setattr(
+ telegram_gateway_module,
+ "read_agent99_dispatch_receipt",
+ unexpected_read,
+ )
+
+ card = telegram_gateway_module.format_aiops_signal_alert_card(
+ "SourceProviderIngestionStale provider_freshness_signal freshness "
+ "window timeout Sentry SigNoz provider freshness last seen missing "
+ "direct ref verifier readback"
+ )
+ projection = await telegram_gateway_module._mirror_ai_automation_alert_card_to_agent99(
+ card,
+ source="unit-test",
+ )
+
+ assert projection is not None
+ assert projection["dispatch_performed"] is False
+ assert projection["status"] == "receipt_unbound_fail_closed"
+ assert reads == []
diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py
index 8cce78a4d..3a69cbe9c 100644
--- a/apps/api/tests/test_agent99_sre_bridge.py
+++ b/apps/api/tests/test_agent99_sre_bridge.py
@@ -4,7 +4,12 @@ import json
import pytest
+from src.services.agent99_controlled_dispatch_ledger import (
+ build_agent99_dispatch_identity,
+ build_agent99_dispatch_receipt_envelope,
+)
from src.services.agent99_sre_bridge import (
+ acquire_agent99_sre_single_flight,
agent99_sre_single_flight_key,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
@@ -54,6 +59,35 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"]
+def test_explicit_backup_identity_wins_over_generic_freshness_words() -> None:
+ payload = build_agent99_sre_alert(
+ alert_id="backup-escrow-missing",
+ alertname="BackupCredentialEscrowEvidenceMissing",
+ severity="critical",
+ namespace="awoooi-prod",
+ target_resource="backup_restore",
+ message=(
+ "backup freshness missing; escrow evidence missing; "
+ "readback verifier required"
+ ),
+ labels={
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ },
+ fingerprint="fp-backup-freshness-collision",
+ )
+
+ assert payload["kind"] == "backup_health"
+ assert payload["suggestedMode"] == "BackupCheck"
+ assert payload["controlledApply"] is False
+ assert "backup-readback" in payload["labels"]
+ assert "no-false-green" in payload["labels"]
+ assert "read-only" in payload["labels"]
+ assert "do not run backup/restore" in payload["instruction"]
+ assert "remote delete" in payload["instruction"]
+ assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
+
+
@pytest.mark.parametrize(
("alertname", "target", "message", "labels", "expected_kind", "expected_mode"),
[
@@ -331,6 +365,7 @@ def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) -
"http_status": 202,
"accepted": True,
"inbox_triggered": True,
+ "delivery_certainty": "delivered",
"stores_raw_response": False,
}
assert "private-path" not in str(receipt)
@@ -345,12 +380,12 @@ def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
@pytest.mark.asyncio
async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None:
- async def duplicate(*_args, **_kwargs) -> bool:
- return False
+ async def duplicate(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": False, "reason": "single_flight_duplicate"}
dispatched: list[dict[str, object]] = []
monkeypatch.setattr(
- "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
duplicate,
)
monkeypatch.setattr(
@@ -366,11 +401,11 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="duplicate-alert",
- alertname="HostDown",
+ alertname="UnknownObservation",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
- message="host_down",
+ message="observation only",
fingerprint="same-fingerprint",
)
@@ -381,12 +416,12 @@ async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) ->
@pytest.mark.asyncio
async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None:
- async def winner(*_args, **_kwargs) -> bool:
- return True
+ async def winner(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
dispatched: list[dict[str, object]] = []
monkeypatch.setattr(
- "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
@@ -402,11 +437,11 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
result = await bridge_alertmanager_to_agent99(
alert_id="winner-alert",
- alertname="HostDown",
+ alertname="UnknownObservation",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
- message="host_down",
+ message="observation only",
fingerprint="winner-fingerprint",
)
@@ -414,7 +449,7 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
assert result["dispatch"] == "relay"
assert result["dispatchReceipt"]["accepted"] is True
assert result["dispatchReceipt"]["inbox_triggered"] is True
- assert result["suggestedMode"] == "Recover"
+ assert result["suggestedMode"] == "Status"
assert len(dispatched) == 1
@@ -422,8 +457,8 @@ async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> No
async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
monkeypatch,
) -> None:
- async def winner(*_args, **_kwargs) -> bool:
- return True
+ async def winner(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
released: list[tuple[str, str]] = []
@@ -431,7 +466,7 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
released.append((fingerprint, alert_id))
monkeypatch.setattr(
- "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
@@ -442,19 +477,20 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
lambda _payload: {
"status": "relay_response_not_accepted",
- "transport": "relay",
- "accepted": False,
- "inbox_triggered": False,
- },
+ "transport": "relay",
+ "accepted": False,
+ "inbox_triggered": False,
+ "delivery_certainty": "not_delivered",
+ },
)
result = await bridge_alertmanager_to_agent99(
alert_id="receipt-rejected",
- alertname="HostDown",
+ alertname="UnknownObservation",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
- message="host_down",
+ message="observation only",
fingerprint="receipt-rejected-fingerprint",
)
@@ -466,8 +502,8 @@ async def test_agent99_bridge_releases_lock_when_receipt_is_not_accepted(
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None:
- async def winner(*_args, **_kwargs) -> bool:
- return True
+ async def winner(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
released: list[tuple[str, str]] = []
@@ -478,7 +514,7 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
raise RuntimeError("relay unavailable")
monkeypatch.setattr(
- "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
@@ -492,15 +528,18 @@ async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) ->
result = await bridge_alertmanager_to_agent99(
alert_id="transport-failure",
- alertname="HostDown",
+ alertname="UnknownObservation",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
- message="host_down",
+ message="observation only",
fingerprint="transport-fingerprint",
)
- assert result == {"status": "failed", "reason": "RuntimeError"}
+ assert result["status"] == "failed"
+ assert result["reason"] == "RuntimeError"
+ assert result["dispatchPerformed"] is False
+ assert result["runtimeClosureVerified"] is False
assert released == [("transport-fingerprint", "transport-failure")]
@@ -546,3 +585,447 @@ async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> No
assert "redis.call('get'" in str(seen["script"])
assert seen["key_count"] == 1
assert seen["owner"] == "alert-1"
+
+
+class _FakeControlledDispatchLedger:
+ def __init__(self) -> None:
+ self.reserved = False
+ self.receipt: dict[str, object] | None = None
+ self.complete_calls = 0
+ self.defer_calls = 0
+
+ async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
+ if self.reserved:
+ return {
+ "status": "idempotent_completed",
+ "claimed": False,
+ "identity": identity.public_dict(),
+ "receipt": self.receipt,
+ }
+ self.reserved = True
+ return {
+ "status": "reserved",
+ "claimed": True,
+ "identity": identity.public_dict(),
+ "receipt": None,
+ "claim_token": "claim-attempt-1",
+ }
+
+ async def complete( # type: ignore[no-untyped-def]
+ self, *, identity, claim_token, dispatch_receipt, **kwargs
+ ) -> dict[str, object]:
+ self.complete_calls += 1
+ self.receipt = {
+ **build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt=dispatch_receipt,
+ ),
+ "receipt_persisted": True,
+ }
+ return self.receipt
+
+ async def mark_delivery_attempt_started( # type: ignore[no-untyped-def]
+ self, *, identity, claim_token
+ ) -> dict[str, object]:
+ self.receipt = {
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "identity": identity.public_dict(),
+ "retry_policy": {"reconcile_only": True, "safe_to_retry": False},
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ }
+ return self.receipt
+
+ async def defer_claim_retryable( # type: ignore[no-untyped-def]
+ self, *, identity, claim_token, reason, retry_after_seconds
+ ) -> dict[str, object]:
+ self.defer_calls += 1
+ self.receipt = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "dispatch_lock_retryable",
+ "identity": identity.public_dict(),
+ "dispatch_accepted": False,
+ "inbox_triggered": False,
+ "dispatch_promoted": False,
+ "controlled_apply_authorized": False,
+ "retry_policy": {
+ "safe_to_retry": True,
+ "retry_same_generation": True,
+ "requires_generation_advance": False,
+ "retry_after_seconds": retry_after_seconds,
+ "reason": reason,
+ },
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ }
+ return self.receipt
+
+
+def test_agent99_dispatch_identity_is_stable_across_projections() -> None:
+ alertmanager = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="source-fingerprint",
+ route_id="agent99_cold_start_recovery",
+ execution_generation="1",
+ )
+ telegram = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-20260711-11C751",
+ source_fingerprint="source-fingerprint",
+ route_id="agent99_cold_start_recovery",
+ execution_generation="1",
+ )
+
+ assert telegram.run_id == alertmanager.run_id
+ assert telegram.trace_id == alertmanager.trace_id
+ assert telegram.idempotency_key == alertmanager.idempotency_key
+ assert telegram.work_item_id == alertmanager.work_item_id
+
+
+@pytest.mark.asyncio
+async def test_mutating_dispatch_is_deduplicated_by_postgres_identity(
+ monkeypatch,
+) -> None:
+ ledger = _FakeControlledDispatchLedger()
+ dispatched: list[dict[str, object]] = []
+
+ async def acquired(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ acquired,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: dispatched.append(payload)
+ or {
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": str(payload["id"]),
+ "kind": str(payload["kind"]),
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ )
+ common = {
+ "alertname": "ColdStartGateBlocked",
+ "severity": "critical",
+ "namespace": "host_recovery",
+ "target_resource": "cold-start-gate",
+ "message": "cold-start gate blocked",
+ "fingerprint": "source-fingerprint",
+ "project_id": "awoooi",
+ "incident_id": "INC-20260711-11C751",
+ "route_id": "agent99_cold_start_recovery",
+ }
+
+ first = await bridge_alertmanager_to_agent99(alert_id="alertmanager-1", **common)
+ second = await bridge_alertmanager_to_agent99(alert_id="telegram-card-2", **common)
+
+ assert first["status"] == "dispatched"
+ assert second["status"] == "deduplicated"
+ assert len(dispatched) == 1
+ assert dispatched[0]["force"] is False
+ assert first["identity"] == second["identity"]
+ assert first["correlatedReceipt"]["runtime_execution_authorized"] is False
+ assert first["correlatedReceipt"]["runtime_closure_verified"] is False
+ assert first["correlatedReceipt"]["verifier"]["status"] == "pending"
+ assert first["correlatedReceipt"]["learning_writeback"]["status"] == (
+ "pending_verifier"
+ )
+
+
+@pytest.mark.asyncio
+async def test_mutating_dispatch_redis_failure_performs_zero_transport(
+ monkeypatch,
+) -> None:
+ ledger = _FakeControlledDispatchLedger()
+ dispatched: list[dict[str, object]] = []
+
+ async def redis_unavailable(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": False, "reason": "redis_unavailable_fail_closed"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ redis_unavailable,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: dispatched.append(payload),
+ )
+
+ result = await bridge_alertmanager_to_agent99(
+ alert_id="cold-start-redis-failure",
+ alertname="ColdStartGateBlocked",
+ severity="critical",
+ namespace="host_recovery",
+ target_resource="cold-start-gate",
+ message="cold-start gate blocked",
+ fingerprint="redis-failure-fingerprint",
+ incident_id="INC-20260711-11C751",
+ route_id="agent99_cold_start_recovery",
+ )
+
+ assert result["status"] == "retryable"
+ assert result["reason"] == "redis_unavailable_fail_closed"
+ assert result["dispatchPerformed"] is False
+ assert dispatched == []
+ assert ledger.complete_calls == 0
+ assert ledger.defer_calls == 1
+ assert result["correlatedReceipt"]["receipt_persisted"] is True
+ assert result["correlatedReceipt"]["runtime_closure_verified"] is False
+
+
+@pytest.mark.asyncio
+async def test_mutating_single_flight_redis_exception_is_fail_closed(
+ monkeypatch,
+) -> None:
+ monkeypatch.setattr(
+ "src.core.redis_client.get_redis",
+ lambda: (_ for _ in ()).throw(RuntimeError("redis down")),
+ )
+
+ decision = await acquire_agent99_sre_single_flight(
+ "same-controlled-idempotency-key",
+ "same-run-owner",
+ fail_closed=True,
+ )
+
+ assert decision == {
+ "acquired": False,
+ "reason": "redis_unavailable_fail_closed",
+ }
+
+
+@pytest.mark.asyncio
+async def test_backupcheck_read_only_dispatch_has_durable_same_run_receipt(
+ monkeypatch,
+) -> None:
+ ledger = _FakeControlledDispatchLedger()
+ dispatched: list[dict[str, object]] = []
+
+ async def acquired(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ acquired,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: dispatched.append(payload)
+ or {
+ "schema_version": "agent99_sre_dispatch_receipt_v1",
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": str(payload["id"]),
+ "kind": str(payload["kind"]),
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ )
+
+ result = await bridge_alertmanager_to_agent99(
+ alert_id="backup-readback",
+ alertname="BackupCredentialEscrowEvidenceMissing",
+ severity="warning",
+ namespace="awoooi-prod",
+ target_resource="backup_restore",
+ message="freshness escrow restore evidence required",
+ labels={"event_type": "backup_restore_escrow_signal"},
+ fingerprint="backup-fingerprint",
+ incident_id="INC-20260711-BACKUP",
+ route_id="agent99:backup_health:BackupCheck",
+ )
+
+ assert result["status"] == "dispatched"
+ assert result["identity"]["run_id"]
+ assert result["identity"]["trace_id"]
+ assert result["identity"]["work_item_id"]
+ assert result["correlatedReceipt"]["receipt_persisted"] is True
+ assert dispatched[0]["suggestedMode"] == "BackupCheck"
+ assert dispatched[0]["controlledApply"] is False
+ assert result["correlatedReceipt"]["controlled_apply_authorized"] is False
+
+
+class _DeliveryUnknownLedger:
+ def __init__(self) -> None:
+ self.reserved = False
+ self.receipt: dict[str, object] | None = None
+
+ async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
+ if self.reserved:
+ return {
+ "status": "idempotent_running",
+ "claimed": False,
+ "receipt": self.receipt,
+ }
+ self.reserved = True
+ self.receipt = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "dispatch_reserved",
+ "identity": identity.public_dict(),
+ "outbox": {"status": "pending_delivery"},
+ }
+ return {
+ "status": "reserved",
+ "claimed": True,
+ "claim_token": "unknown-delivery-claim",
+ "receipt": self.receipt,
+ }
+
+ async def complete(self, **_kwargs): # type: ignore[no-untyped-def]
+ return {
+ **(self.receipt or {}),
+ "status": "receipt_persistence_failed_no_runtime_closure",
+ "receipt_persisted": False,
+ }
+
+ async def mark_delivery_attempt_started( # type: ignore[no-untyped-def]
+ self, *, identity, claim_token
+ ) -> dict[str, object]:
+ self.receipt = {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "identity": identity.public_dict(),
+ "retry_policy": {"safe_to_retry": False, "reconcile_only": True},
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ }
+ return self.receipt
+
+ async def defer_claim_retryable(self, **_kwargs): # type: ignore[no-untyped-def]
+ raise AssertionError("lock is acquired in this test")
+
+
+@pytest.mark.asyncio
+async def test_accepted_transport_with_receipt_write_failure_is_not_replayed(
+ monkeypatch,
+) -> None:
+ ledger = _DeliveryUnknownLedger()
+ dispatched: list[dict[str, object]] = []
+
+ async def acquired(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ acquired,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: dispatched.append(payload)
+ or {
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": str(payload["id"]),
+ "kind": str(payload["kind"]),
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ )
+ common = {
+ "alertname": "ColdStartGateBlocked",
+ "severity": "critical",
+ "namespace": "host_recovery",
+ "target_resource": "cold-start-gate",
+ "message": "cold-start gate blocked",
+ "fingerprint": "unknown-delivery-fingerprint",
+ "incident_id": "INC-20260711-11C751",
+ "route_id": "agent99:host_recovery:Recover",
+ }
+
+ first = await bridge_alertmanager_to_agent99(alert_id="first", **common)
+ second = await bridge_alertmanager_to_agent99(alert_id="second", **common)
+
+ assert first["reason"] == "dispatch_receipt_persistence_failed"
+ assert second["dispatchPerformed"] is False
+ assert len(dispatched) == 1
+
+
+class _GenerationAdvanceLedger(_FakeControlledDispatchLedger):
+ async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def]
+ if identity.execution_generation == "1":
+ return {
+ "status": "idempotent_failed",
+ "claimed": False,
+ "receipt": {
+ "status": "dispatch_not_delivered_retry_safe",
+ "identity": identity.public_dict(),
+ "retry_policy": {
+ "safe_to_retry": True,
+ "requires_generation_advance": True,
+ "max_generation": 3,
+ },
+ },
+ }
+ return await super().reserve(identity=identity, payload=payload)
+
+
+@pytest.mark.asyncio
+async def test_safe_not_delivered_failure_advances_generation_before_dispatch(
+ monkeypatch,
+) -> None:
+ ledger = _GenerationAdvanceLedger()
+ dispatched: list[dict[str, object]] = []
+
+ async def acquired(*_args, **_kwargs) -> dict[str, object]:
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ acquired,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: dispatched.append(payload)
+ or {
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": str(payload["id"]),
+ "kind": str(payload["kind"]),
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ )
+
+ result = await bridge_alertmanager_to_agent99(
+ alert_id="retry-generation",
+ alertname="ColdStartGateBlocked",
+ severity="critical",
+ namespace="host_recovery",
+ target_resource="cold-start-gate",
+ message="cold-start gate blocked",
+ fingerprint="retry-generation-fingerprint",
+ incident_id="INC-20260711-11C751",
+ route_id="agent99:host_recovery:Recover",
+ )
+
+ assert result["status"] == "dispatched"
+ assert result["identity"]["execution_generation"] == "2"
+ assert len(dispatched) == 1
diff --git a/apps/api/tests/test_alert_approval_guard.py b/apps/api/tests/test_alert_approval_guard.py
index 827791cfe..1c0becbba 100644
--- a/apps/api/tests/test_alert_approval_guard.py
+++ b/apps/api/tests/test_alert_approval_guard.py
@@ -53,6 +53,77 @@ async def test_blocks_llm_kubectl_default_namespace_for_prod_alert() -> None:
assert result.reason == "namespace_not_allowed:default"
+@pytest.mark.asyncio
+async def test_cold_start_invalid_k8s_target_becomes_fail_closed_agent99_candidate() -> None:
+ result = await guard_alert_approval_action(
+ action="kubectl scale deployment/cold-start-gate --replicas=2 -n default",
+ alert_namespace="default",
+ alertname="ColdStartGateBlocked",
+ alert_category="general",
+ target_resource="cold-start-gate",
+ )
+
+ assert result.blocked is True
+ assert result.reason == "namespace_not_allowed:default"
+ assert result.action == (
+ "DRAFT_READY - CONTROLLED_AGENT99_RECOVERY: "
+ "agent99-mode Recover controlledApply=true"
+ )
+ assert result.metadata["blocked_action"].startswith("kubectl scale")
+ assert result.metadata["controlled_target_reroute"] is True
+ assert result.metadata["controlled_target_executor"] == "Agent99"
+
+ contract = result.metadata["repair_candidate_promotion_contract"]
+ assert contract["route_id"] == "agent99_recover_after_owner_review"
+ assert contract["status"] == "controlled_route_candidate_pending_receipts"
+ assert contract["runtime_execution_authorized"] is False
+ assert contract["runtime_write_allowed"] is False
+ assert contract["completion_status"] == "partial"
+ assert contract["blocked_count"] > 0
+ assert "dispatch_receipt" in contract["blocked_fields"]
+ assert "post_apply_verifier" in contract["blocked_fields"]
+ assert "km_writeback" in contract["blocked_fields"]
+ assert "playbook_trust_writeback" in contract["blocked_fields"]
+ assert "kubectl_for_host_control_plane_target" in contract["forbidden_operations"]
+ assert "generic_ansible_keyword_fallback" in contract["forbidden_operations"]
+
+ closure = contract["controlled_automation_closure"]
+ stages = {row["stage"]: row["status"] for row in closure["stages"]}
+ assert stages == {
+ "sensor": "pending",
+ "normalize": "passed",
+ "check": "pending",
+ "controlled_apply": "pending",
+ "verify": "pending",
+ "learn_writeback": "pending",
+ }
+ assert {row["asset_type"] for row in closure["automation_asset_ledger"]} == {
+ "Sensor",
+ "PlayBook",
+ "Verifier",
+ "KM",
+ }
+
+
+@pytest.mark.asyncio
+async def test_cold_start_valid_namespace_still_cannot_become_fake_k8s_action() -> None:
+ result = await guard_alert_approval_action(
+ action=(
+ "kubectl rollout restart deployment/cold-start-gate "
+ "-n awoooi-prod"
+ ),
+ alert_namespace="awoooi-prod",
+ alertname="ColdStartGateBlocked",
+ alert_category="general",
+ target_resource="cold-start-gate",
+ )
+
+ assert result.blocked is True
+ assert result.reason == "kubernetes_action_not_applicable:control_plane_recovery"
+ assert result.metadata["controlled_target_executor"] == "Agent99"
+ assert result.action.startswith("DRAFT_READY - CONTROLLED_AGENT99_RECOVERY")
+
+
@pytest.mark.asyncio
async def test_blocks_hallucinated_mutating_k8s_resource() -> None:
resolver = StubResolver(success=False, candidates=["awoooi-api"])
diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py
index 811b8dfba..9485c1dc4 100644
--- a/apps/api/tests/test_awooop_operator_timeline_labels.py
+++ b/apps/api/tests/test_awooop_operator_timeline_labels.py
@@ -17,6 +17,11 @@ from src.api.v1.platform.operator_runs import (
ListCicdEventsResponse,
ListRunsResponse,
)
+from src.services.agent99_controlled_dispatch_ledger import (
+ build_agent99_dispatch_identity,
+ build_agent99_dispatch_receipt_envelope,
+)
+from src.services.agent99_sre_bridge import build_agent99_sre_alert
from src.services.ollama_failover_manager import OllamaEndpoint, OllamaRoutingResult
from src.services.ollama_health_monitor import HealthReport, HealthStatus
from src.services.platform_operator_service import (
@@ -1056,6 +1061,645 @@ def test_ai_alert_card_delivery_summary_keeps_no_false_green_status() -> None:
assert summary["production_write_count"] == 0
+def test_backup_restore_delivery_readback_is_durable_but_never_false_green() -> None:
+ run_id = UUID("0b25cafa-00a4-520b-8946-a3ea467b13fc")
+ message_id = UUID("97ff9f41-7cdc-4893-8aab-634286fb128c")
+ item = _ai_alert_card_delivery_item({
+ "message_id": message_id,
+ "run_id": run_id,
+ "project_id": "awoooi",
+ "channel_type": "telegram",
+ "message_type": "final",
+ "provider_message_id": "37451",
+ "send_status": "sent",
+ "send_error": None,
+ "queued_at": datetime(2026, 7, 11, 10, 57, 30),
+ "sent_at": datetime(2026, 7, 11, 10, 57, 35),
+ "triggered_by_state": "legacy_gateway",
+ "alert_card": {
+ "schema_version": "ai_automation_alert_card_mirror_v1",
+ "card_schema": "ai_automation_alert_card_v1",
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ "target": "backup_restore",
+ "gates": [
+ "controlled_playbook_queue",
+ "runtime_write_gate=controlled",
+ ],
+ "candidate_only": False,
+ "controlled_playbook_queue": True,
+ "runtime_write_gate_count": 1,
+ "runtime_write_gate_state": "controlled",
+ "delivery_receipt_readback_required": True,
+ },
+ "source_refs": {
+ "alert_ids": ["backup_restore_escrow_signal"],
+ "fingerprints": [
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ],
+ },
+ "run_state": "completed",
+ "agent_id": "legacy-telegram-gateway",
+ "run_created_at": datetime(2026, 7, 11, 10, 57, 30),
+ })
+
+ contract = item["backup_restore_automation"]
+ assert contract is not None
+ assert contract["contract_origin"] == "synthesized_at_read"
+ assert contract["contract_persisted_at_send"] is False
+ assert contract["contract_provenance"][
+ "source_contract_present_at_send"
+ ] is False
+ assert contract["contract_provenance"]["recorded_policy_present"] is False
+ assert contract["policy"]["policy_version"] == (
+ "backup_restore_readback_policy_v2"
+ )
+ assert contract["receipt"]["status"] == (
+ "projected_from_historical_source_row"
+ )
+ assert contract["receipt"]["source_row_exists"] is True
+ assert contract["receipt"]["is_agent99_execution_receipt"] is False
+ assert contract["work_item"]["work_item_id"].startswith("BRR-WI-")
+ assert contract["candidate"]["candidate_id"].startswith("BRR-CAND-")
+ assert contract["verifier"]["verifier_id"].startswith("BRR-VER-")
+ assert contract["candidate"]["execution_state"] == "not_started"
+ assert contract["candidate"]["agent99_dispatch"]["status"] == (
+ "dispatch_receipt_missing"
+ )
+ assert contract["verifier"]["status"] == (
+ "projected_dispatch_receipt_missing_verifier_not_started"
+ )
+ assert contract["verifier"]["agent99_outcome"]["status"] == (
+ "outcome_source_unavailable"
+ )
+ assert item["declared_runtime_write_gate_count"] == 1
+ assert item["declared_runtime_write_allowed"] is True
+ assert item["declared_controlled_playbook_queue"] is True
+ assert item["effective_policy_projection"]["runtime_write_gate_count"] == 0
+ assert item["runtime_write_gate_count"] == 0
+ assert item["runtime_write_allowed"] is False
+ assert item["controlled_playbook_queue"] is False
+ assert item["runtime_write_gate_state"] == "evidence_only"
+ assert item["learning_writeback_ready"] is False
+ assert item["learning_writeback_status"] == (
+ "blocked_backup_restore_evidence_or_verifier"
+ )
+ assert item["learning_writeback_refs"]["work_item_id"] == (
+ contract["work_item"]["work_item_id"]
+ )
+ assert all(
+ binding["status"] != "ready_for_consumer_context"
+ for binding in item["learning_registry_bindings"]
+ )
+
+ summary = _ai_alert_card_delivery_summary_from_row(
+ {
+ "total": 98,
+ "sent_total": 98,
+ "failed_total": 0,
+ "pending_total": 0,
+ "shadow_total": 0,
+ "delivery_receipt_required_total": 98,
+ "declared_runtime_write_gate_open_count": 96,
+ "runtime_write_gate_open_count": 0,
+ "declared_learning_writeback_ready_total": 98,
+ "learning_writeback_ready_total": 0,
+ },
+ project_id="awoooi",
+ event_type="backup_restore_escrow_signal",
+ lane="backup_restore_escrow_triage",
+ )
+ assert summary["status"] == "blocked_backup_restore_evidence_or_verifier"
+ assert summary["declared_runtime_write_gate_open_count"] == 96
+ assert summary["runtime_write_gate_open_count"] == 0
+ assert summary["runtime_write_allowed"] is False
+ assert summary["learning_writeback_ready_total"] == 0
+ assert summary["declared_learning_writeback_ready_total"] == 98
+ assert summary["backup_runtime_write_gate_excluded_total"] == 96
+ assert summary["backup_learning_writeback_excluded_total"] == 98
+ assert summary["learning_writeback_missing_total"] == 98
+ assert summary["backup_restore_readback_required"] is True
+ assert summary["no_false_green_enforced"] is True
+ assert summary["production_write_count"] == 0
+
+
+def test_unfiltered_mixed_summary_uses_effective_per_row_counts() -> None:
+ summary = _ai_alert_card_delivery_summary_from_row(
+ {
+ "total": 100,
+ "sent_total": 100,
+ "failed_total": 0,
+ "pending_total": 0,
+ "shadow_total": 0,
+ "delivery_receipt_required_total": 100,
+ "backup_restore_total": 98,
+ "declared_runtime_write_gate_open_count": 97,
+ "runtime_write_gate_open_count": 1,
+ "declared_learning_writeback_ready_total": 100,
+ "learning_writeback_ready_total": 2,
+ },
+ project_id="awoooi",
+ event_type=None,
+ lane=None,
+ )
+
+ assert summary["status"] == (
+ "observed_effective_backup_policy_projection_applied"
+ )
+ assert summary["declared_runtime_write_gate_open_count"] == 97
+ assert summary["backup_restore_total"] == 98
+ assert summary["runtime_write_gate_open_count"] == 1
+ assert summary["runtime_write_allowed"] is True
+ assert summary["backup_runtime_write_gate_excluded_total"] == 96
+ assert summary["declared_learning_writeback_ready_total"] == 100
+ assert summary["learning_writeback_ready_total"] == 2
+ assert summary["backup_learning_writeback_excluded_total"] == 98
+ assert summary["learning_writeback_missing_total"] == 98
+ assert summary["no_false_green_enforced"] is True
+ projection = summary["effective_policy_projection"]
+ assert projection["scope"] == "mixed_or_unfiltered"
+ assert projection["applied"] is True
+ assert projection["policy"]["source_hash"].startswith("sha256:")
+
+
+def test_unfiltered_readback_only_backup_still_enforces_no_false_green() -> None:
+ summary = _ai_alert_card_delivery_summary_from_row(
+ {
+ "total": 1,
+ "sent_total": 1,
+ "failed_total": 0,
+ "pending_total": 0,
+ "shadow_total": 0,
+ "delivery_receipt_required_total": 1,
+ "backup_restore_total": 1,
+ "declared_runtime_write_gate_open_count": 0,
+ "runtime_write_gate_open_count": 0,
+ "declared_learning_writeback_ready_total": 0,
+ "learning_writeback_ready_total": 0,
+ },
+ project_id="awoooi",
+ event_type=None,
+ lane=None,
+ )
+
+ assert summary["backup_restore_total"] == 1
+ assert summary["runtime_write_allowed"] is False
+ assert summary["learning_writeback_ready_total"] == 0
+ assert summary["no_false_green_enforced"] is True
+ assert summary["backup_restore_readback_required"] is True
+ assert summary["status"] == (
+ "observed_effective_backup_policy_projection_applied"
+ )
+
+
+def test_backup_item_consumes_agent99_dispatch_receipt_without_faking_outcome() -> None:
+ item = _ai_alert_card_delivery_item({
+ "message_id": "source-message-1",
+ "run_id": "source-run-1",
+ "project_id": "awoooi",
+ "channel_type": "telegram",
+ "message_type": "final",
+ "send_status": "sent",
+ "alert_card": {
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ "target": "backup_restore",
+ "gates": [
+ "controlled_playbook_queue=readback_only",
+ "runtime_write_gate=0_until_verifier",
+ ],
+ "candidate_only": True,
+ "controlled_playbook_queue": False,
+ "runtime_write_gate_count": 0,
+ "runtime_write_gate_state": "evidence_only",
+ "delivery_receipt_readback_required": True,
+ "backup_restore_automation": {
+ "schema_version": "backup_restore_readback_automation_v1",
+ "verifier": {"checks": []},
+ },
+ },
+ "source_refs": {
+ "fingerprints": [
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ],
+ },
+ "agent99_dispatch_receipt": {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": "backup-alert-1",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ "run_id": "agent99-run-1",
+ "trace_id": (
+ "00-12345678901234567890123456789012-1234567890123456-01"
+ ),
+ "work_item_id": (
+ "agent99-dispatch:awoooi:INC-BACKUP:backupcheck"
+ ),
+ },
+ })
+
+ contract = item["backup_restore_automation"]
+ assert contract["contract_origin"] == "persisted_at_send"
+ assert contract["contract_persisted_at_send"] is True
+ assert contract["contract_provenance"][
+ "source_contract_present_at_send"
+ ] is True
+ assert contract["contract_provenance"]["recorded_contract_origin"] == (
+ "legacy_unspecified"
+ )
+ assert contract["contract_provenance"]["recorded_policy_present"] is False
+ assert contract["contract_provenance"][
+ "recorded_immutable_declared_present"
+ ] is False
+ assert contract["candidate"]["execution_state"] == "dispatched"
+ assert contract["candidate"]["status"] == "dispatched_verifier_pending"
+ assert contract["candidate"]["agent99_dispatch"]["agent99_run_id"] == (
+ "agent99-run-1"
+ )
+ assert contract["candidate"]["agent99_dispatch"]["identity_complete"] is True
+ assert contract["candidate"]["identity_binding"]["brr_work_item_id"] == (
+ contract["work_item"]["work_item_id"]
+ )
+ assert contract["candidate"]["identity_binding"]["brr_verifier_id"] == (
+ contract["verifier"]["verifier_id"]
+ )
+ assert contract["receipt"]["source_receipt_id"] == "source-message-1"
+ assert contract["receipt"]["is_agent99_execution_receipt"] is False
+ assert contract["verifier"]["status"] == "dispatched_verifier_pending"
+ assert contract["verifier"]["terminal"] is False
+ assert contract["verifier"]["agent99_outcome"]["status"] == (
+ "dispatch_accepted_outcome_pending"
+ )
+ assert contract["next_action"] == (
+ "ingest_agent99_backupcheck_outcome_then_run_independent_dr_verifier"
+ )
+
+
+def test_backup_item_consumes_real_bridge_and_ledger_identity() -> None:
+ alert = build_agent99_sre_alert(
+ alert_id="backup-alert-real-builder",
+ alertname="BackupRestoreEscrowSignal",
+ severity="warning",
+ namespace="awoooi",
+ target_resource="backup_restore",
+ message="backup freshness degraded; restore drill evidence missing",
+ fingerprint="backup-real-builder-fingerprint",
+ labels={"event_type": "backup_restore_escrow_signal"},
+ )
+ assert alert["kind"] == "backup_health"
+ assert alert["suggestedMode"] == "BackupCheck"
+ assert alert["controlledApply"] is False
+
+ identity = build_agent99_dispatch_identity(
+ project_id="awoooi",
+ incident_id="INC-BACKUP-REAL",
+ source_fingerprint="backup-real-builder-fingerprint",
+ route_id="agent99:backup_health:BackupCheck",
+ )
+ current_receipt = build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt={
+ "schema_version": "agent99_sre_dispatch_receipt_v1",
+ "status": "accepted",
+ "transport": "relay",
+ "alert_id": str(alert["id"]),
+ "kind": "backup_health",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ )
+ public_identity = identity.public_dict()
+ item = _ai_alert_card_delivery_item({
+ "message_id": "source-message-real-builder",
+ "run_id": "source-run-real-builder",
+ "project_id": "awoooi",
+ "alert_card": {
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ "target": "backup_restore",
+ "delivery_receipt_readback_required": True,
+ "backup_restore_automation": {
+ "schema_version": "backup_restore_readback_automation_v1",
+ },
+ },
+ "agent99_dispatch_receipt": {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "dispatch_accepted_verifier_pending",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "incident_id": "INC-BACKUP-REAL",
+ "run_id": public_identity["run_id"],
+ "trace_id": public_identity["trace_id"],
+ "work_item_id": public_identity["work_item_id"],
+ "receipt_persisted": True,
+ },
+ "agent99_current_receipt": current_receipt,
+ "agent99_current_run_state": "waiting_tool",
+ "agent99_current_trace_id": public_identity["trace_id"],
+ })
+
+ contract = item["backup_restore_automation"]
+ dispatch = contract["candidate"]["agent99_dispatch"]
+ assert dispatch["dispatched"] is True
+ assert dispatch["agent99_run_id"] == public_identity["run_id"]
+ assert dispatch["trace_id"] == public_identity["trace_id"]
+ assert dispatch["agent99_work_item_id"] == public_identity["work_item_id"]
+ assert contract["candidate"]["identity_binding"]["complete"] is True
+ assert contract["verifier"]["terminal"] is False
+
+
+def test_backup_item_advances_from_current_durable_ledger_terminal() -> None:
+ run_id = "c977681c-7753-5d02-9b36-aa42b5d74df0"
+ trace_id = "00-12345678901234567890123456789012-1234567890123456-01"
+ work_item_id = "agent99-dispatch:awoooi:INC-BACKUP:backupcheck"
+ stage_identity = {
+ "run_id": run_id,
+ "trace_id": trace_id,
+ "work_item_id": work_item_id,
+ }
+ item = _ai_alert_card_delivery_item({
+ "message_id": "source-message-terminal",
+ "run_id": "source-run-terminal",
+ "project_id": "awoooi",
+ "channel_type": "telegram",
+ "message_type": "final",
+ "send_status": "sent",
+ "alert_card": {
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ "target": "backup_restore",
+ "gates": [
+ "controlled_playbook_queue=readback_only",
+ "runtime_write_gate=0_until_verifier",
+ ],
+ "declared_gates": [
+ "controlled_playbook_queue",
+ "runtime_write_gate=controlled",
+ ],
+ "declared_runtime_write_gate_count": 1,
+ "declared_runtime_write_gate_state": "controlled",
+ "declared_controlled_playbook_queue": True,
+ "candidate_only": True,
+ "controlled_playbook_queue": False,
+ "runtime_write_gate_count": 0,
+ "runtime_write_gate_state": "evidence_only",
+ "delivery_receipt_readback_required": True,
+ "backup_restore_automation": {
+ "schema_version": "backup_restore_readback_automation_v1",
+ "contract_origin": "persisted_at_send",
+ "verifier": {"checks": []},
+ },
+ },
+ "source_refs": {
+ "fingerprints": [
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ],
+ },
+ "agent99_dispatch_receipt": {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "dispatch_accepted_verifier_pending",
+ "transport": "relay",
+ "alert_id": "backup-alert-terminal",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "incident_id": "INC-BACKUP",
+ **stage_identity,
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ },
+ "agent99_current_receipt": {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "status": "closed_verified_learning_written",
+ "identity": {
+ "incident_id": "INC-BACKUP",
+ **stage_identity,
+ "idempotency_key": "agent99-controlled:test",
+ },
+ "dispatch_receipt": {
+ "transport": "relay",
+ "alert_id": "backup-alert-terminal",
+ "kind": "backup_health",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "dispatch_accepted": True,
+ "post_verifier_passed": True,
+ "runtime_closure_verified": True,
+ "closure_complete": True,
+ "verifier": {
+ "schema_version": "agent99_independent_verifier_receipt_v1",
+ "status": "success",
+ **stage_identity,
+ "outcome_state": "resolved",
+ "transport_ok": True,
+ "verifier_passed": True,
+ "source_event_resolved": True,
+ "verified_at": "2026-07-11T20:20:00+08:00",
+ "evidence_refs": {
+ "agent99_outcome_receipt_id": "agent99-outcome-1",
+ "post_verifier_evidence_ref": "post-verifier-1",
+ "source_event_evidence_ref": "source-event-1",
+ "backup_status_evidence_ref": "backup-status-1",
+ "freshness_evidence_ref": "freshness-1",
+ "offsite_verify_evidence_ref": "offsite-1",
+ "escrow_evidence_ref": "escrow-1",
+ "restore_drill_evidence_ref": "restore-drill-1",
+ "source_resolution_receipt_ref": "source-resolved-1",
+ },
+ },
+ "learning_writeback": {
+ "schema_version": "agent99_learning_writeback_receipt_v1",
+ "status": "success",
+ **stage_identity,
+ "receipt_refs": {
+ "incident_closure_receipt_id": "inc-close-1",
+ "telegram_lifecycle_receipt_id": "tg-life-1",
+ "km_writeback_ack_id": "km-ack-1",
+ "playbook_trust_writeback_ack_id": "pb-ack-1",
+ "dr_scorecard_writeback_ack_id": "dr-scorecard-ack-1",
+ },
+ },
+ },
+ "agent99_current_run_state": "completed",
+ "agent99_current_trace_id": trace_id,
+ })
+
+ contract = item["backup_restore_automation"]
+ assert contract["candidate"]["execution_state"] == "completed_verified"
+ assert contract["candidate"]["identity_binding"]["complete"] is True
+ assert contract["verifier"]["status"] == "success_terminal"
+ assert contract["verifier"]["result"] == "success"
+ assert contract["verifier"]["backup_evidence_refs_complete"] is True
+ assert contract["verifier"]["terminal"] is True
+ assert contract["verifier"]["agent99_outcome"]["status"] == (
+ "terminal_verified_learning_written"
+ )
+ assert contract["effective_policy_projection"][
+ "learning_writeback_ready"
+ ] is True
+ assert contract["closure_state"] == "closed_verified_learning_written"
+ assert contract["next_action"] == "monitor_backup_restore_recurrence"
+ assert item["declared_runtime_write_gate_count"] == 1
+ assert item["runtime_write_allowed"] is False
+
+
+def test_backup_current_ledger_identity_mismatch_fails_closed() -> None:
+ item = _ai_alert_card_delivery_item({
+ "message_id": "source-message-mismatch",
+ "run_id": "source-run-mismatch",
+ "project_id": "awoooi",
+ "alert_card": {
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ "target": "backup_restore",
+ "delivery_receipt_readback_required": True,
+ "backup_restore_automation": {
+ "schema_version": "backup_restore_readback_automation_v1",
+ },
+ },
+ "agent99_dispatch_receipt": {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "incident_id": "INC-BACKUP",
+ "run_id": "expected-run",
+ "trace_id": "expected-trace",
+ "work_item_id": "expected-work-item",
+ },
+ "agent99_current_receipt": {
+ "schema_version": "agent99_controlled_dispatch_receipt_v1",
+ "identity": {
+ "incident_id": "INC-BACKUP",
+ "run_id": "different-run",
+ "trace_id": "different-trace",
+ "work_item_id": "different-work-item",
+ },
+ "dispatch_receipt": {
+ "kind": "backup_health",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "dispatch_accepted": True,
+ "runtime_closure_verified": True,
+ "closure_complete": True,
+ },
+ "agent99_current_run_state": "completed",
+ "agent99_current_trace_id": "different-trace",
+ })
+
+ contract = item["backup_restore_automation"]
+ dispatch = contract["candidate"]["agent99_dispatch"]
+ assert dispatch["dispatched"] is False
+ assert dispatch["status"] == "current_ledger_identity_mismatch_fail_closed"
+ assert dispatch["runtime_closure_verified"] is False
+ assert contract["candidate"]["execution_state"] == "not_started"
+ assert contract["verifier"]["terminal"] is False
+ assert contract["closure_state"] == "partial_degraded_safe_next_action"
+
+
+@pytest.mark.asyncio
+async def test_unfiltered_query_selects_declared_effective_and_dispatch_receipts(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ queries: list[str] = []
+
+ class FakeResult:
+ def __init__(self, *, first=None, rows=None):
+ self._first = first
+ self._rows = rows or []
+
+ def mappings(self):
+ return self
+
+ def first(self):
+ return self._first
+
+ def all(self):
+ return self._rows
+
+ class FakeDb:
+ async def execute(self, statement, _params=None):
+ query = str(statement)
+ queries.append(query)
+ if "COUNT(*) AS total" in query:
+ return FakeResult(first={
+ "total": 100,
+ "sent_total": 100,
+ "failed_total": 0,
+ "pending_total": 0,
+ "shadow_total": 0,
+ "delivery_receipt_required_total": 100,
+ "backup_restore_total": 98,
+ "declared_runtime_write_gate_open_count": 97,
+ "runtime_write_gate_open_count": 1,
+ "declared_learning_writeback_ready_total": 100,
+ "learning_writeback_ready_total": 2,
+ "latest_sent_at": None,
+ "latest_queued_at": None,
+ })
+ return FakeResult(rows=[])
+
+ class FakeDbContext:
+ async def __aenter__(self):
+ return FakeDb()
+
+ async def __aexit__(self, exc_type, exc, tb):
+ return False
+
+ async def passthrough_store(_kind, _key, value, **_kwargs):
+ return value
+
+ monkeypatch.setattr(
+ platform_operator_service,
+ "get_db_context",
+ lambda _project_id: FakeDbContext(),
+ )
+ monkeypatch.setattr(
+ platform_operator_service,
+ "store_operator_summary_async",
+ passthrough_store,
+ )
+
+ payload = await platform_operator_service.list_ai_alert_card_delivery_readback(
+ project_id="awoooi",
+ event_type=None,
+ lane=None,
+ page=1,
+ per_page=5,
+ refresh=True,
+ )
+
+ summary_query = next(query for query in queries if "COUNT(*) AS total" in query)
+ list_query = next(query for query in queries if "m.message_id" in query)
+ assert "AS declared_runtime_write_gate_open_count" in summary_query
+ assert "AS backup_restore_total" in summary_query
+ assert "declared_runtime_write_gate_count" in summary_query
+ assert "AS declared_learning_writeback_ready_total" in summary_query
+ assert summary_query.count("<> 'backup_restore_escrow_signal'") >= 2
+ assert "-> 'agent99_dispatch_receipt' AS agent99_dispatch_receipt" in list_query
+ assert "LEFT JOIN LATERAL" in list_query
+ assert "agent99_current_receipt" in list_query
+ assert payload["summary"]["declared_runtime_write_gate_open_count"] == 97
+ assert payload["summary"]["backup_restore_total"] == 98
+ assert payload["summary"]["runtime_write_gate_open_count"] == 1
+ assert payload["summary"]["declared_learning_writeback_ready_total"] == 100
+ assert payload["summary"]["learning_writeback_ready_total"] == 2
+ assert payload["summary"]["no_false_green_enforced"] is True
+
+
def test_list_ai_alert_cards_response_preserves_delivery_metadata() -> None:
run_id = UUID("5c0306e0-591a-5445-9a33-80f499426b38")
message_id = UUID("66cdb6ad-46a4-48f5-9d3b-b1ac9c0b2e92")
@@ -1356,6 +2000,9 @@ async def test_list_ai_alert_card_delivery_readback_failsofts_slow_db(
async def __aexit__(self, exc_type, exc, tb):
return False
+ async def no_cached_readback(**_kwargs):
+ return None
+
monkeypatch.setattr(
platform_operator_service,
"_AI_ALERT_CARD_QUERY_TIMEOUT_SECONDS",
@@ -1376,6 +2023,11 @@ async def test_list_ai_alert_card_delivery_readback_failsofts_slow_db(
"_load_ai_alert_card_delivery_readback_direct",
_no_ai_alert_card_direct_readback,
)
+ monkeypatch.setattr(
+ platform_operator_service,
+ "_get_cached_ai_alert_card_delivery_readback",
+ no_cached_readback,
+ )
payload = await platform_operator_service.list_ai_alert_card_delivery_readback(
project_id="awoooi",
diff --git a/apps/api/tests/test_backup_restore_alertmanager_ingress.py b/apps/api/tests/test_backup_restore_alertmanager_ingress.py
new file mode 100644
index 000000000..f63f29c59
--- /dev/null
+++ b/apps/api/tests/test_backup_restore_alertmanager_ingress.py
@@ -0,0 +1,397 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+from uuid import UUID
+
+import pytest
+from fastapi import BackgroundTasks
+from starlette.requests import Request
+
+from src.api.v1 import webhooks
+from src.services import backup_restore_alertmanager_ingress as ingress
+from src.services.agent99_controlled_dispatch_ledger import (
+ build_agent99_dispatch_receipt_envelope,
+)
+from src.services.agent99_sre_bridge import bridge_alertmanager_to_agent99
+
+
+def _request() -> Request:
+ return Request(
+ {
+ "type": "http",
+ "method": "POST",
+ "path": "/api/v1/webhooks/alertmanager",
+ "scheme": "http",
+ "server": ("testserver", 80),
+ "client": ("127.0.0.1", 50000),
+ "query_string": b"",
+ "headers": [],
+ }
+ )
+
+
+def _signal_kwargs() -> dict[str, object]:
+ return {
+ "project_id": "awoooi",
+ "alert_id": "backup-alert-1",
+ "alertname": "BackupCredentialEscrowEvidenceMissing",
+ "severity": "critical",
+ "namespace": "awoooi-prod",
+ "target_resource": "backup_restore",
+ "message": "backup freshness and escrow evidence missing",
+ "labels": {
+ "event_type": "backup_restore_escrow_signal",
+ "lane": "backup_restore_escrow_triage",
+ },
+ "annotations": {"summary": "readback verifier required"},
+ "source_fingerprint": "source-fingerprint",
+ "source_event_fingerprint": "alertmanager-native-fingerprint",
+ "source_started_at": "2026-07-11T10:00:00Z",
+ "alert_category": "backup",
+ "notification_type": "TYPE-1",
+ "source_url": "http://alertmanager/graph",
+ }
+
+
+def test_backup_ingress_owner_is_stable_per_source_occurrence() -> None:
+ first = ingress.build_backup_restore_ingress_owner(
+ project_id="awoooi",
+ source_fingerprint="source-fingerprint",
+ source_event_fingerprint="native-fingerprint",
+ source_started_at="2026-07-11T10:00:00Z",
+ )
+ replay = ingress.build_backup_restore_ingress_owner(
+ project_id="awoooi",
+ source_fingerprint="source-fingerprint",
+ source_event_fingerprint="native-fingerprint",
+ source_started_at="2026-07-11T10:00:00Z",
+ )
+ new_firing = ingress.build_backup_restore_ingress_owner(
+ project_id="awoooi",
+ source_fingerprint="source-fingerprint",
+ source_event_fingerprint="native-fingerprint",
+ source_started_at="2026-07-12T10:00:00Z",
+ )
+
+ assert first == replay
+ assert new_firing["approval_id"] != first["approval_id"]
+ assert first["route_id"] == "agent99:backup_health:BackupCheck"
+ assert first["single_dispatch_owner"] == "alertmanager_raw_signal"
+ assert first["telegram_role"] == "receipt_projection_only"
+ assert first["controlled_apply"] is False
+ assert first["read_only"] is True
+
+
+@pytest.mark.asyncio
+async def test_alertmanager_type1_backup_queues_durable_owner_before_info_shortcut(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ grouped = SimpleNamespace(is_grouped=False)
+ grouping = SimpleNamespace(evaluate=AsyncMock(return_value=grouped))
+ process = AsyncMock(return_value={"status": "queued"})
+ record = AsyncMock()
+ append = AsyncMock()
+
+ monkeypatch.setattr(webhooks, "get_alert_grouping_service", lambda: grouping)
+ monkeypatch.setattr(
+ "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
+ lambda: SimpleNamespace(append=append),
+ )
+ monkeypatch.setattr(webhooks, "record_alertmanager_event", record)
+ monkeypatch.setattr(
+ webhooks,
+ "process_backup_restore_alertmanager_signal",
+ process,
+ )
+ monkeypatch.setattr(
+ webhooks,
+ "get_approval_service",
+ lambda: (_ for _ in ()).throw(
+ AssertionError("generic approval/TYPE-1 branch must not run")
+ ),
+ )
+
+ tasks = BackgroundTasks()
+ payload = webhooks.AlertmanagerPayload(
+ status="firing",
+ alerts=[
+ webhooks.AlertmanagerAlert(
+ status="firing",
+ labels={
+ "alertname": "HostBackupFailed",
+ "severity": "warning",
+ "component": "backup_restore",
+ "namespace": "awoooi-prod",
+ },
+ annotations={
+ "summary": "backup freshness missing; readback required"
+ },
+ startsAt=datetime.now(UTC).isoformat(),
+ fingerprint="native-backup-fingerprint",
+ generatorURL="http://alertmanager/graph",
+ )
+ ],
+ )
+
+ response = await webhooks.alertmanager_webhook(_request(), payload, tasks)
+
+ assert response.success is True
+ assert "durable read-only BackupCheck" in response.message
+ assert response.approval_created is False
+ assert process.await_count == 0
+
+ await tasks()
+
+ process.assert_awaited_once()
+ call = process.await_args.kwargs
+ assert call["alertname"] == "HostBackupFailed"
+ assert call["source_event_fingerprint"] == "native-backup-fingerprint"
+ assert call["notification_type"] == "TYPE-1"
+ assert call["alert_category"] == "backup"
+ assert call["target_resource"] == "backup_restore"
+
+
+class _FakeApprovalService:
+ def __init__(self, calls: list[str]) -> None:
+ self.calls = calls
+ self.approval: SimpleNamespace | None = None
+ self.created_request = None
+
+ async def get_approval(self, approval_id: UUID):
+ self.calls.append("owner_read")
+ if self.approval is not None:
+ assert self.approval.id == approval_id
+ return self.approval
+
+ async def find_by_fingerprint(self, **_kwargs):
+ self.calls.append("legacy_owner_read")
+ return None
+
+ async def create_approval_with_fingerprint(self, *, request, fingerprint):
+ self.calls.append("owner_create")
+ self.created_request = request
+ assert fingerprint == "source-fingerprint"
+ self.approval = SimpleNamespace(
+ id=UUID(request.metadata["preallocated_approval_id"]),
+ incident_id=request.incident_id,
+ hit_count=1,
+ risk_level=request.risk_level,
+ )
+ return self.approval
+
+ async def increment_hit_count(self, approval_id: UUID):
+ self.calls.append("owner_recurrence")
+ assert self.approval is not None
+ assert self.approval.id == approval_id
+ self.approval.hit_count += 1
+ return self.approval
+
+ async def update_incident_id(self, approval_id: UUID, incident_id: str):
+ self.calls.append("incident_bind")
+ assert self.approval is not None
+ assert self.approval.id == approval_id
+ self.approval.incident_id = incident_id
+
+
+@pytest.mark.asyncio
+async def test_backup_ingress_binds_canonical_incident_then_dispatches_and_reads_back(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ calls: list[str] = []
+ approval_service = _FakeApprovalService(calls)
+ incident_id = "INC-20260711-BACKUP"
+ identity: dict[str, str] = {}
+
+ async def create_incident(**_kwargs) -> str:
+ calls.append("incident_create")
+ return incident_id
+
+ async def canonical_readback(value: str, *, project_id: str):
+ calls.append("incident_canonical_readback")
+ assert value == incident_id
+ assert project_id == "awoooi"
+ return SimpleNamespace(status="investigating", persisted_to_pg=True)
+
+ async def dispatch(**kwargs):
+ calls.append("agent99_dispatch")
+ assert kwargs["incident_id"] == incident_id
+ assert kwargs["route_id"] == "agent99:backup_health:BackupCheck"
+ identity.update({
+ "incident_id": incident_id,
+ "run_id": "12345678-1234-5678-9234-567812345678",
+ "trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
+ "work_item_id": kwargs["work_item_id"],
+ })
+ return {
+ "status": "dispatched",
+ "dispatchPerformed": True,
+ "identity": dict(identity),
+ }
+
+ async def readback(**kwargs):
+ calls.append("dispatch_receipt_readback")
+ assert kwargs == {"project_id": "awoooi", "incident_id": incident_id}
+ return {
+ "status": "dispatch_accepted_verifier_pending",
+ "identity": dict(identity),
+ "dispatch_receipt": {
+ "kind": "backup_health",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "verifier": {"status": "pending"},
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ }
+
+ monkeypatch.setattr(ingress, "get_approval_service", lambda: approval_service)
+ monkeypatch.setattr(ingress, "create_incident_for_approval", create_incident)
+ monkeypatch.setattr(
+ ingress,
+ "get_incident_service",
+ lambda: SimpleNamespace(get_for_readback=canonical_readback),
+ )
+ monkeypatch.setattr(ingress, "bridge_alertmanager_to_agent99", dispatch)
+ monkeypatch.setattr(ingress, "read_agent99_dispatch_receipt", readback)
+ monkeypatch.setattr(ingress, "record_alertmanager_event", AsyncMock())
+
+ result = await ingress.process_backup_restore_alertmanager_signal(
+ **_signal_kwargs()
+ )
+
+ assert calls.index("owner_create") < calls.index("incident_create")
+ assert calls.index("incident_create") < calls.index("incident_bind")
+ assert calls.index("incident_bind") < calls.index("incident_canonical_readback")
+ assert calls.index("incident_canonical_readback") < calls.index(
+ "agent99_dispatch"
+ )
+ assert calls.index("agent99_dispatch") < calls.index(
+ "dispatch_receipt_readback"
+ )
+ assert result["status"] == "backupcheck_dispatched_verifier_pending"
+ assert result["receipt_persisted"] is True
+ assert result["accepted"] is True
+ assert result["verifier"]["status"] == "pending"
+ assert result["verifier"]["terminal"] is False
+ assert result["runtime_execution_authorized"] is False
+ assert result["runtime_closure_verified"] is False
+ assert result["production_write_performed"] is False
+ assert approval_service.created_request.risk_level.value == "low"
+ assert approval_service.created_request.metadata["telegram_role"] == (
+ "receipt_projection_only"
+ )
+ assert "run_restore" in result["prohibited_actions"]
+
+
+class _FakeDispatchLedger:
+ def __init__(self) -> None:
+ self.receipt = None
+ self.complete_calls = 0
+
+ async def reserve(self, *, identity, payload):
+ if self.receipt is not None:
+ return {
+ "status": "idempotent_waiting_tool",
+ "claimed": False,
+ "identity": identity.public_dict(),
+ "receipt": self.receipt,
+ }
+ return {
+ "status": "reserved",
+ "claimed": True,
+ "claim_token": "claim-token-1",
+ "identity": identity.public_dict(),
+ "receipt": None,
+ }
+
+ async def complete(self, *, identity, dispatch_receipt, **_kwargs):
+ self.complete_calls += 1
+ self.receipt = {
+ **build_agent99_dispatch_receipt_envelope(
+ identity=identity,
+ dispatch_receipt=dispatch_receipt,
+ ),
+ "receipt_persisted": True,
+ }
+ return self.receipt
+
+ async def mark_delivery_attempt_started(self, *, identity, **_kwargs):
+ self.receipt = {
+ "status": "dispatch_delivery_unknown_reconcile_only",
+ "identity": identity.public_dict(),
+ "retry_policy": {"safe_to_retry": False, "reconcile_only": True},
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ }
+ return self.receipt
+
+
+@pytest.mark.asyncio
+async def test_backup_bridge_postgres_identity_deduplicates_transport(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ ledger = _FakeDispatchLedger()
+ transported: list[dict[str, object]] = []
+
+ async def lock(*_args, **_kwargs):
+ return {"acquired": True, "reason": "acquired"}
+
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger",
+ lambda: ledger,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight",
+ lock,
+ )
+ monkeypatch.setattr(
+ "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt",
+ lambda payload: transported.append(payload)
+ or {
+ "schema_version": "agent99_sre_dispatch_receipt_v1",
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": str(payload["id"]),
+ "kind": str(payload["kind"]),
+ "accepted": True,
+ "inbox_triggered": True,
+ "delivery_certainty": "delivered",
+ },
+ )
+ common = {
+ "alertname": "BackupCredentialEscrowEvidenceMissing",
+ "severity": "critical",
+ "namespace": "awoooi-prod",
+ "target_resource": "backup_restore",
+ "message": "backup freshness and escrow evidence missing",
+ "labels": {"event_type": "backup_restore_escrow_signal"},
+ "fingerprint": "stable-backup-source-fingerprint",
+ "project_id": "awoooi",
+ "incident_id": "INC-20260711-BACKUP",
+ "approval_id": "12345678-1234-5678-9234-567812345678",
+ "route_id": "agent99:backup_health:BackupCheck",
+ "work_item_id": "backupcheck:awoooi:stable",
+ }
+
+ first = await bridge_alertmanager_to_agent99(
+ alert_id="backup-alert-1",
+ **common,
+ )
+ replay = await bridge_alertmanager_to_agent99(
+ alert_id="backup-alert-2",
+ **common,
+ )
+
+ assert first["status"] == "dispatched"
+ assert replay["status"] == "deduplicated"
+ assert first["identity"] == replay["identity"]
+ assert first["dispatchPerformed"] is True
+ assert replay["dispatchPerformed"] is False
+ assert ledger.complete_calls == 1
+ assert len(transported) == 1
+ assert transported[0]["kind"] == "backup_health"
+ assert transported[0]["suggestedMode"] == "BackupCheck"
+ assert transported[0]["controlledApply"] is False
+ assert "do not run backup/restore" in str(transported[0]["instruction"])
diff --git a/apps/api/tests/test_backup_restore_legacy_backfill_job.py b/apps/api/tests/test_backup_restore_legacy_backfill_job.py
new file mode 100644
index 000000000..c92529f8d
--- /dev/null
+++ b/apps/api/tests/test_backup_restore_legacy_backfill_job.py
@@ -0,0 +1,1383 @@
+from __future__ import annotations
+
+import json
+from datetime import UTC, datetime, timedelta
+from pathlib import Path
+from typing import Any
+from unittest.mock import AsyncMock
+from uuid import NAMESPACE_URL, UUID, uuid5
+
+import pytest
+
+from src.core.context import (
+ clear_project_context,
+ get_current_project_id,
+ set_project_context,
+)
+from src.jobs import backup_restore_legacy_backfill_job as job
+
+
+class _Result:
+ def __init__(
+ self,
+ *,
+ row: dict[str, Any] | None = None,
+ rows: list[dict[str, Any]] | None = None,
+ value: Any = None,
+ ) -> None:
+ self.row = row
+ self.rows = rows or []
+ self.value = value
+
+ def mappings(self) -> _Result:
+ return self
+
+ def one_or_none(self) -> dict[str, Any] | None:
+ return self.row
+
+ def all(self) -> list[dict[str, Any]]:
+ return self.rows
+
+ def scalar_one_or_none(self) -> Any:
+ return self.value
+
+
+class _DbContext:
+ def __init__(self, db: Any) -> None:
+ self.db = db
+
+ async def __aenter__(self) -> Any:
+ return self.db
+
+ async def __aexit__(self, *_args: Any) -> bool:
+ return False
+
+
+def _source_row(index: int) -> dict[str, Any]:
+ return {
+ "message_id": uuid5(NAMESPACE_URL, f"legacy-backup-message:{index}"),
+ "queued_at": datetime(2026, 7, 11, tzinfo=UTC) + timedelta(seconds=index),
+ "source_fingerprint": (
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ),
+ "event_type": job.BACKUP_RESTORE_EVENT_TYPE,
+ "lane": job.BACKUP_RESTORE_LANE,
+ "target": "backup_restore",
+ }
+
+
+def _claim_from_row(index: int) -> job.LegacyBackupClaim:
+ item = job._item_payload_from_row(
+ project_id="awoooi",
+ row=_source_row(index),
+ )
+ return job.LegacyBackupClaim(
+ run_id=UUID(item["backfill_run_id"]),
+ claim_token=f"claim-{index}",
+ attempt_count=1,
+ max_attempts=5,
+ item=item,
+ )
+
+
+def _card_from_claim(claim: job.LegacyBackupClaim) -> job.LegacyBackupCard:
+ item = claim.item
+ return job.LegacyBackupCard(
+ message_id=str(item["source_message_id"]),
+ queued_at=str(item["source_queued_at"]),
+ source_fingerprint=str(item["source_fingerprint"]),
+ event_type=job.BACKUP_RESTORE_EVENT_TYPE,
+ lane=job.BACKUP_RESTORE_LANE,
+ target="backup_restore",
+ source_envelope={
+ "ai_automation_alert_card": {
+ "event_type": job.BACKUP_RESTORE_EVENT_TYPE,
+ "lane": job.BACKUP_RESTORE_LANE,
+ "target": "backup_restore",
+ },
+ "source_refs": {"fingerprints": [item["source_fingerprint"]]},
+ },
+ )
+
+
+def _durable_dispatch_result(kwargs: dict[str, Any]) -> dict[str, Any]:
+ owner = job.build_backup_restore_ingress_owner(
+ project_id=kwargs["project_id"],
+ source_fingerprint=kwargs["source_fingerprint"],
+ source_event_fingerprint=kwargs["source_event_fingerprint"],
+ source_started_at=kwargs["source_started_at"],
+ )
+ run_id = uuid5(
+ NAMESPACE_URL,
+ f"agent99-backupcheck:{kwargs['source_event_fingerprint']}",
+ )
+ return {
+ "status": "backupcheck_dispatched_verifier_pending",
+ "owner": {
+ "incident_id": owner["incident_id"],
+ "approval_id": owner["approval_id"],
+ "work_item_id": owner["work_item_id"],
+ "route_id": job.BACKUP_RESTORE_AGENT99_ROUTE_ID,
+ "kind": "backup_health",
+ "mode": "BackupCheck",
+ "controlled_apply": False,
+ "read_only": True,
+ },
+ "identity": {
+ "incident_id": owner["incident_id"],
+ "run_id": str(run_id),
+ "trace_id": f"trace-{run_id}",
+ "work_item_id": owner["work_item_id"],
+ "idempotency_key": kwargs["source_event_fingerprint"],
+ },
+ "accepted": True,
+ "receipt_persisted": True,
+ "dispatch_performed": True,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "production_write_performed": False,
+ "verifier": {"status": "pending", "receipt": {"status": "pending"}},
+ }
+
+
+def test_legacy_occurrence_identity_is_stable_and_source_specific() -> None:
+ kwargs = {
+ "project_id": "awoooi",
+ "message_id": "2f0eb0e7-a5b2-55ab-8d48-397c19998725",
+ "source_fingerprint": "legacy-fingerprint-1",
+ "queued_at": "2026-07-11T10:00:00+00:00",
+ }
+ first = job.build_legacy_backup_occurrence(**kwargs)
+ replay = job.build_legacy_backup_occurrence(**kwargs)
+ next_message = job.build_legacy_backup_occurrence(
+ **{**kwargs, "message_id": "22c0b48d-9674-543d-bf0b-ec25f1386406"}
+ )
+ next_fingerprint = job.build_legacy_backup_occurrence(
+ **{**kwargs, "source_fingerprint": "legacy-fingerprint-2"}
+ )
+
+ assert first == replay
+ for other in (next_message, next_fingerprint):
+ assert other["occurrence_id"] != first["occurrence_id"]
+ assert other["incident_id"] != first["incident_id"]
+ assert other["work_item_id"] != first["work_item_id"]
+ assert other["backfill_run_id"] != first["backfill_run_id"]
+ assert first["source_message_id"] in first["occurrence_id"]
+ assert first["dispatch_source_fingerprint"].startswith("legacy-backup-occurrence:")
+ assert first["controlled_apply"] is False
+ assert first["telegram_dispatch_performed"] is False
+
+
+def test_cursor_never_reports_completed_when_any_item_failed() -> None:
+ detail = {
+ "source_exhausted": True,
+ "scope_cap_reached": False,
+ "failed_total": 1,
+ }
+
+ assert job._cursor_terminal_state(detail, remaining=0) == (
+ "blocked_failed_items_fail_closed",
+ "failed",
+ )
+ assert job._cursor_terminal_state(
+ {**detail, "failed_total": 0},
+ remaining=0,
+ ) == ("completed", "completed")
+ assert job._cursor_terminal_state(
+ {**detail, "scope_cap_reached": True},
+ remaining=0,
+ ) == ("blocked_scope_cap", "waiting_tool")
+ assert job._cursor_terminal_state(
+ {**detail, "failed_total": 0, "source_drift_detected": True},
+ remaining=0,
+ ) == ("blocked_source_drift_fail_closed", "failed")
+ assert job._cursor_terminal_state(
+ {
+ **detail,
+ "failed_total": 0,
+ "source_total_at_start": 98,
+ "item_total": 97,
+ },
+ remaining=0,
+ ) == ("blocked_accounting_drift_fail_closed", "failed")
+
+
+@pytest.mark.asyncio
+async def test_116_rows_progress_through_two_bounded_durable_cohorts(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ rows = [_source_row(index) for index in range(116)]
+
+ class FakeDb:
+ def __init__(self) -> None:
+ self.cursor_detail: dict[str, Any] | None = None
+ self.snapshot_reads = 0
+ self.snapshot_sizes: list[int] = []
+ self.idempotency: set[str] = set()
+ self.item_total = 0
+ self.completed_total = 0
+
+ async def execute(
+ self,
+ statement: Any,
+ params: dict[str, Any],
+ ) -> _Result:
+ assert params["project_id"] == "awoooi"
+ if statement is job._CURSOR_INSERT_SQL:
+ if self.cursor_detail is None:
+ self.cursor_detail = json.loads(params["error_detail"])
+ return _Result()
+ if statement is job._CURSOR_SELECT_SQL:
+ assert self.cursor_detail is not None
+ return _Result(
+ row={
+ "run_id": job._cursor_run_id("awoooi"),
+ "state": "waiting_tool",
+ "error_detail": json.dumps(self.cursor_detail),
+ }
+ )
+ if statement is job._SOURCE_SNAPSHOT_SQL:
+ assert params["limit"] == 100
+ upper = rows[-1]
+ if self.snapshot_reads == 0:
+ source_rows = rows[:100]
+ remaining = 116
+ assert params["source_upper_queued_at"] is None
+ else:
+ source_rows = rows[100:]
+ remaining = 16
+ assert params["source_upper_message_id"] == str(upper["message_id"])
+ assert isinstance(params["source_upper_queued_at"], datetime)
+ assert params["source_upper_queued_at"] == upper["queued_at"]
+ self.snapshot_reads += 1
+ self.snapshot_sizes.append(len(source_rows))
+ return _Result(
+ rows=[
+ {
+ **row,
+ "source_total_in_scope": remaining,
+ "source_upper_message_id": upper["message_id"],
+ "source_upper_queued_at": upper["queued_at"],
+ }
+ for row in source_rows
+ ]
+ )
+ if statement is job._ITEM_RUN_INSERT_SQL:
+ return _Result()
+ if statement is job._ITEM_IDEMPOTENCY_INSERT_SQL:
+ key = str(params["provider_event_id"])
+ if key in self.idempotency:
+ return _Result(value=None)
+ self.idempotency.add(key)
+ return _Result(value=params["run_id"])
+ if statement is job._ITEM_STATE_COUNTS_SQL:
+ return _Result(
+ row={
+ "item_total": self.item_total,
+ "remaining_total": 0,
+ "completed_total": self.completed_total,
+ "failed_total": 0,
+ }
+ )
+ if statement is job._CURSOR_UPDATE_SQL:
+ self.cursor_detail = json.loads(params["error_detail"])
+ return _Result()
+ raise AssertionError(f"unexpected statement: {statement}")
+
+ db = FakeDb()
+ monkeypatch.setattr(job, "get_db_context", lambda _project_id: _DbContext(db))
+
+ first_cohort_pages = []
+ for _ in range(5):
+ result = await job._reserve_legacy_backup_page(
+ project_id="awoooi",
+ scan_limit=20,
+ max_rows=100,
+ )
+ first_cohort_pages.append(result["scanned"])
+
+ assert first_cohort_pages == [20, 20, 20, 20, 20]
+ assert result["source_total_at_start"] == 116
+ assert result["source_exhausted"] is False
+ assert result["cohort_exhausted"] is True
+ assert result["scope_cap_reached"] is False
+ assert len(db.idempotency) == 100
+
+ db.item_total = db.completed_total = 100
+ first_terminal = await job._update_cursor_counters(
+ project_id="awoooi",
+ completed=100,
+ deduplicated=0,
+ failed=0,
+ retried=0,
+ )
+ assert first_terminal["status"] == "cohort_completed_next_pending"
+ assert first_terminal["completed_cohort_count"] == 1
+ assert first_terminal["source_snapshot"] is None
+
+ second = await job._reserve_legacy_backup_page(
+ project_id="awoooi",
+ scan_limit=20,
+ max_rows=100,
+ )
+ assert second["scanned"] == second["reserved"] == 16
+ assert second["source_exhausted"] is True
+ assert second["cohort_index"] == 2
+ assert second["cohort_source_total"] == 16
+ assert len(db.idempotency) == 116
+
+ db.item_total = db.completed_total = 116
+ final = await job._update_cursor_counters(
+ project_id="awoooi",
+ completed=16,
+ deduplicated=0,
+ failed=0,
+ retried=0,
+ )
+
+ assert final["status"] == "completed"
+ assert final["completed_cohort_count"] == 2
+ assert final["source_total_at_start"] == 116
+ assert final["scanned_total"] == 116
+ assert db.snapshot_reads == 2
+ assert db.snapshot_sizes == [100, 16]
+ assert max(db.snapshot_sizes) <= 100
+
+
+@pytest.mark.asyncio
+async def test_reserver_snapshots_exact_98_once_and_advances_durable_cursor(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ rows = [_source_row(index) for index in range(98)]
+
+ class FakeDb:
+ def __init__(self) -> None:
+ self.cursor_detail: dict[str, Any] | None = None
+ self.snapshot_reads = 0
+ self.item_runs: set[str] = set()
+ self.idempotency: set[str] = set()
+
+ async def execute(
+ self,
+ statement: Any,
+ params: dict[str, Any],
+ ) -> _Result:
+ assert params["project_id"] == "awoooi"
+ if statement is job._CURSOR_INSERT_SQL:
+ if self.cursor_detail is None:
+ self.cursor_detail = json.loads(params["error_detail"])
+ return _Result()
+ if statement is job._CURSOR_SELECT_SQL:
+ assert self.cursor_detail is not None
+ return _Result(
+ row={
+ "run_id": params["run_id"],
+ "state": "waiting_tool",
+ "error_detail": json.dumps(self.cursor_detail),
+ }
+ )
+ if statement is job._SOURCE_SNAPSHOT_SQL:
+ self.snapshot_reads += 1
+ assert params["limit"] == 100
+ return _Result(rows=rows)
+ if statement is job._ITEM_RUN_INSERT_SQL:
+ self.item_runs.add(str(params["run_id"]))
+ return _Result()
+ if statement is job._ITEM_IDEMPOTENCY_INSERT_SQL:
+ key = str(params["provider_event_id"])
+ if key in self.idempotency:
+ return _Result(value=None)
+ self.idempotency.add(key)
+ return _Result(value=params["run_id"])
+ if statement is job._CURSOR_UPDATE_SQL:
+ self.cursor_detail = json.loads(params["error_detail"])
+ return _Result()
+ raise AssertionError(f"unexpected statement: {statement}")
+
+ db = FakeDb()
+ monkeypatch.setattr(job, "get_db_context", lambda _project_id: _DbContext(db))
+
+ batches = []
+ for _ in range(10):
+ result = await job._reserve_legacy_backup_page(
+ project_id="awoooi",
+ scan_limit=17,
+ max_rows=100,
+ )
+ batches.append(result["scanned"])
+ if result["source_exhausted"]:
+ break
+
+ assert batches == [17, 17, 17, 17, 17, 13]
+ assert db.snapshot_reads == 1
+ assert len(db.item_runs) == len(db.idempotency) == 98
+ assert db.cursor_detail is not None
+ assert db.cursor_detail["source_total_at_start"] == 98
+ assert db.cursor_detail["scanned_total"] == 98
+ assert db.cursor_detail["source_exhausted"] is True
+ assert db.cursor_detail["source_snapshot_sha256"]
+
+ replay = await job._reserve_legacy_backup_page(
+ project_id="awoooi",
+ scan_limit=17,
+ max_rows=100,
+ )
+ assert replay["scanned"] == replay["reserved"] == 0
+ assert db.snapshot_reads == 1
+
+
+@pytest.mark.asyncio
+async def test_claim_sweeps_expired_max_attempt_before_selecting_retryable_work(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ statements: list[Any] = []
+
+ class FakeDb:
+ async def execute(
+ self,
+ statement: Any,
+ params: dict[str, Any],
+ ) -> _Result:
+ statements.append(statement)
+ assert params["project_id"] == "awoooi"
+ assert params["agent_id"] == job.BACKFILL_ITEM_AGENT_ID
+ if statement is job._EXHAUST_EXPIRED_CLAIMS_SQL:
+ return _Result(rows=[{"run_id": _claim_from_row(90).run_id}])
+ if statement is job._CLAIM_SQL:
+ assert params["limit"] == 20
+ return _Result(rows=[])
+ raise AssertionError(f"unexpected statement: {statement}")
+
+ monkeypatch.setattr(
+ job,
+ "get_db_context",
+ lambda _project_id: _DbContext(FakeDb()),
+ )
+
+ claims = await job._claim_legacy_backup_items(project_id="awoooi", limit=25)
+
+ assert claims == []
+ assert statements == [job._EXHAUST_EXPIRED_CLAIMS_SQL, job._CLAIM_SQL]
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("item_total", "completed_total", "expected_status", "expected_cursor_state"),
+ [
+ (98, 98, "completed", "completed"),
+ (97, 97, "blocked_accounting_drift_fail_closed", "failed"),
+ ],
+)
+async def test_terminal_status_uses_authoritative_item_state_counts(
+ monkeypatch: pytest.MonkeyPatch,
+ item_total: int,
+ completed_total: int,
+ expected_status: str,
+ expected_cursor_state: str,
+) -> None:
+ updated_state = ""
+ detail = job._initial_cursor("awoooi")
+ detail.update(
+ {
+ "source_total_at_start": 98,
+ "source_exhausted": True,
+ "scanned_total": 98,
+ }
+ )
+
+ class FakeDb:
+ async def execute(
+ self,
+ statement: Any,
+ params: dict[str, Any],
+ ) -> _Result:
+ nonlocal updated_state
+ if statement is job._CURSOR_SELECT_SQL:
+ return _Result(row={"error_detail": json.dumps(detail)})
+ if statement is job._ITEM_STATE_COUNTS_SQL:
+ return _Result(
+ row={
+ "item_total": item_total,
+ "remaining_total": 0,
+ "completed_total": completed_total,
+ "failed_total": 0,
+ }
+ )
+ if statement is job._CURSOR_UPDATE_SQL:
+ updated_state = str(params["state"])
+ return _Result()
+ raise AssertionError(f"unexpected statement: {statement}")
+
+ monkeypatch.setattr(
+ job,
+ "get_db_context",
+ lambda _project_id: _DbContext(FakeDb()),
+ )
+
+ result = await job._update_cursor_counters(
+ project_id="awoooi",
+ completed=0,
+ deduplicated=0,
+ failed=0,
+ retried=0,
+ )
+
+ assert result["status"] == expected_status
+ assert result["completed_total"] == completed_total
+ assert updated_state == expected_cursor_state
+
+
+@pytest.mark.asyncio
+async def test_exact_98_legacy_cards_replay_in_bounded_restart_safe_batches(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ rows = [_source_row(index) for index in range(98)]
+ pending: list[job.LegacyBackupClaim] = []
+ projected: dict[str, dict[str, Any]] = {}
+ finished: dict[str, str] = {}
+ processor_calls: list[dict[str, Any]] = []
+ reserve_batch_sizes: list[int] = []
+ scan_index = 0
+ source_exhausted = False
+ cumulative = {
+ "completed_total": 0,
+ "deduplicated_total": 0,
+ "failed_total": 0,
+ "retry_total": 0,
+ }
+
+ async def reserve(
+ *, project_id: str, scan_limit: int, max_rows: int
+ ) -> dict[str, Any]:
+ nonlocal scan_index, source_exhausted
+ assert project_id == "awoooi"
+ assert max_rows == 100
+ limit = min(scan_limit, job.MAX_BATCH_LIMIT, max_rows - scan_index)
+ page = rows[scan_index : scan_index + limit]
+ for row in page:
+ item = job._item_payload_from_row(project_id=project_id, row=row)
+ pending.append(
+ job.LegacyBackupClaim(
+ run_id=UUID(item["backfill_run_id"]),
+ claim_token=f"claim-{item['source_message_id']}",
+ attempt_count=1,
+ max_attempts=5,
+ item=item,
+ )
+ )
+ scan_index += len(page)
+ source_exhausted = scan_index == len(rows)
+ reserve_batch_sizes.append(len(page))
+ return {
+ "scanned": len(page),
+ "reserved": len(page),
+ "deduplicated": 0,
+ "source_total_at_start": 98,
+ "source_exhausted": source_exhausted,
+ "source_drift_detected": False,
+ "scope_cap_reached": False,
+ "cursor": {
+ "queued_at": (
+ str(page[-1]["queued_at"]) if page else str(rows[-1]["queued_at"])
+ ),
+ "message_id": (
+ str(page[-1]["message_id"]) if page else str(rows[-1]["message_id"])
+ ),
+ },
+ }
+
+ async def claim(*, project_id: str, limit: int) -> list[job.LegacyBackupClaim]:
+ assert project_id == "awoooi"
+ page = pending[:limit]
+ del pending[:limit]
+ return page
+
+ async def load(
+ *, project_id: str, claim: job.LegacyBackupClaim
+ ) -> job.LegacyBackupCard:
+ assert project_id == "awoooi"
+ return _card_from_claim(claim)
+
+ async def processor(**kwargs: Any) -> dict[str, Any]:
+ assert get_current_project_id() == "awoooi"
+ assert kwargs["project_id"] == "awoooi"
+ assert kwargs["labels"]["event_type"] == job.BACKUP_RESTORE_EVENT_TYPE
+ assert kwargs["labels"]["lane"] == job.BACKUP_RESTORE_LANE
+ assert kwargs["source_event_fingerprint"].startswith("legacy-backup:")
+ processor_calls.append(kwargs)
+ return _durable_dispatch_result(kwargs)
+
+ async def persist(
+ *,
+ project_id: str,
+ claim: job.LegacyBackupClaim,
+ card: job.LegacyBackupCard,
+ occurrence: dict[str, Any],
+ projection: dict[str, Any],
+ ) -> str:
+ assert project_id == "awoooi"
+ assert claim.run_id == UUID(occurrence["backfill_run_id"])
+ assert occurrence["source_message_id"] == card.message_id
+ assert projection["telegram_dispatch_performed"] is False
+ assert projection["runtime_execution_authorized"] is False
+ assert card.message_id not in projected
+ projected[card.message_id] = dict(projection)
+ return "projected"
+
+ async def finish(
+ *,
+ project_id: str,
+ claim: job.LegacyBackupClaim,
+ state: str,
+ result: dict[str, Any],
+ error_code: str | None,
+ ) -> bool:
+ assert project_id == "awoooi"
+ assert state == "completed"
+ assert error_code is None
+ assert result["controlled_apply"] is False
+ assert result["telegram_dispatch_performed"] is False
+ assert result["backfill_run_id"] == str(claim.run_id)
+ finished[str(claim.run_id)] = state
+ return True
+
+ async def update_cursor(
+ *,
+ project_id: str,
+ completed: int,
+ deduplicated: int,
+ failed: int,
+ retried: int,
+ ) -> dict[str, Any]:
+ assert project_id == "awoooi"
+ cumulative["completed_total"] += completed
+ cumulative["deduplicated_total"] += deduplicated
+ cumulative["failed_total"] += failed
+ cumulative["retry_total"] += retried
+ return {
+ **cumulative,
+ "remaining_item_total": len(pending),
+ "status": (
+ "completed" if source_exhausted and not pending else "in_progress"
+ ),
+ }
+
+ monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve)
+ monkeypatch.setattr(job, "_claim_legacy_backup_items", claim)
+ monkeypatch.setattr(job, "_load_legacy_backup_card", load)
+ monkeypatch.setattr(job, "_persist_legacy_dispatch_projection", persist)
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+ monkeypatch.setattr(job, "_update_cursor_counters", update_cursor)
+
+ results: list[dict[str, Any]] = []
+ for _ in range(10):
+ result = await job.run_backup_restore_legacy_backfill_once(
+ project_id="awoooi",
+ scan_limit=17,
+ process_limit=17,
+ max_rows=100,
+ processor=processor,
+ )
+ results.append(result)
+ if result["status"] == "completed":
+ break
+
+ assert results[-1]["status"] == "completed"
+ assert sum(result["scanned"] for result in results) == 98
+ assert sum(result["dispatch_total"] for result in results) == 98
+ assert cumulative == {
+ "completed_total": 98,
+ "deduplicated_total": 0,
+ "failed_total": 0,
+ "retry_total": 0,
+ }
+ assert len(processor_calls) == len(projected) == len(finished) == 98
+ assert max(reserve_batch_sizes) == 17
+ assert reserve_batch_sizes == [17, 17, 17, 17, 17, 13]
+ assert (
+ len({call["labels"]["legacy_source_message_id"] for call in processor_calls})
+ == 98
+ )
+ assert len({call["source_fingerprint"] for call in processor_calls}) == 98
+ assert len({value["incident_id"] for value in projected.values()}) == 98
+ assert len({value["work_item_id"] for value in projected.values()}) == 98
+ assert len({value["run_id"] for value in projected.values()}) == 98
+ assert {value["auth_mode"] for value in projected.values()} == {
+ "injected_processor"
+ }
+
+ replay = await job.run_backup_restore_legacy_backfill_once(
+ project_id="awoooi",
+ scan_limit=17,
+ process_limit=17,
+ max_rows=100,
+ processor=processor,
+ )
+ assert replay["status"] == "completed"
+ assert replay["scanned"] == replay["claimed"] == replay["dispatch_total"] == 0
+ assert len(processor_calls) == 98
+
+
+@pytest.mark.asyncio
+async def test_missing_source_fingerprint_fails_closed_without_processor(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ row = _source_row(1)
+ row["source_fingerprint"] = ""
+ item = job._item_payload_from_row(project_id="awoooi", row=row)
+ claim = job.LegacyBackupClaim(
+ run_id=UUID(item["backfill_run_id"]),
+ claim_token="claim-missing-fingerprint",
+ attempt_count=1,
+ max_attempts=5,
+ item=item,
+ )
+ card = job.LegacyBackupCard(
+ message_id=str(row["message_id"]),
+ queued_at=str(row["queued_at"]),
+ source_fingerprint="",
+ event_type=job.BACKUP_RESTORE_EVENT_TYPE,
+ lane=job.BACKUP_RESTORE_LANE,
+ target="backup_restore",
+ source_envelope={},
+ )
+ processor = AsyncMock()
+ finish = AsyncMock(return_value=True)
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+
+ assert outcome == "failed"
+ processor.assert_not_awaited()
+ assert finish.await_args.kwargs["state"] == "failed"
+ assert finish.await_args.kwargs["error_code"] == "E-BACKFILL-FINGERPRINT"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ "processor_result",
+ [
+ RuntimeError("relay unavailable"),
+ {
+ "status": "backupcheck_dispatch_or_readback_failed",
+ "accepted": False,
+ "receipt_persisted": False,
+ },
+ ],
+)
+async def test_transport_or_readback_failure_is_retryable_and_restores_context(
+ monkeypatch: pytest.MonkeyPatch,
+ processor_result: Exception | dict[str, Any],
+) -> None:
+ claim = _claim_from_row(7)
+ card = _card_from_claim(claim)
+ finish = AsyncMock(return_value=True)
+ persist = AsyncMock(return_value="projected")
+
+ async def processor(**_kwargs: Any) -> dict[str, Any]:
+ assert get_current_project_id() == "awoooi"
+ if isinstance(processor_result, Exception):
+ raise processor_result
+ return processor_result
+
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+ monkeypatch.setattr(job, "_persist_legacy_dispatch_projection", persist)
+ prior_tokens = set_project_context("parent-project", source="test")
+ try:
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+ assert get_current_project_id() == "parent-project"
+ finally:
+ clear_project_context(prior_tokens)
+
+ assert outcome == "retried"
+ persist.assert_not_awaited()
+ assert finish.await_args.kwargs["state"] == "waiting_tool"
+ assert finish.await_args.kwargs["error_code"] == "E-BACKFILL-RETRY"
+
+
+@pytest.mark.asyncio
+async def test_transport_retry_budget_exhaustion_is_terminal_fail_closed(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ original = _claim_from_row(8)
+ claim = job.LegacyBackupClaim(
+ run_id=original.run_id,
+ claim_token=original.claim_token,
+ attempt_count=original.max_attempts,
+ max_attempts=original.max_attempts,
+ item=original.item,
+ )
+ finish = AsyncMock(return_value=True)
+
+ async def processor(**_kwargs: Any) -> dict[str, Any]:
+ raise RuntimeError("relay unavailable")
+
+ monkeypatch.setattr(
+ job,
+ "_load_legacy_backup_card",
+ AsyncMock(return_value=_card_from_claim(claim)),
+ )
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+
+ assert outcome == "failed"
+ assert finish.await_args.kwargs["state"] == "failed"
+ assert finish.await_args.kwargs["error_code"] == "E-BACKFILL-EXHAUSTED"
+
+
+def test_existing_projection_deduplicates_only_exact_durable_identity() -> None:
+ claim = _claim_from_row(4)
+ card = _card_from_claim(claim)
+ occurrence = job.build_legacy_backup_occurrence(
+ project_id="awoooi",
+ message_id=card.message_id,
+ source_fingerprint=card.source_fingerprint,
+ queued_at=card.queued_at,
+ )
+ card.source_envelope["agent99_dispatch_receipt"] = {
+ "schema_version": job.PROJECTION_SCHEMA_VERSION,
+ "incident_id": occurrence["incident_id"],
+ "work_item_id": occurrence["work_item_id"],
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "receipt_persisted": True,
+ "run_id": "agent99-run-4",
+ "trace_id": "trace-agent99-run-4",
+ }
+ card.source_envelope["backup_restore_backfill"] = {
+ "occurrence_id": occurrence["occurrence_id"]
+ }
+
+ assert job._existing_projection_matches(card, occurrence=occurrence) is True
+
+ partial = dict(card.source_envelope["agent99_dispatch_receipt"])
+ partial["receipt_persisted"] = False
+ card.source_envelope["agent99_dispatch_receipt"] = partial
+ assert job._existing_projection_matches(card, occurrence=occurrence) is False
+
+ card.source_envelope["agent99_dispatch_receipt"]["receipt_persisted"] = True
+ card.source_envelope["agent99_dispatch_receipt"]["incident_id"] = "INC-WRONG"
+ with pytest.raises(ValueError, match="identity_conflict"):
+ job._existing_projection_matches(card, occurrence=occurrence)
+
+
+@pytest.mark.asyncio
+async def test_claim_completion_compare_and_set_failure_reports_lease_lost(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ claim = _claim_from_row(9)
+ card = _card_from_claim(claim)
+ occurrence = job.build_legacy_backup_occurrence(
+ project_id="awoooi",
+ message_id=card.message_id,
+ source_fingerprint=card.source_fingerprint,
+ queued_at=card.queued_at,
+ )
+ card.source_envelope["agent99_dispatch_receipt"] = {
+ "schema_version": job.PROJECTION_SCHEMA_VERSION,
+ "incident_id": occurrence["incident_id"],
+ "work_item_id": occurrence["work_item_id"],
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "receipt_persisted": True,
+ "run_id": "agent99-run-lease-lost",
+ "trace_id": "trace-agent99-run-lease-lost",
+ }
+ processor = AsyncMock()
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(
+ job,
+ "_finish_backfill_claim",
+ AsyncMock(return_value=False),
+ )
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+
+ assert outcome == "lease_lost"
+ processor.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+ ("readback", "expected_status"),
+ [
+ (
+ {
+ "claim_active": False,
+ "source_exists": True,
+ "source_identity_matches": True,
+ },
+ "lease_lost",
+ ),
+ (
+ {
+ "claim_active": True,
+ "source_exists": True,
+ "source_identity_matches": False,
+ },
+ "source_identity_drift",
+ ),
+ (
+ {
+ "claim_active": True,
+ "source_exists": True,
+ "source_identity_matches": True,
+ },
+ "projection_conflict",
+ ),
+ ],
+)
+async def test_projection_write_classifies_lease_and_source_identity_conflicts(
+ monkeypatch: pytest.MonkeyPatch,
+ readback: dict[str, Any],
+ expected_status: str,
+) -> None:
+ claim = _claim_from_row(10)
+ card = _card_from_claim(claim)
+ occurrence = job.build_legacy_backup_occurrence(
+ project_id="awoooi",
+ message_id=card.message_id,
+ source_fingerprint=card.source_fingerprint,
+ queued_at=card.queued_at,
+ )
+
+ class FakeDb:
+ async def execute(
+ self,
+ statement: Any,
+ params: dict[str, Any],
+ ) -> _Result:
+ assert params["project_id"] == "awoooi"
+ assert params["claim_run_id"] == claim.run_id
+ assert params["claim_token"] == claim.claim_token
+ assert isinstance(params["source_queued_at"], datetime)
+ assert params["source_queued_at"].isoformat() == claim.item["source_queued_at"]
+ assert params["source_fingerprint"] == claim.item["source_fingerprint"]
+ assert params["source_lane"] == claim.item["lane"]
+ assert params["source_target"] == claim.item["target"]
+ if statement is job._SOURCE_PROJECTION_UPDATE_SQL:
+ return _Result(value=None)
+ if statement is job._PROJECTION_CONFLICT_READBACK_SQL:
+ return _Result(row=readback)
+ raise AssertionError(f"unexpected statement: {statement}")
+
+ monkeypatch.setattr(
+ job,
+ "get_db_context",
+ lambda _project_id: _DbContext(FakeDb()),
+ )
+
+ status = await job._persist_legacy_dispatch_projection(
+ project_id="awoooi",
+ claim=claim,
+ card=card,
+ occurrence=occurrence,
+ projection={
+ "run_id": "agent99-run-10",
+ "trace_id": "agent99-trace-10",
+ "auth_mode": "lan_allowlist",
+ },
+ )
+
+ assert status == expected_status
+
+
+@pytest.mark.asyncio
+async def test_lost_lease_cannot_project_after_dispatch(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ claim = _claim_from_row(11)
+ card = _card_from_claim(claim)
+ finish = AsyncMock()
+
+ async def processor(**kwargs: Any) -> dict[str, Any]:
+ return _durable_dispatch_result(kwargs)
+
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(
+ job,
+ "_persist_legacy_dispatch_projection",
+ AsyncMock(return_value="lease_lost"),
+ )
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+
+ assert outcome == "lease_lost"
+ finish.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_source_drift_after_dispatch_is_terminal_and_fail_visible(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ claim = _claim_from_row(12)
+ card = _card_from_claim(claim)
+ finish = AsyncMock(return_value=True)
+
+ async def processor(**kwargs: Any) -> dict[str, Any]:
+ return _durable_dispatch_result(kwargs)
+
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(
+ job,
+ "_persist_legacy_dispatch_projection",
+ AsyncMock(return_value="source_identity_drift"),
+ )
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ )
+
+ assert outcome == "failed"
+ assert finish.await_args.kwargs["state"] == "failed"
+ assert finish.await_args.kwargs["error_code"] == "E-BACKFILL-IDENTITY"
+ assert finish.await_args.kwargs["result"]["status"] == (
+ "source_identity_drift_after_dispatch_fail_closed"
+ )
+
+
+@pytest.mark.asyncio
+async def test_lan_allowlist_auth_mode_is_persisted_in_dispatch_projection(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ claim = _claim_from_row(10)
+ card = _card_from_claim(claim)
+ persist = AsyncMock(return_value="projected")
+ finish = AsyncMock(return_value=True)
+
+ async def processor(**kwargs: Any) -> dict[str, Any]:
+ return _durable_dispatch_result(kwargs)
+
+ monkeypatch.setattr(job, "_load_legacy_backup_card", AsyncMock(return_value=card))
+ monkeypatch.setattr(job, "_persist_legacy_dispatch_projection", persist)
+ monkeypatch.setattr(job, "_finish_backfill_claim", finish)
+
+ outcome = await job._replay_backfill_claim(
+ project_id="awoooi",
+ claim=claim,
+ processor=processor,
+ relay_auth_mode="lan_allowlist",
+ )
+
+ assert outcome == "completed"
+ projection = persist.await_args.kwargs["projection"]
+ assert projection["auth_mode"] == "lan_allowlist"
+ assert projection["controlled_apply"] is False
+ assert projection["telegram_dispatch_performed"] is False
+
+
+def test_sql_and_startup_contract_are_bounded_rls_scoped_and_no_telegram_send() -> None:
+ snapshot_sql = str(job._SOURCE_SNAPSHOT_SQL)
+ claim_sql = str(job._CLAIM_SQL)
+ exhaust_sql = str(job._EXHAUST_EXPIRED_CLAIMS_SQL)
+ idempotency_sql = str(job._ITEM_IDEMPOTENCY_INSERT_SQL)
+ projection_sql = str(job._SOURCE_PROJECTION_UPDATE_SQL)
+ conflict_readback_sql = str(job._PROJECTION_CONFLICT_READBACK_SQL)
+ source = Path(job.__file__).read_text(encoding="utf-8")
+ src_dir = Path(job.__file__).resolve().parent.parent
+ config_source = (src_dir / "core" / "config.py").read_text(encoding="utf-8")
+ main_source = (src_dir / "main.py").read_text(encoding="utf-8")
+ worker_source = (src_dir / "workers" / "signal_worker.py").read_text(
+ encoding="utf-8"
+ )
+
+ assert "m.project_id = :project_id" in snapshot_sql
+ assert "ORDER BY m.queued_at ASC, m.message_id ASC" in snapshot_sql
+ assert "{ai_automation_alert_card,target}" in snapshot_sql
+ assert "{agent99_dispatch_receipt,receipt_persisted}" in snapshot_sql
+ assert "{agent99_dispatch_receipt,run_id}" in snapshot_sql
+ assert "LIMIT :limit" in snapshot_sql
+ assert "COUNT(*) OVER () AS source_total_in_scope" in snapshot_sql
+ assert "source_upper_queued_at" in snapshot_sql
+ assert "FOR UPDATE SKIP LOCKED" in claim_sql
+ assert "attempt_count < max_attempts" in claim_sql
+ assert "attempt_count >= max_attempts" not in claim_sql
+ assert "state = 'waiting_tool'" in claim_sql
+ assert "lease_until < NOW()" in claim_sql
+ assert "attempt_count >= max_attempts" in exhaust_sql
+ assert "E-BACKFILL-LEASE-EXHAUSTED" in exhaust_sql
+ assert "error_code IS NULL" in exhaust_sql
+ assert "awooop_run_idempotency" in idempotency_sql
+ assert "project_id, channel_type, provider_event_id" in idempotency_sql
+ assert "m.project_id = :project_id" in projection_sql
+ assert "m.message_id = CAST(:message_id AS UUID)" in projection_sql
+ assert "item.worker_id = :claim_token" in projection_sql
+ assert "item.lease_until > clock_timestamp()" in projection_sql
+ assert "FOR UPDATE" in projection_sql
+ assert "m.queued_at = CAST(:source_queued_at AS TIMESTAMPTZ)" in projection_sql
+ assert "{source_refs,fingerprints,0}" in projection_sql
+ assert "{ai_automation_alert_card,lane}" in projection_sql
+ assert "{ai_automation_alert_card,target}" in projection_sql
+ assert "{agent99_dispatch_receipt,run_id}' = :run_id" in projection_sql
+ assert "CAST(:message_id AS TEXT)" in projection_sql
+ assert "source_identity_matches" in conflict_readback_sql
+ assert "item.worker_id = :claim_token" in conflict_readback_sql
+ assert job.MAX_BATCH_LIMIT == 20
+ assert job.MAX_SCOPE_ROWS == 100
+ assert "telegram_gateway" not in source
+ assert "subprocess" not in source
+ for mutation_call in (
+ "run_backup(",
+ "run_restore(",
+ "remote_delete(",
+ "change_retention(",
+ "write_escrow_marker(",
+ ):
+ assert mutation_call not in source
+ assert "ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL" in config_source
+ assert "BACKUP_RESTORE_LEGACY_BACKFILL_MAX_ROWS" in config_source
+ assert "run_backup_restore_legacy_backfill_loop" not in main_source
+ assert "signal_worker_backup_restore_legacy_backfill_loop_started" in worker_source
+ assert "build_backup_restore_backfill_relay_preflight" in worker_source
+ assert (
+ "signal_worker_backup_restore_legacy_backfill_loop_skipped_fail_closed"
+ in worker_source
+ )
+ assert "backup_restore_legacy_backfill_task.cancel()" in worker_source
+
+
+@pytest.mark.asyncio
+async def test_missing_project_context_performs_no_database_or_dispatch(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ reserve = AsyncMock()
+ processor = AsyncMock()
+ monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve)
+
+ result = await job.run_backup_restore_legacy_backfill_once(
+ project_id="",
+ processor=processor,
+ )
+
+ assert result["status"] == "project_context_missing_fail_closed"
+ assert result["dispatch_total"] == 0
+ assert result["runtime_execution_authorized"] is False
+ assert result["telegram_dispatch_performed"] is False
+ reserve.assert_not_awaited()
+ processor.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_default_processor_requires_trusted_relay_before_database_scan(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ reserve = AsyncMock()
+ monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve)
+ monkeypatch.setattr(job.settings, "AGENT99_SRE_ALERT_RELAY_URL", "")
+ monkeypatch.setattr(job.settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "")
+
+ result = await job.run_backup_restore_legacy_backfill_once(project_id="awoooi")
+
+ assert result["status"] == "relay_preflight_failed_fail_closed"
+ assert result["auth_mode"] == "blocked"
+ assert result["relay_preflight"]["endpoint_class"] == "untrusted_or_missing"
+ assert result["dispatch_total"] == 0
+ reserve.assert_not_awaited()
+
+
+@pytest.mark.parametrize(
+ ("relay_url", "relay_token", "ready", "auth_mode", "endpoint_class"),
+ [
+ ("", "", False, "blocked", "untrusted_or_missing"),
+ (
+ "http://192.168.0.99:8787/agent99/sre-alert/",
+ "",
+ True,
+ "lan_allowlist",
+ "repo_known_private_lan",
+ ),
+ (
+ "https://192.168.0.99:8787/agent99/sre-alert/",
+ "",
+ False,
+ "blocked",
+ "untrusted_or_missing",
+ ),
+ (
+ "http://192.168.0.99:8787/not-agent99/",
+ "",
+ False,
+ "blocked",
+ "untrusted_or_missing",
+ ),
+ (
+ "https://relay.example.test/agent99/sre-alert/",
+ "test-token",
+ True,
+ "token",
+ "configured_token_endpoint",
+ ),
+ (
+ "https://user@relay.example.test/agent99/sre-alert/",
+ "test-token",
+ False,
+ "blocked",
+ "untrusted_or_missing",
+ ),
+ (
+ "http://relay.example.test/agent99/sre-alert/",
+ "test-token",
+ False,
+ "blocked",
+ "untrusted_or_missing",
+ ),
+ (
+ "http://192.168.0.99:8787/not-agent99/",
+ "test-token",
+ False,
+ "blocked",
+ "untrusted_or_missing",
+ ),
+ (
+ "http://192.168.0.99:8787/agent99/sre-alert/",
+ "test-token",
+ True,
+ "lan_allowlist",
+ "repo_known_private_lan",
+ ),
+ ],
+)
+def test_relay_preflight_is_exact_and_public_safe(
+ relay_url: str,
+ relay_token: str,
+ ready: bool,
+ auth_mode: str,
+ endpoint_class: str,
+) -> None:
+ result = job.build_backup_restore_backfill_relay_preflight(
+ relay_url=relay_url,
+ relay_token=relay_token,
+ )
+
+ assert result["ready"] is ready
+ assert result["auth_mode"] == auth_mode
+ assert result["endpoint_class"] == endpoint_class
+ assert result["token_present"] is bool(relay_token)
+ assert result["stores_secret"] is False
+ assert relay_token not in json.dumps(result) if relay_token else True
+
+
+@pytest.mark.asyncio
+async def test_default_processor_accepts_repo_known_lan_without_token(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ reserve = AsyncMock(
+ return_value={
+ "scanned": 0,
+ "reserved": 0,
+ "deduplicated": 0,
+ "source_total_at_start": 0,
+ "source_exhausted": True,
+ "source_drift_detected": False,
+ "scope_cap_reached": False,
+ "cursor": {},
+ }
+ )
+ claim = AsyncMock(return_value=[])
+ update_cursor = AsyncMock(
+ return_value={"status": "completed", "remaining_item_total": 0}
+ )
+ monkeypatch.setattr(job, "_reserve_legacy_backup_page", reserve)
+ monkeypatch.setattr(job, "_claim_legacy_backup_items", claim)
+ monkeypatch.setattr(job, "_update_cursor_counters", update_cursor)
+ monkeypatch.setattr(
+ job.settings,
+ "AGENT99_SRE_ALERT_RELAY_URL",
+ "http://192.168.0.99:8787/agent99/sre-alert/",
+ )
+ monkeypatch.setattr(job.settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "")
+
+ result = await job.run_backup_restore_legacy_backfill_once(project_id="awoooi")
+
+ assert result["status"] == "completed"
+ assert result["auth_mode"] == "lan_allowlist"
+ assert result["relay_preflight"]["ready"] is True
+ reserve.assert_awaited_once()
+ claim.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_one_time_loop_exits_after_durable_snapshot_completes(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ tick = AsyncMock(
+ return_value={
+ "status": "completed",
+ "source_total_at_start": 98,
+ "telegram_dispatch_performed": False,
+ }
+ )
+ sleep = AsyncMock()
+ monkeypatch.setattr(job.settings, "ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL", True)
+ monkeypatch.setattr(
+ job.settings,
+ "AGENT99_SRE_ALERT_RELAY_URL",
+ "http://192.168.0.99:8787/agent99/sre-alert/",
+ )
+ monkeypatch.setattr(job.settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "")
+ monkeypatch.setattr(
+ job.settings,
+ "BACKUP_RESTORE_LEGACY_BACKFILL_STARTUP_SLEEP_SECONDS",
+ 0.0,
+ )
+ monkeypatch.setattr(job, "run_backup_restore_legacy_backfill_once", tick)
+ monkeypatch.setattr(job.asyncio, "sleep", sleep)
+
+ await job.run_backup_restore_legacy_backfill_loop()
+
+ tick.assert_awaited_once()
+ sleep.assert_awaited_once_with(0.0)
+
+
+@pytest.mark.asyncio
+async def test_default_enabled_loop_exits_without_sleep_when_relay_is_untrusted(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ tick = AsyncMock()
+ sleep = AsyncMock()
+ monkeypatch.setattr(job.settings, "ENABLE_BACKUP_RESTORE_LEGACY_BACKFILL", True)
+ monkeypatch.setattr(
+ job.settings,
+ "AGENT99_SRE_ALERT_RELAY_URL",
+ "http://192.168.0.98:8787/agent99/sre-alert/",
+ )
+ monkeypatch.setattr(job.settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "")
+ monkeypatch.setattr(job, "run_backup_restore_legacy_backfill_once", tick)
+ monkeypatch.setattr(job.asyncio, "sleep", sleep)
+
+ await job.run_backup_restore_legacy_backfill_loop()
+
+ tick.assert_not_awaited()
+ sleep.assert_not_awaited()
diff --git a/apps/api/tests/test_backup_restore_signal_automation.py b/apps/api/tests/test_backup_restore_signal_automation.py
new file mode 100644
index 000000000..a5656c1f4
--- /dev/null
+++ b/apps/api/tests/test_backup_restore_signal_automation.py
@@ -0,0 +1,369 @@
+from src.services.backup_restore_signal_automation import (
+ build_backup_restore_signal_automation_contract,
+ extract_backup_restore_evidence,
+)
+
+
+def test_backup_restore_contract_is_durable_and_fail_closed() -> None:
+ contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ evidence_text=(
+ 'Top evidence\n'
+ '├ alertname="BackupAggregateRunFailed" '
+ 'escrow_missing=5 restic failed\n'
+ '建議下一步'
+ ),
+ source_refs={
+ "fingerprints": [
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ]
+ },
+ receipt_id="97ff9f41-7cdc-4893-8aab-634286fb128c",
+ run_id="0b25cafa-00a4-520b-8946-a3ea467b13fc",
+ source_row_exists=True,
+ contract_origin="synthesized_at_read",
+ declared_gates=[
+ "controlled_playbook_queue",
+ "runtime_write_gate=controlled",
+ ],
+ declared_runtime_write_gate_count=1,
+ declared_runtime_write_gate_state="controlled",
+ declared_controlled_playbook_queue=True,
+ declared_learning_writeback_status="ready",
+ )
+
+ assert contract["work_item"]["work_item_id"].startswith("BRR-WI-")
+ assert contract["work_item"]["owner"] == "backup_dr"
+ assert contract["contract_origin"] == "synthesized_at_read"
+ assert contract["contract_persisted_at_send"] is False
+ assert contract["policy"]["policy_version"] == (
+ "backup_restore_readback_policy_v2"
+ )
+ assert contract["policy"]["effective_at"] == "2026-07-11T19:30:58+08:00"
+ assert contract["policy"]["source_hash"].startswith("sha256:")
+ provenance = contract["contract_provenance"]
+ assert provenance["source_contract_present_at_send"] is False
+ assert provenance["recorded_policy_present"] is False
+ assert provenance["recorded_immutable_declared_present"] is False
+ assert provenance["synthesized_at_read"] is True
+ assert contract["immutable_declared"]["runtime_write_gate_count"] == 1
+ assert contract["immutable_declared"]["controlled_playbook_queue"] is True
+ assert contract["effective_policy_projection"]["runtime_write_gate_count"] == 0
+ assert contract["work_item"]["status"] == "projected_open_evidence_failed"
+ assert contract["candidate"]["status"] == (
+ "projected_source_row_observed_dispatch_receipt_missing"
+ )
+ assert contract["candidate"]["execution_state"] == "not_started"
+ assert contract["candidate"]["agent99_dispatch"]["dispatched"] is False
+ assert contract["receipt"]["status"] == (
+ "projected_from_historical_source_row"
+ )
+ assert contract["receipt"]["source_row_exists"] is True
+ assert contract["receipt"]["contract_persisted_at_send"] is False
+ assert contract["receipt"]["is_agent99_execution_receipt"] is False
+ assert contract["verifier"]["status"] == (
+ "projected_dispatch_receipt_missing_verifier_not_started"
+ )
+ assert contract["verifier"]["terminal"] is False
+ assert contract["verifier"]["no_false_green"] is True
+ assert set(contract["verifier"]["failed_fields"]) == {
+ "backup_status",
+ "escrow",
+ }
+ assert "source_resolution_receipt" in contract["verifier"]["missing_fields"]
+ assert contract["runtime_write_allowed"] is False
+ assert contract["runtime_execution_authorized"] is False
+ assert contract["production_write_performed"] is False
+ assert "run_restore" in contract["prohibited_actions"]
+ assert "remote_delete" in contract["prohibited_actions"]
+ assert "retention_change" in contract["prohibited_actions"]
+ assert "escrow_marker_write" in contract["prohibited_actions"]
+ assert all(
+ {"asset_id", "owner", "status", "next_action"} <= set(asset)
+ for asset in contract["asset_ledger"]
+ )
+
+
+def test_backup_restore_contract_never_closes_without_source_resolution() -> None:
+ observations = [
+ {
+ "field": field,
+ "status": "healthy",
+ "observed": True,
+ "safe_value": "explicit_success",
+ }
+ for field in (
+ "backup_status",
+ "freshness",
+ "offsite_verify",
+ "escrow",
+ "restore_drill",
+ )
+ ]
+
+ contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ evidence_observations=observations,
+ receipt_id="durable-message",
+ run_id="durable-run",
+ source_row_exists=True,
+ contract_origin="synthesized_at_read",
+ )
+
+ assert contract["work_item"]["status"] == (
+ "projected_open_evidence_incomplete"
+ )
+ assert contract["verifier"]["evidence_result"] == "degraded"
+ assert contract["verifier"]["terminal"] is False
+ assert contract["verifier"]["missing_fields"] == [
+ "source_resolution_receipt"
+ ]
+ assert contract["closure_state"] == "partial_degraded_safe_next_action"
+
+
+def test_backup_restore_work_item_clusters_recurrence_but_receipts_stay_unique() -> None:
+ shared = {
+ "project_id": "awoooi",
+ "source_refs": {
+ "fingerprints": [
+ "ai_automation_alert_card:backup_restore_escrow_signal:"
+ "backup_restore_escrow_triage"
+ ]
+ },
+ "source_row_exists": True,
+ "contract_origin": "synthesized_at_read",
+ }
+ first = build_backup_restore_signal_automation_contract(
+ **shared,
+ receipt_id="message-1",
+ run_id="run-1",
+ )
+ second = build_backup_restore_signal_automation_contract(
+ **shared,
+ receipt_id="message-2",
+ run_id="run-2",
+ )
+
+ assert first["work_item"]["work_item_id"] == second["work_item"]["work_item_id"]
+ assert first["candidate"]["candidate_id"] != second["candidate"]["candidate_id"]
+ assert first["receipt"]["receipt_id"] != second["receipt"]["receipt_id"]
+ assert first["verifier"]["verifier_id"] != second["verifier"]["verifier_id"]
+
+
+def test_backup_dispatch_requires_accepted_triggered_backupcheck_receipt() -> None:
+ receipt = {
+ "schema_version": "telegram_agent99_dispatch_receipt_projection_v1",
+ "status": "accepted_inbox_triggered",
+ "transport": "relay",
+ "alert_id": "awoooi-backup-alert",
+ "kind": "backup_health",
+ "suggested_mode": "BackupCheck",
+ "accepted": True,
+ "inbox_triggered": True,
+ "receipt_persisted": True,
+ "runtime_closure_verified": False,
+ "run_id": "agent99-run-123",
+ "trace_id": "00-12345678901234567890123456789012-1234567890123456-01",
+ "work_item_id": "agent99-dispatch:awoooi:INC-BACKUP:backupcheck",
+ }
+ contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="source-message",
+ run_id="source-run",
+ source_row_exists=True,
+ contract_origin="persisted_at_send",
+ contract_persisted_at_send=True,
+ agent99_dispatch_receipt=receipt,
+ )
+
+ dispatch = contract["candidate"]["agent99_dispatch"]
+ assert dispatch["status"] == "accepted_inbox_triggered"
+ assert dispatch["dispatched"] is True
+ assert dispatch["receipt_persisted"] is True
+ assert dispatch["identity_complete"] is True
+ assert dispatch["agent99_run_id"] == "agent99-run-123"
+ provenance = contract["contract_provenance"]
+ assert provenance["source_contract_present_at_send"] is True
+ assert provenance["recorded_policy_present"] is True
+ assert provenance["recorded_policy_source_hash"].startswith("sha256:")
+ assert provenance["recorded_immutable_declared_present"] is True
+ binding = contract["candidate"]["identity_binding"]
+ assert binding["status"] == "bound_dispatch_verifier_pending"
+ assert binding["complete"] is True
+ assert binding["brr_work_item_id"] == contract["work_item"]["work_item_id"]
+ assert binding["brr_verifier_id"] == contract["verifier"]["verifier_id"]
+ assert binding["agent99_run_id"] == "agent99-run-123"
+ assert binding["agent99_work_item_id"].startswith("agent99-dispatch:")
+ assert contract["candidate"]["execution_state"] == "dispatched"
+ assert contract["candidate"]["status"] == "dispatched_verifier_pending"
+ assert contract["verifier"]["status"] == "dispatched_verifier_pending"
+ assert contract["verifier"]["terminal"] is False
+ assert contract["verifier"]["agent99_outcome"]["status"] == (
+ "dispatch_accepted_outcome_pending"
+ )
+ assert contract["verifier"]["agent99_outcome"]["required_source_schema"] == (
+ "agent99_outcome_contract_v1"
+ )
+
+ wrong_route = dict(receipt, suggested_mode="ProviderFreshness")
+ blocked = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="source-message",
+ run_id="source-run",
+ source_row_exists=True,
+ contract_origin="persisted_at_send",
+ contract_persisted_at_send=True,
+ agent99_dispatch_receipt=wrong_route,
+ )
+ assert blocked["candidate"]["execution_state"] == "not_started"
+ assert blocked["candidate"]["agent99_dispatch"]["status"] == (
+ "dispatch_route_mismatch_fail_closed"
+ )
+ assert blocked["verifier"]["terminal"] is False
+
+ not_durable = dict(receipt, receipt_persisted=False)
+ blocked = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="source-message",
+ run_id="source-run",
+ source_row_exists=True,
+ contract_origin="persisted_at_send",
+ contract_persisted_at_send=True,
+ agent99_dispatch_receipt=not_durable,
+ )
+ assert blocked["candidate"]["execution_state"] == "not_started"
+ assert blocked["candidate"]["agent99_dispatch"]["status"] == (
+ "dispatch_receipt_not_persisted_fail_closed"
+ )
+
+ legacy = dict(
+ receipt,
+ schema_version="telegram_agent99_dispatch_receipt_v1",
+ receipt_persisted=False,
+ )
+ legacy_contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="legacy-source-message",
+ run_id="legacy-source-run",
+ source_row_exists=True,
+ contract_origin="synthesized_at_read",
+ agent99_dispatch_receipt=legacy,
+ )
+ assert legacy_contract["candidate"]["execution_state"] == "not_started"
+ assert legacy_contract["candidate"]["agent99_dispatch"]["status"] == (
+ "legacy_dispatch_receipt_unbound"
+ )
+ assert legacy_contract["candidate"]["identity_binding"]["complete"] is False
+ assert legacy_contract["verifier"]["terminal"] is False
+
+ failed_verifier_receipt = dict(
+ receipt,
+ post_verifier_passed=False,
+ current_run_state="failed",
+ current_verifier={
+ "schema_version": "agent99_independent_verifier_receipt_v1",
+ "status": "failed",
+ "run_id": receipt["run_id"],
+ "trace_id": receipt["trace_id"],
+ "work_item_id": receipt["work_item_id"],
+ "verifier_passed": False,
+ "source_event_resolved": False,
+ },
+ )
+ failed_contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="failed-source-message",
+ run_id="failed-source-run",
+ source_row_exists=True,
+ contract_origin="persisted_at_send",
+ contract_persisted_at_send=True,
+ agent99_dispatch_receipt=failed_verifier_receipt,
+ )
+ assert failed_contract["candidate"]["status"] == (
+ "verifier_failed_retry_required"
+ )
+ assert failed_contract["verifier"]["status"] == "failed_terminal"
+ assert failed_contract["verifier"]["result"] == "failed"
+ assert failed_contract["verifier"]["terminal"] is True
+ assert failed_contract["closure_state"] == (
+ "partial_degraded_safe_next_action"
+ )
+
+ false_green_receipt = dict(
+ receipt,
+ post_verifier_passed=True,
+ runtime_closure_verified=True,
+ closure_complete=True,
+ current_run_state="completed",
+ current_verifier={
+ "schema_version": "agent99_independent_verifier_receipt_v1",
+ "status": "success",
+ "run_id": receipt["run_id"],
+ "trace_id": receipt["trace_id"],
+ "work_item_id": receipt["work_item_id"],
+ "outcome_state": "resolved",
+ "transport_ok": True,
+ "verifier_passed": True,
+ "source_event_resolved": True,
+ "verified_at": "2026-07-11T20:20:00+08:00",
+ "evidence_refs": {},
+ },
+ current_learning_writeback={
+ "schema_version": "agent99_learning_writeback_receipt_v1",
+ "status": "success",
+ "run_id": receipt["run_id"],
+ "trace_id": receipt["trace_id"],
+ "work_item_id": receipt["work_item_id"],
+ "receipt_refs": {
+ "incident_closure_receipt_id": "inc-close-1",
+ "telegram_lifecycle_receipt_id": "tg-life-1",
+ "km_writeback_ack_id": "km-ack-1",
+ "playbook_trust_writeback_ack_id": "pb-ack-1",
+ "dr_scorecard_writeback_ack_id": "dr-ack-1",
+ },
+ },
+ )
+ false_green_contract = build_backup_restore_signal_automation_contract(
+ project_id="awoooi",
+ receipt_id="false-green-source-message",
+ run_id="false-green-source-run",
+ source_row_exists=True,
+ contract_origin="persisted_at_send",
+ contract_persisted_at_send=True,
+ agent99_dispatch_receipt=false_green_receipt,
+ )
+ assert false_green_contract["verifier"]["terminal"] is False
+ assert false_green_contract["candidate"]["agent99_dispatch"][
+ "runtime_closure_verified"
+ ] is False
+ assert set(false_green_contract["verifier"][
+ "missing_backup_evidence_refs"
+ ]) == {
+ "agent99_outcome_receipt_id",
+ "post_verifier_evidence_ref",
+ "source_event_evidence_ref",
+ "backup_status_evidence_ref",
+ "freshness_evidence_ref",
+ "offsite_verify_evidence_ref",
+ "escrow_evidence_ref",
+ "restore_drill_evidence_ref",
+ "source_resolution_receipt_ref",
+ }
+
+
+def test_backup_evidence_negative_phrases_override_good_tokens() -> None:
+ observations = extract_backup_restore_evidence(
+ "Top evidence\n"
+ "├ backup_status: 1 but not ok\n"
+ "└ restore_drill: 1 but not verified\n"
+ "建議下一步"
+ )
+ by_field = {item["field"]: item for item in observations}
+
+ assert by_field["backup_status"]["status"] == "failed"
+ assert by_field["backup_status"]["safe_value"] == "explicit_negative_result"
+ assert by_field["restore_drill"]["status"] == "failed"
+ assert by_field["restore_drill"]["safe_value"] == (
+ "explicit_negative_result"
+ )
diff --git a/apps/api/tests/test_controlled_alert_target_router.py b/apps/api/tests/test_controlled_alert_target_router.py
new file mode 100644
index 000000000..9ba973f3e
--- /dev/null
+++ b/apps/api/tests/test_controlled_alert_target_router.py
@@ -0,0 +1,242 @@
+from __future__ import annotations
+
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from src.api.v1 import webhooks
+from src.services.awooop_ansible_audit_service import (
+ _catalog_hints,
+ build_ansible_decision_audit_payload,
+ build_ansible_truth,
+)
+from src.services.controlled_alert_target_router import (
+ build_controlled_recovery_handoff,
+ build_controlled_recovery_promotion_contract,
+ resolve_controlled_alert_target,
+)
+
+
+def _cold_start_incident() -> dict:
+ return {
+ "incident_id": "INC-20260711-11C751",
+ "project_id": "awoooi",
+ "alertname": "ColdStartGateBlocked",
+ "alert_category": "general",
+ "affected_services": ["cold-start-gate"],
+ "signals": [
+ {
+ "alert_name": "ColdStartGateBlocked",
+ "labels": {
+ "component": "cold-start-gate",
+ "namespace": "default",
+ "host": "110",
+ },
+ "annotations": {
+ "summary": (
+ "full-stack cold-start recovery blocked; backup and "
+ "Docker evidence must be checked"
+ )
+ },
+ }
+ ],
+ }
+
+
+def test_normalizer_separates_source_namespace_from_execution_domain() -> None:
+ route = resolve_controlled_alert_target(
+ alertname="ColdStartGateBlocked",
+ target_resource="cold-start-gate",
+ namespace="default",
+ )
+
+ assert route is not None
+ assert route["source_namespace"] == "default"
+ assert route["normalized_execution_namespace"] == "host_recovery"
+ assert route["kubernetes_namespace_applicable"] is False
+ assert route["executor"] == "Agent99"
+ assert route["runtime_execution_authorized"] is False
+
+ assert resolve_controlled_alert_target(
+ alertname="SentryRpsZero",
+ target_resource="sentry",
+ namespace="awoooi-prod",
+ ) is None
+
+
+def test_cold_start_contract_requires_same_run_receipts_before_closure() -> None:
+ contract = build_controlled_recovery_promotion_contract(
+ alertname="ColdStartGateBlocked",
+ target_resource="cold-start-gate",
+ namespace="default",
+ incident_id="INC-20260711-11C751",
+ )
+
+ assert contract is not None
+ fields = {row["field"]: row for row in contract["fields"]}
+ assert fields["target_selector"]["status"] == "ready"
+ assert fields["controlled_apply_route"]["status"] == "ready"
+ for required in (
+ "source_sensor_receipt",
+ "check_mode_receipt",
+ "dispatch_receipt",
+ "post_apply_verifier",
+ "incident_closure_receipt",
+ "telegram_receipt",
+ "km_writeback",
+ "playbook_trust_writeback",
+ ):
+ assert fields[required]["status"] == "pending"
+ assert contract["runtime_execution_authorized"] is False
+ assert contract["owner_review_required"] is False
+ assert contract["needs_human"] is False
+ assert contract["completion_status"] == "partial"
+
+
+def test_cold_start_is_excluded_from_generic_ansible_keyword_catalog() -> None:
+ hints = _catalog_hints(_cold_start_incident(), None)
+
+ assert hints["match_mode"] == "domain_controlled_executor_route_v1"
+ assert hints["decision_effect"] == "delegate_to_domain_executor"
+ assert hints["candidates"] == []
+ assert hints["controlled_executor"]["executor"] == "Agent99"
+ assert "ansible:110-devops" in hints["unmatched_catalog_ids"]
+ assert "ansible:188-momo-backup-user" in hints["unmatched_catalog_ids"]
+
+ truth = build_ansible_truth([], incident=_cold_start_incident(), drift=None)
+ assert truth["not_used_reason"] == "non_kubernetes_control_plane_route"
+ assert truth["candidate_catalog"]["controlled_executor"]["route_id"] == (
+ "agent99_recover_after_owner_review"
+ )
+
+
+def test_cold_start_does_not_emit_ansible_candidate_audit() -> None:
+ incident = SimpleNamespace(
+ incident_id="INC-20260711-11C751",
+ project_id="awoooi",
+ alertname="ColdStartGateBlocked",
+ alert_category="general",
+ notification_type="TYPE-3",
+ severity=SimpleNamespace(value="P3"),
+ affected_services=["cold-start-gate"],
+ signals=[
+ SimpleNamespace(
+ alert_name="ColdStartGateBlocked",
+ labels={"component": "cold-start-gate", "namespace": "default"},
+ annotations={"summary": "full-stack cold-start blocked"},
+ )
+ ],
+ )
+
+ payload = build_ansible_decision_audit_payload(
+ incident=incident,
+ proposal_data={"source": "test", "risk_level": "low"},
+ decision_path="alert_webhook_controlled_router",
+ not_used_reason="test",
+ )
+
+ assert payload is None
+
+
+def test_handoff_stays_partial_until_dispatch_verifier_and_learning() -> None:
+ handoff = build_controlled_recovery_handoff(
+ alertname="custom",
+ target_resource="cold-start-gate",
+ namespace="default",
+ incident_id="INC-20260711-11C751",
+ )
+
+ assert handoff is not None
+ assert handoff["single_writer_executor"] == "Agent99"
+ assert handoff["queued"] is False
+ assert handoff["side_effect_performed"] is False
+ assert handoff["runtime_execution_authorized"] is False
+ assert handoff["owner_review_required"] is False
+ assert handoff["needs_human"] is False
+ assert handoff["active_blockers"] == [
+ "agent99_dispatch_receipt_readback_pending",
+ "post_apply_verifier_missing",
+ "km_playbook_writeback_missing",
+ ]
+
+
+@pytest.mark.asyncio
+async def test_webhook_router_never_queues_generic_ansible_for_cold_start(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ queue = AsyncMock(side_effect=AssertionError("generic Ansible must not be queued"))
+ append = AsyncMock()
+ incident_lookup = AsyncMock(side_effect=AssertionError("incident lookup not needed"))
+ bridge = AsyncMock(
+ return_value={
+ "status": "dispatched",
+ "dispatchPerformed": True,
+ "identity": {
+ "run_id": "7cf8fdf7-0966-5ac6-95b2-09e32808b248",
+ "trace_id": "00-11111111111111111111111111111111-2222222222222222-01",
+ "work_item_id": "agent99-dispatch:awoooi:INC-20260711-11C751:recovery",
+ "idempotency_key": "agent99-controlled:stable-key",
+ },
+ "dispatchReceipt": {
+ "status": "accepted_inbox_triggered",
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "correlatedReceipt": {
+ "status": "dispatch_accepted_verifier_pending",
+ "receipt_persisted": True,
+ "runtime_execution_authorized": False,
+ "runtime_closure_verified": False,
+ "verifier": {"status": "pending"},
+ "learning_writeback": {"status": "pending_verifier"},
+ },
+ }
+ )
+
+ monkeypatch.setattr(
+ "src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
+ queue,
+ )
+ monkeypatch.setattr(
+ "src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
+ lambda: SimpleNamespace(append=append),
+ )
+ monkeypatch.setattr(
+ webhooks,
+ "get_incident_service",
+ lambda: SimpleNamespace(get_from_working_memory=incident_lookup),
+ )
+ monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge)
+
+ handoff = await webhooks._try_auto_repair_background(
+ incident_id="INC-20260711-11C751",
+ approval_id="00000000-0000-0000-0000-000000000751",
+ alert_type="custom",
+ target_resource="cold-start-gate",
+ namespace="default",
+ risk_level="low",
+ source_alert_id="alert-11c751",
+ source_alertname="ColdStartGateBlocked",
+ source_fingerprint="source-fingerprint",
+ )
+
+ queue.assert_not_awaited()
+ incident_lookup.assert_not_awaited()
+ bridge.assert_awaited_once()
+ assert handoff["single_writer_executor"] == "Agent99"
+ assert handoff["execution_priority"] == 30
+ assert handoff["queued"] is True
+ assert handoff["runtime_execution_authorized"] is False
+ assert handoff["runtime_closure_verified"] is False
+ assert handoff["automation_run_id"] == (
+ "7cf8fdf7-0966-5ac6-95b2-09e32808b248"
+ )
+ append.assert_awaited_once()
+ receipt = append.await_args.kwargs
+ assert receipt["action_detail"] == "controlled_check_mode_queued"
+ assert receipt["success"] is True
+ assert receipt["context"]["safe_next_action"] == (
+ "run_independent_cold_start_verifier_then_km_playbook_writeback"
+ )
+ assert receipt["context"]["runtime_closure_verified"] is False
diff --git a/apps/api/tests/test_cs1_auto_execute.py b/apps/api/tests/test_cs1_auto_execute.py
index f74a4bb62..f510c8ff8 100644
--- a/apps/api/tests/test_cs1_auto_execute.py
+++ b/apps/api/tests/test_cs1_auto_execute.py
@@ -2,6 +2,7 @@
from __future__ import annotations
+import hashlib
import inspect
from types import SimpleNamespace
from unittest.mock import AsyncMock
@@ -116,26 +117,31 @@ async def test_webhook_router_writes_queue_receipt_without_side_effect(
get_from_working_memory=AsyncMock(return_value=incident)
)
append = AsyncMock()
-
- queue_kwargs: dict = {}
-
- async def queue(**kwargs):
- queue_kwargs.update(kwargs)
- return {
- "schema_version": "ai_decision_controlled_executor_handoff_v1",
- "status": "controlled_check_mode_queued",
- "automation_run_id": "00000000-0000-0000-0000-000000000101",
- "queued": True,
- "side_effect_performed": False,
- "single_writer_executor": "awoooi-ansible-executor-broker",
- "active_blockers": [],
+ queue = AsyncMock()
+ bridge = AsyncMock(
+ return_value={
+ "status": "deduplicated",
+ "dispatchPerformed": False,
+ "identity": {
+ "run_id": "00000000-0000-0000-0000-000000000101",
+ "trace_id": "00000000-0000-0000-0000-000000000102",
+ "work_item_id": "agent99-dispatch:awoooi:INC-QUEUE:backup",
+ "idempotency_key": "agent99:backup:INC-QUEUE",
+ },
+ "dispatchReceipt": {
+ "accepted": True,
+ "inbox_triggered": True,
+ },
+ "correlatedReceipt": {"receipt_persisted": True},
}
+ )
monkeypatch.setattr(webhooks, "get_incident_service", lambda: incident_service)
monkeypatch.setattr(
"src.jobs.awooop_ansible_candidate_backfill_job.enqueue_ai_decision_ansible_candidate",
queue,
)
+ monkeypatch.setattr(webhooks, "bridge_alertmanager_to_agent99", bridge)
monkeypatch.setattr(
"src.repositories.alert_operation_log_repository.get_alert_operation_log_repository",
lambda: SimpleNamespace(append=append),
@@ -153,7 +159,12 @@ async def test_webhook_router_writes_queue_receipt_without_side_effect(
assert handoff["queued"] is True
assert handoff["side_effect_performed"] is False
assert handoff["execution_priority"] == 30
- assert queue_kwargs["proposal_data"]["execution_priority"] == 30
+ queue.assert_not_awaited()
+ bridge.assert_awaited_once()
+ expected_fingerprint = hashlib.sha256(
+ b"awoooi-prod:momo-postgres:MomoPostgresBackupFailed:momo-postgres"
+ ).hexdigest()[:32]
+ assert bridge.await_args.kwargs["fingerprint"] == expected_fingerprint
append.assert_awaited_once()
assert append.await_args.kwargs["context"]["side_effect_performed"] is False
assert append.await_args.kwargs["context"]["execution_priority"] == 30
diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py
index f53ff7af2..0f49ae496 100644
--- a/apps/api/tests/test_executor_network_policy_boundary.py
+++ b/apps/api/tests/test_executor_network_policy_boundary.py
@@ -118,5 +118,7 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
assert f"ssh-keyscan -T 5 {expected_hosts}" in workflow
assert "EXPECTED_HOSTS=5" in workflow
assert workflow.count(expected_hosts) >= 4
- assert "len(reachable) == len(sys.argv) - 1" in workflow
+ assert "except ConnectionRefusedError:" in workflow
+ assert "broker_ssh_refused_but_egress_permitted=" in workflow
+ assert "connected and len(permitted) == len(sys.argv) - 1" in workflow
assert "ssh-mcp-key known_hosts 更新失敗，停止部署" in workflow
diff --git a/apps/api/tests/test_incident_signal_source_compatibility.py b/apps/api/tests/test_incident_signal_source_compatibility.py
new file mode 100644
index 000000000..a924eaa3c
--- /dev/null
+++ b/apps/api/tests/test_incident_signal_source_compatibility.py
@@ -0,0 +1,273 @@
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+from pydantic import ValidationError
+
+from src.api.v1 import auto_repair as auto_repair_api
+from src.api.v1 import incidents as incidents_api
+from src.api.v1 import webhooks as webhooks_api
+from src.models.incident import IncidentStatus, Severity
+from src.repositories import incident_repository as repository_module
+from src.repositories.incident_repository import IncidentDBRepository
+from src.services.incident_engine import IncidentEngineAdapter
+from src.services.incident_service import IncidentService
+
+SENSOR_SOURCES = ("journal", "node-exporter", "sensor-probe", "sensor-agent")
+D037_INCIDENT_ID = "INC-20260711-D037E5"
+
+
+def _record(
+ incident_id: str,
+ *,
+ status: IncidentStatus,
+ source: str | None,
+ malformed: bool = False,
+ auto_repair_count: int = 0,
+) -> SimpleNamespace:
+ timestamp = datetime(2026, 7, 11, 12, 0, tzinfo=UTC)
+ signal = {
+ "alert_name": f"Signal-{incident_id}",
+ "annotations": {},
+ "labels": {},
+ }
+ if not malformed:
+ signal.update({
+ "signal_id": incident_id[-8:],
+ "severity": Severity.P2.value,
+ "source": source,
+ "fired_at": timestamp.isoformat(),
+ })
+ frequency_snapshot = None
+ if auto_repair_count:
+ frequency_snapshot = {
+ "anomaly_key": f"anomaly-{incident_id}",
+ "auto_repair_count": auto_repair_count,
+ "last_repair_action": "ansible repair host service",
+ "last_repair_success": True,
+ }
+ return SimpleNamespace(
+ incident_id=incident_id,
+ status=status,
+ severity=Severity.P2,
+ signals=[signal],
+ affected_services=["awoooi-api"],
+ proposal_ids=[],
+ frequency_snapshot=frequency_snapshot,
+ created_at=timestamp,
+ updated_at=timestamp,
+ resolved_at=(timestamp if status is IncidentStatus.RESOLVED else None),
+ closed_at=None,
+ )
+
+
+class _RowsResult:
+ def __init__(self, records: list[SimpleNamespace]) -> None:
+ self._records = records
+
+ def scalars(self) -> _RowsResult:
+ return self
+
+ def all(self) -> list[SimpleNamespace]:
+ return self._records
+
+
+class _Db:
+ def __init__(self, records: list[SimpleNamespace]) -> None:
+ self.records = records
+ self.statements: list[object] = []
+
+ async def execute(self, statement: object) -> _RowsResult:
+ self.statements.append(statement)
+ return _RowsResult(self.records)
+
+
+def _install_fake_db(
+ monkeypatch: pytest.MonkeyPatch,
+ records: list[SimpleNamespace],
+) -> tuple[_Db, list[str | None]]:
+ db = _Db(records)
+ project_ids: list[str | None] = []
+
+ @asynccontextmanager
+ async def fake_get_db_context(project_id: str | None = None):
+ project_ids.append(project_id)
+ yield db
+
+ monkeypatch.setattr(repository_module, "get_db_context", fake_get_db_context)
+ return db, project_ids
+
+
+@pytest.mark.asyncio
+async def test_active_readback_accepts_sensor_sources_quarantines_malformed_and_excludes_d037(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ records = [
+ _record(
+ f"INC-20260711-SENSOR-{index}",
+ status=IncidentStatus.INVESTIGATING,
+ source=source,
+ )
+ for index, source in enumerate(SENSOR_SOURCES, start=1)
+ ]
+ records.extend([
+ _record(
+ D037_INCIDENT_ID,
+ status=IncidentStatus.RESOLVED,
+ source="alertmanager",
+ ),
+ _record(
+ "INC-20260711-3A4165",
+ status=IncidentStatus.INVESTIGATING,
+ source=None,
+ malformed=True,
+ ),
+ ])
+ db, project_ids = _install_fake_db(monkeypatch, records)
+
+ readback = await IncidentDBRepository().get_active_readback(
+ project_id="awoooi"
+ )
+
+ assert [incident.signals[0].source for incident in readback.incidents] == list(
+ SENSOR_SOURCES
+ )
+ assert D037_INCIDENT_ID not in {
+ incident.incident_id for incident in readback.incidents
+ }
+ assert [item.model_dump() for item in readback.degraded_records] == [{
+ "incident_id": "INC-20260711-3A4165",
+ "reason_code": "incident_signal_schema_invalid",
+ "quarantined_from_response": True,
+ }]
+ assert project_ids == ["awoooi"]
+ assert IncidentStatus.RESOLVED not in db.statements[0].compile().params["status_1"]
+
+
+@pytest.mark.asyncio
+async def test_unknown_source_is_fail_visible_without_blocking_valid_active_rows(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ records = [
+ _record(
+ "INC-20260711-UNKNOWN",
+ status=IncidentStatus.INVESTIGATING,
+ source="unregistered-sensor",
+ )
+ ]
+ _install_fake_db(monkeypatch, records)
+ service = IncidentService()
+ redis_read = AsyncMock()
+ monkeypatch.setattr(service, "get_from_working_memory", redis_read)
+
+ readback = await service.get_active_incidents_readback(project_id="awoooi")
+
+ assert readback.incidents == []
+ assert readback.degraded is True
+ assert readback.degraded_records[0].reason_code == (
+ "incident_signal_source_unsupported"
+ )
+ redis_read.assert_not_awaited()
+
+ class _DecisionManager:
+ async def _find_existing_tokens_for_incidents(self, _incident_ids):
+ return {}
+
+ monkeypatch.setattr(incidents_api, "get_incident_service", lambda: service)
+ monkeypatch.setattr(
+ incidents_api,
+ "get_decision_manager",
+ lambda: _DecisionManager(),
+ )
+ monkeypatch.setattr(auto_repair_api, "get_incident_service", lambda: service)
+
+ active = await incidents_api.list_incidents(
+ project_id="awoooi",
+ generate_missing_decisions=False,
+ )
+ history = await auto_repair_api.get_repair_history(
+ project_id="awoooi",
+ limit=20,
+ )
+
+ for response in (active, history):
+ assert response.degraded is True
+ assert response.degraded_count == 1
+ assert response.degraded_records[0].incident_id == (
+ "INC-20260711-UNKNOWN"
+ )
+ assert response.redis_fallback_used is False
+
+
+@pytest.mark.asyncio
+async def test_auto_repair_history_remains_usable_with_journal_and_malformed_rows(
+ monkeypatch: pytest.MonkeyPatch,
+) -> None:
+ records = [
+ _record(
+ "INC-20260711-JOURNAL-HISTORY",
+ status=IncidentStatus.INVESTIGATING,
+ source="journal",
+ auto_repair_count=1,
+ ),
+ _record(
+ D037_INCIDENT_ID,
+ status=IncidentStatus.RESOLVED,
+ source="alertmanager",
+ auto_repair_count=1,
+ ),
+ _record(
+ "INC-20260711-3A4165",
+ status=IncidentStatus.INVESTIGATING,
+ source=None,
+ malformed=True,
+ ),
+ ]
+ _install_fake_db(monkeypatch, records)
+ monkeypatch.setattr(
+ auto_repair_api,
+ "get_incident_service",
+ IncidentService,
+ )
+
+ history = await auto_repair_api.get_repair_history(
+ project_id="awoooi",
+ limit=20,
+ )
+
+ assert history.count == 1
+ assert history.items[0].incident_id == "INC-20260711-JOURNAL-HISTORY"
+ assert D037_INCIDENT_ID not in {item.incident_id for item in history.items}
+ assert history.degraded is True
+ assert history.degraded_records[0].incident_id == "INC-20260711-3A4165"
+ assert history.degraded_records[0].reason_code == (
+ "incident_signal_schema_invalid"
+ )
+
+
+def test_signal_ingress_rejects_unregistered_source_before_stream_write() -> None:
+ with pytest.raises(ValidationError):
+ webhooks_api.SignalPayload(
+ source="unregistered-sensor",
+ alert_name="UnknownProducer",
+ severity="warning",
+ namespace="infra",
+ target="host",
+ )
+
+
+@pytest.mark.asyncio
+async def test_incident_engine_rejects_unknown_source_before_durable_processing() -> None:
+ brain_engine = SimpleNamespace(process_signal=AsyncMock(return_value=None))
+ adapter = IncidentEngineAdapter(brain_engine)
+
+ assert await adapter.process_signal({"source": "journal"}) is None
+ brain_engine.process_signal.assert_awaited_once_with({"source": "journal"})
+
+ with pytest.raises(ValueError, match="unsupported_signal_source"):
+ await adapter.process_signal({"source": "unregistered-sensor"})
+ assert brain_engine.process_signal.await_count == 1
diff --git a/apps/api/tests/test_runtime_bootstrap_guards.py b/apps/api/tests/test_runtime_bootstrap_guards.py
index 2e5d7ab7f..16009b268 100644
--- a/apps/api/tests/test_runtime_bootstrap_guards.py
+++ b/apps/api/tests/test_runtime_bootstrap_guards.py
@@ -365,3 +365,49 @@ def test_init_db_backfills_all_optional_knowledge_read_columns() -> None:
assert "ADD COLUMN IF NOT EXISTS path_type" in source
assert "ADD COLUMN IF NOT EXISTS symptoms_hash" in source
assert "CREATE INDEX IF NOT EXISTS ix_knowledge_symptoms_hash" in source
+
+
+def test_init_db_bootstraps_agent99_retry_schema_before_serving() -> None:
+ import inspect
+
+ from src.db import base as db_base
+
+ source = inspect.getsource(db_base._run_init_db_ddl)
+ column_ddl = (
+ "ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMP NULL"
+ )
+ index_ddl = "CREATE INDEX IF NOT EXISTS idx_run_state_retry_due"
+
+ assert "ALTER TABLE awooop_run_state" in source
+ assert column_ddl in source
+ assert index_ddl in source
+ assert "WHERE state = 'pending' AND next_attempt_at IS NOT NULL" in source
+ assert source.index(column_ddl) < source.index("ALTER TABLE approval_records")
+
+
+def test_agent99_retry_migration_matches_startup_bootstrap() -> None:
+ migrations = Path(__file__).resolve().parents[1] / "migrations"
+ up = (
+ migrations / "agent99_dispatch_next_attempt_at_2026-07-11.sql"
+ ).read_text(encoding="utf-8")
+ down = (
+ migrations / "agent99_dispatch_next_attempt_at_2026-07-11_down.sql"
+ ).read_text(encoding="utf-8")
+
+ assert "ADD COLUMN IF NOT EXISTS next_attempt_at TIMESTAMP NULL" in up
+ assert "CREATE INDEX IF NOT EXISTS idx_run_state_retry_due" in up
+ assert "WHERE state = 'pending' AND next_attempt_at IS NOT NULL" in up
+ assert down.index("DROP INDEX IF EXISTS idx_run_state_retry_due") < (
+ down.index("DROP COLUMN IF EXISTS next_attempt_at")
+ )
+
+
+def test_api_runs_retry_schema_bootstrap_before_agent99_reconciler() -> None:
+ import inspect
+
+ from src import main as api_main
+
+ source = inspect.getsource(api_main.lifespan)
+ assert source.index("await init_db()") < source.index(
+ "run_agent99_controlled_dispatch_reconciler_loop()"
+ )
diff --git a/apps/api/tests/test_telegram_gateway_error_sanitizer.py b/apps/api/tests/test_telegram_gateway_error_sanitizer.py
index df84691b5..b3c22ae76 100644
--- a/apps/api/tests/test_telegram_gateway_error_sanitizer.py
+++ b/apps/api/tests/test_telegram_gateway_error_sanitizer.py
@@ -4,6 +4,7 @@ from src.services.telegram_gateway import (
_agent99_actionable_outbound_metadata,
_agent99_alert_from_actionable_outbound,
_agent99_alert_from_ai_automation_card,
+ _alert_notification_source_envelope_extra,
_merge_outbound_source_envelope_extra,
_outbound_source_envelope,
_sanitize_telegram_error,
@@ -187,6 +188,76 @@ full_log=/var/ossec/logs/alerts/alerts.json Authorization: Bearer abcdefghijklmn
assert "abcdefghijkl" not in str(envelope)
+def test_backup_restore_card_persists_fail_closed_readback_contract() -> None:
+ raw_alert = (
+ 'alertname="BackupAggregateRunFailed" host="110" '
+ 'backup_status=0 escrow_missing=5 restic failed '
+ 'Authorization: Bearer abcdefghijklmnopqrstuvwxyz'
+ )
+ card = format_aiops_signal_alert_card(raw_alert)
+ envelope = _outbound_source_envelope(
+ "sendMessage",
+ {"chat_id": "-100123", "text": card, "parse_mode": "HTML"},
+ )
+
+ metadata = envelope["ai_automation_alert_card"]
+ contract = metadata["backup_restore_automation"]
+ assert metadata["event_type"] == "backup_restore_escrow_signal"
+ assert metadata["lane"] == "backup_restore_escrow_triage"
+ assert metadata["controlled_playbook_queue"] is False
+ assert metadata["runtime_write_gate_count"] == 0
+ assert metadata["runtime_write_gate_state"] == "evidence_only"
+ assert metadata["no_false_green"] is True
+ assert contract["work_item"]["owner"] == "backup_dr"
+ assert contract["candidate"]["mode"] == "BackupCheck"
+ assert contract["candidate"]["execution_state"] == "not_started"
+ assert metadata["declared_runtime_write_gate_count"] == 0
+ assert metadata["declared_runtime_write_gate_state"] == "closed"
+ assert metadata["declared_controlled_playbook_queue"] is True
+ assert contract["contract_origin"] == "generated_for_send"
+ assert contract["contract_persisted_at_send"] is False
+ assert contract["contract_provenance"][
+ "source_contract_present_at_send"
+ ] is False
+ assert contract["contract_provenance"]["recorded_policy_present"] is False
+ assert contract["contract_provenance"][
+ "recorded_immutable_declared_present"
+ ] is False
+ assert contract["policy"]["policy_version"] == (
+ "backup_restore_readback_policy_v2"
+ )
+ assert contract["receipt"]["status"] == (
+ "pending_source_message_persistence"
+ )
+ assert contract["receipt"]["is_agent99_execution_receipt"] is False
+ assert contract["verifier"]["status"] == (
+ "source_message_pending_verifier_not_started"
+ )
+ assert contract["verifier"]["agent99_outcome"]["status"] == (
+ "outcome_source_unavailable"
+ )
+ assert contract["runtime_write_allowed"] is False
+ assert contract["production_write_performed"] is False
+ assert "runtime_write_gate=0_until_verifier" in metadata["gates"]
+ assert "abcdefghijklmnopqrstuvwxyz" not in str(envelope)
+
+ notification_receipt = _alert_notification_source_envelope_extra(
+ safe_text=card,
+ effective_parse_mode="HTML",
+ reply_markup_present=False,
+ )["alert_notification"]
+ assert notification_receipt["controlled_apply_allowed"] is False
+ assert notification_receipt["manual_default_route_allowed"] is False
+
+ agent99_payload = _agent99_alert_from_ai_automation_card(card, source="unit-test")
+ assert agent99_payload is not None
+ assert agent99_payload["kind"] == "backup_health"
+ assert agent99_payload["suggestedMode"] == "BackupCheck"
+ assert agent99_payload["controlledApply"] is False
+ assert "BackupCheck readback candidate only" in agent99_payload["instruction"]
+ assert agent99_payload["awoooi"]["backupRestoreAutomation"] == contract
+
+
def test_ai_alert_card_builds_agent99_provider_freshness_signal() -> None:
raw_alert = """
SourceProviderIngestionStale provider_freshness_signal freshness window timeout
diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py
index e4963af8c..6b6a6fc4c 100644
--- a/apps/api/tests/test_telegram_message_templates.py
+++ b/apps/api/tests/test_telegram_message_templates.py
@@ -271,7 +271,12 @@ def test_aiops_signal_formatter_covers_non_host_alert_lanes(
assert event_type in result
assert lane in result
assert "controlled_playbook_queue" in result
- assert "runtime_write_gate=controlled" in result
+ if event_type == "backup_restore_escrow_signal":
+ assert "controlled_playbook_queue=readback_only" in result
+ assert "runtime_write_gate=0_until_verifier" in result
+ assert "runtime_write_gate=controlled" not in result
+ else:
+ assert "runtime_write_gate=controlled" in result
assert "Top evidence" in result
assert "禁止事項" in result