From 08f3a70ce61676dbf3e13ce8f1c19dc24bfe7a14 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 16 Jul 2026 21:21:43 +0800 Subject: [PATCH] fix(sre): close bounded runtime contract gaps --- .gitea/workflows/cd.yaml | 13 +- agent99-bootstrap.ps1 | 1 + agent99-contract-check.ps1 | 3 + agent99-control-plane.ps1 | 77 ++- agent99-db-bounded-executor.ps1 | 364 +++++++++++ agent99-deploy.ps1 | 1 + apps/api/src/api/v1/agents.py | 6 +- apps/api/src/models/agent99_completion.py | 35 ++ .../services/agent99_telegram_lifecycle.py | 136 ++++ .../services/awooop_ansible_audit_service.py | 2 +- .../awooop_ansible_check_mode_service.py | 22 +- .../services/awooop_ansible_post_verifier.py | 20 +- apps/api/src/services/db_bounded_executor.py | 594 ++++++++++++++++++ .../paid_provider_canary_validation.py | 281 ++++++++- apps/api/src/services/telegram_gateway.py | 83 ++- .../test_agent99_telegram_lifecycle_api.py | 280 +++++++++ .../tests/test_awooop_truth_chain_service.py | 10 +- apps/api/tests/test_db_bounded_executor.py | 353 +++++++++++ .../test_host111_ollama_ansible_catalog.py | 110 +++- .../tests/test_paid_provider_canary_runner.py | 73 ++- .../test_paid_provider_canary_validation.py | 277 +++++++- ...3s_controlled_automation_work_items_api.py | 2 +- ...ructure-asset-reconciliation.snapshot.json | 6 +- ...rolled-automation-work-items.snapshot.json | 20 +- infra/ansible/inventory/hosts.yml | 2 +- .../ansible/playbooks/111-ollama-fallback.yml | 246 +++++--- .../test_cd_controlled_runtime_profile.py | 12 + scripts/ops/run-paid-provider-canary.py | 24 +- ..._db_bounded_executor_runtime_entrypoint.py | 118 ++++ .../agent99-remote-atomic-deploy-receiver.ps1 | 13 +- .../deploy-agent99-via-windows99-ssh.sh | 9 +- ...remote_atomic_deploy_transport_contract.py | 5 +- 32 files changed, 3041 insertions(+), 157 deletions(-) create mode 100644 agent99-db-bounded-executor.ps1 create mode 100644 apps/api/src/services/db_bounded_executor.py create mode 100644 apps/api/tests/test_db_bounded_executor.py create mode 100644 scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 3edee3b9f..2c6cc9286 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -4931,7 +4931,11 @@ jobs: # post-deploy smokes would only retest the previous production artifact. if: ${{ github.event_name != 'push' || (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'cancel-stale-cd') && !contains(github.event.head_commit.message, '[metadata-only]')) }} needs: [build-and-deploy] - timeout-minutes: 30 + # 2026-07-16 Codex: the fail-hard pressure gate below may legitimately + # consume 20 minutes while a bounded backup and its metric freshness + # window drain. Preserve enough time for the independent post-deploy + # verifier instead of replacing one false failure with a job timeout. + timeout-minutes: 45 # 2026-04-30 Codex: keep post-deploy on the host runner too. Playwright # install-deps can also kill the act-managed job container with RWLayer=nil. runs-on: awoooi-non110-host @@ -4958,6 +4962,13 @@ jobs: - name: Wait for Host Web Build Pressure # 2026-06-28 Codex: post-deploy is browser-heavy; fail closed on host # pressure until runner load is isolated from production. + # 2026-07-16 Codex: the bounded production pg_dump can legitimately run + # for up to 15 minutes and Docker CPU metrics can remain fresh for five + # more minutes. Keep the gate fail-hard, but align this post-deploy + # wait with that 20-minute evidence window so a healthy backup does not + # turn an already-successful rollout into a false CD terminal failure. + env: + HOST_WEB_BUILD_PRESSURE_ATTEMPTS: "120" run: bash scripts/ci/wait-host-web-build-pressure.sh - name: Get Commit Info diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 68771bfd5..5904a2eb2 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -222,6 +222,7 @@ $agentFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-db-bounded-executor.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 97137bd92..7cb3d41c6 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -18,6 +18,7 @@ $requiredFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-db-bounded-executor.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -126,6 +127,7 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js } $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.ps1"), [Text.Encoding]::UTF8) +$dbBoundedExecutor = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-db-bounded-executor.ps1"), [Text.Encoding]::UTF8) $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) @@ -137,6 +139,7 @@ Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") - Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present" Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded" Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized" +Add-Check "database:fixed_bounded_executor" ($dbBoundedExecutor.Contains('$CommandId = "telegram_receipt_index_apply_v1"') -and $dbBoundedExecutor.Contains('$CanonicalAsset = "public.awooop_outbound_message"') -and $dbBoundedExecutor.Contains('$Kubectl = "/usr/local/bin/kubectl"') -and $dbBoundedExecutor.Contains('domain = "database"') -and $dbBoundedExecutor.Contains('dispatchOnly = $true') -and $dbBoundedExecutor.Contains('Invoke-Agent99ReadyApiPodReadback') -and $dbBoundedExecutor.Contains('Invoke-Agent99FixedDbStage "check" $ExecutorPod') -and $dbBoundedExecutor.Contains('Invoke-Agent99FixedDbStage "apply" $ExecutorPod') -and $dbBoundedExecutor.Contains('Invoke-Agent99FixedDbStage "verify" $VerifierPod') -and $dbBoundedExecutor.Contains('sudo -n $Kubectl exec') -and -not $dbBoundedExecutor.Contains("Invoke-Expression") -and -not $dbBoundedExecutor.Contains("-EncodedCommand") -and -not $dbBoundedExecutor.Contains("/usr/local/sbin/awoooi-db-bounded-executor")) "Agent99 dispatches one fixed database command through the allowlisted host120 kubectl path and a second Ready API pod verifier" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal" Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 46fc4b7b0..501687399 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1636,8 +1636,9 @@ function Invoke-AgentTelegramCardSelfTest { } function Invoke-AgentTelegramDeliverySelfTest { - $testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss") + $testId = "telegram-delivery-test-" + (Get-Date -Format "yyyyMMdd-HHmmss-fff") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) $data = [pscustomobject]@{ + id = $testId testId = $testId visualPath = $null } @@ -1754,10 +1755,24 @@ function Invoke-AgentCanonicalLifecycleIngress { [int]$TimeoutSeconds ) - Invoke-RestMethod -Method Post -Uri $Url -Headers @{ + $webResponse = Invoke-WebRequest -UseBasicParsing -Method Post -Uri $Url -Headers @{ "X-Agent99-Lifecycle-Token" = $Token "X-Project-ID" = "awoooi" } -ContentType "application/json; charset=utf-8" -Body ($Payload | ConvertTo-Json -Depth 8 -Compress) -TimeoutSec $TimeoutSeconds + $retryAfterSeconds = 0 + $retryAfterRaw = [string]$webResponse.Headers["Retry-After"] + $parsedRetryAfter = 0 + if ($retryAfterRaw -and [int]::TryParse($retryAfterRaw, [ref]$parsedRetryAfter)) { + $retryAfterSeconds = [math]::Min(2, [math]::Max(0, $parsedRetryAfter)) + } + $body = $webResponse.Content | ConvertFrom-Json + [pscustomobject]@{ + statusCode = [int]$webResponse.StatusCode + retryAfterSeconds = $retryAfterSeconds + body = $body + rawResponseStored = $false + secretValueLogged = $false + } } function Send-AgentTelegram { @@ -1921,8 +1936,45 @@ function Send-AgentTelegram { occurred_at = (Get-Date -Format o) } + $initialProviderSendPerformed = $false + $reconcileAttempted = $false + $canonicalIngressAttemptCount = 0 + $reconcileFailureCode = $null try { - $response = Invoke-AgentCanonicalLifecycleIngress $gatewayUrl $token $payload $timeoutSeconds + $ingress = Invoke-AgentCanonicalLifecycleIngress $gatewayUrl $token $payload $timeoutSeconds + $canonicalIngressAttemptCount = 1 + $response = $ingress.body + $initialProviderSendPerformed = Convert-AgentBool (Get-AgentObjectValue $response "provider_send_performed" $false) + if ([int]$ingress.statusCode -eq 202) { + $reconcileReceipt = Get-AgentObjectValue $response "reconcile_receipt" $null + $reconcileReceiptValid = [bool]( + [string](Get-AgentObjectValue $response "delivery_status" "") -eq "pending_unknown" -and + $initialProviderSendPerformed -and + $reconcileReceipt -and + [string](Get-AgentObjectValue $response "delivery_id" "") -eq $deliveryId -and + [string](Get-AgentObjectValue $response "incident_id" "") -eq [string]$card.incidentId -and + [string](Get-AgentObjectValue $reconcileReceipt "schema_version" "") -eq "agent99_telegram_lifecycle_reconcile_v1" -and + [string](Get-AgentObjectValue $reconcileReceipt "delivery_id" "") -eq $deliveryId -and + [string](Get-AgentObjectValue $reconcileReceipt "incident_id" "") -eq [string]$card.incidentId -and + [string](Get-AgentObjectValue $reconcileReceipt "state_key" "") -eq [string]$card.stateKey -and + [string](Get-AgentObjectValue $reconcileReceipt "provider_message_id" "") -match "^[1-9][0-9]{0,31}$" -and + [string](Get-AgentObjectValue $response "provider_message_id" "") -eq [string](Get-AgentObjectValue $reconcileReceipt "provider_message_id" "") -and + [string](Get-AgentObjectValue $reconcileReceipt "destination_binding" "") -match "^[0-9a-f]{64}$" -and + [string](Get-AgentObjectValue $response "destination_binding" "") -eq [string](Get-AgentObjectValue $reconcileReceipt "destination_binding" "") -and + [int]$ingress.retryAfterSeconds -gt 0 -and + [int]$ingress.retryAfterSeconds -le 2 + ) + if ($reconcileReceiptValid) { + $reconcileAttempted = $true + $payload["reconcile_receipt"] = $reconcileReceipt + Start-Sleep -Seconds ([int]$ingress.retryAfterSeconds) + $ingress = Invoke-AgentCanonicalLifecycleIngress $gatewayUrl $token $payload $timeoutSeconds + $canonicalIngressAttemptCount = 2 + $response = $ingress.body + } else { + $reconcileFailureCode = "canonical_gateway_reconcile_receipt_invalid" + } + } $sameIdentity = [bool]( [string](Get-AgentObjectValue $response "delivery_id" "") -eq $deliveryId -and [string](Get-AgentObjectValue $response "incident_id" "") -eq [string]$card.incidentId @@ -1931,6 +1983,7 @@ function Send-AgentTelegram { $destinationVerified = Convert-AgentBool (Get-AgentObjectValue $response "destination_binding_verified" $false) $providerMessageId = [string](Get-AgentObjectValue $response "provider_message_id" "") $deliveryOk = [bool]( + [int]$ingress.statusCode -eq 200 -and (Convert-AgentBool (Get-AgentObjectValue $response "ok" $false)) -and $sameIdentity -and $durableAck -and @@ -1943,10 +1996,19 @@ function Send-AgentTelegram { $attempt.visualError = [string](Get-AgentObjectValue $response "visual_delivery_status" "visual_delivery_not_verified") } $attempt.sent = $deliveryOk - $attempt.error = if ($deliveryOk) { $null } else { "canonical_gateway_receipt_not_verified" } + if (-not $deliveryOk -and -not $reconcileFailureCode -and [int]$ingress.statusCode -eq 202) { + $reconcileFailureCode = "canonical_gateway_reconcile_still_pending" + } + $attempt.error = if ($deliveryOk) { $null } elseif ($reconcileFailureCode) { $reconcileFailureCode } else { "canonical_gateway_receipt_not_verified" } $attempt.relay = [pscustomobject]@{ ok = $deliveryOk - providerSendPerformed = Convert-AgentBool (Get-AgentObjectValue $response "provider_send_performed" $false) + providerSendPerformed = [bool]( + $initialProviderSendPerformed -or + (Convert-AgentBool (Get-AgentObjectValue $response "provider_send_performed" $false)) + ) + reconcileProviderSendPerformed = if ($reconcileAttempted) { Convert-AgentBool (Get-AgentObjectValue $response "provider_send_performed" $false) } else { $null } + reconcileAttempted = $reconcileAttempted + canonicalIngressAttemptCount = $canonicalIngressAttemptCount routeStatus = [string](Get-AgentObjectValue $response "delivery_status" "unknown") durableAck = $durableAck destinationBindingVerified = $destinationVerified @@ -1962,7 +2024,10 @@ function Send-AgentTelegram { $attempt.error = if ($failure.errorCode) { [string]$failure.errorCode } else { "canonical_gateway_http_error" } $attempt.relay = [pscustomobject]@{ ok = $false - providerSendPerformed = $false + providerSendPerformed = $initialProviderSendPerformed + reconcileProviderSendPerformed = if ($reconcileAttempted) { $false } else { $null } + reconcileAttempted = $reconcileAttempted + canonicalIngressAttemptCount = $canonicalIngressAttemptCount routeStatus = "delivery_failed" durableAck = $false destinationBindingVerified = $false diff --git a/agent99-db-bounded-executor.ps1 b/agent99-db-bounded-executor.ps1 new file mode 100644 index 000000000..9feb59171 --- /dev/null +++ b/agent99-db-bounded-executor.ps1 @@ -0,0 +1,364 @@ +param( + [ValidateSet("Check", "Apply", "Verify")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [string]$TraceId, + [Parameter(Mandatory = $true)] + [string]$RunId, + [Parameter(Mandatory = $true)] + [string]$WorkItemId, + [Parameter(Mandatory = $true)] + [string]$SourceRevision, + [string]$AgentRoot = "C:\Wooo\Agent99" +) + +$ErrorActionPreference = "Stop" + +$CommandId = "telegram_receipt_index_apply_v1" +$CanonicalAsset = "public.awooop_outbound_message" +$MigrationSha256 = "50aa05dea1718475051bdf4d45b1361291ec2335512b4d5c7d0a7eb77f935f5e" +$NormalizedSqlSha256 = "7e2f18b618c5c97bf7b428e1b224d4ef117c5551d5dc66b84838b14e546bf63a" +$JumpHost = "192.168.0.110" +$ControlPlaneHost = "192.168.0.120" +$RemoteUser = "wooo" +$Kubectl = "/usr/local/bin/kubectl" +$Namespace = "awoooi-prod" +$PodSelector = "app=awoooi-api,environment=prod,system=awoooi" +$Container = "api" +$IdentityFile = Join-Path $AgentRoot "keys\agent99_ed25519" +$SafeIdPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$" +$ApiPodPattern = "^awoooi-api-[a-z0-9][a-z0-9-]{0,126}$" +$ExecutorPod = $null +$VerifierPod = $null +$TransientTransportOutputUsed = $false +$TransportCleanupSucceeded = $true + +function Write-Agent99DbTerminal { + param( + [string]$Terminal, + [string]$ErrorCode, + [object]$CheckReceipt, + [object]$ApplyReceipt, + [object]$VerifierReceipt, + [object]$WritesPerformed + ) + + [pscustomobject]@{ + schemaVersion = "agent99_db_bounded_executor_dispatch_v1" + commandId = $CommandId + canonicalAsset = $CanonicalAsset + domain = "database" + executor = "db_bounded_executor" + verifier = "db_independent_verifier" + executionHub = "host:192.168.0.99" + dispatchPath = "windows99_agent99_to_host110_to_host120_to_api_pod" + dispatchOnly = $true + windowsVmwareMutationPerformed = $false + kubectlPath = $Kubectl + namespace = $Namespace + podSelector = $PodSelector + executorPod = $ExecutorPod + verifierPod = $VerifierPod + traceId = $TraceId + runId = $RunId + workItemId = $WorkItemId + sourceRevision = $SourceRevision + migrationSha256 = $MigrationSha256 + normalizedSqlSha256 = $NormalizedSqlSha256 + terminal = $Terminal + errorCode = if ($ErrorCode) { $ErrorCode } else { $null } + checkReceipt = $CheckReceipt + applyReceipt = $ApplyReceipt + verifierReceipt = $VerifierReceipt + independentVerifier = [bool]( + $ApplyReceipt -and + $VerifierReceipt -and + [string]$ApplyReceipt.pod_name -and + [string]$VerifierReceipt.pod_name -and + [string]$ApplyReceipt.pod_name -ne [string]$VerifierReceipt.pod_name -and + [string]$ApplyReceipt.process_identity_sha256 -ne [string]$VerifierReceipt.process_identity_sha256 + ) + writesPerformed = $WritesPerformed + rawSqlAccepted = $false + arbitraryCommandAllowed = $false + crossDomainFallbackAllowed = $false + secretValueRead = $false + transientTransportOutputUsed = $TransientTransportOutputUsed + transientTransportOutputDeletedBeforeReceipt = [bool]($TransientTransportOutputUsed -and $TransportCleanupSucceeded) + rawTransportOutputPersisted = [bool]($TransientTransportOutputUsed -and -not $TransportCleanupSucceeded) + } | ConvertTo-Json -Compress -Depth 12 +} + +function Test-Agent99DbReceipt { + param([object]$Receipt, [string]$ExpectedStage, [string]$ExpectedPod) + + if (-not $Receipt) { return $false } + $terminal = [string]$Receipt.terminal + $terminalAllowed = switch ($ExpectedStage) { + "check" { $terminal -in @("check_ready", "blocked", "failed") } + "apply" { $terminal -in @("apply_verified_local", "blocked", "verification_failed", "failed") } + "verify" { $terminal -in @("verified_healthy", "verification_failed", "failed") } + default { $false } + } + $writeContract = if ($ExpectedStage -eq "apply") { + [bool]( + $Receipt.write_attempted -is [bool] -and + ( + $Receipt.writes_performed -is [bool] -or + ($terminal -eq "failed" -and $Receipt.write_attempted -eq $true -and $null -eq $Receipt.writes_performed) + ) + ) + } else { + [bool]($Receipt.write_attempted -eq $false -and $Receipt.writes_performed -eq $false) + } + $catalogContract = [bool]( + ($terminal -ne "check_ready" -or $Receipt.catalog.apply_ready -eq $true) -and + ($terminal -notin @("apply_verified_local", "verified_healthy") -or ( + $Receipt.catalog.index_shape_exact -eq $true -and + $Receipt.catalog.catalog_drift -eq $false + )) + ) + return [bool]( + $terminalAllowed -and + $writeContract -and + $catalogContract -and + [string]$Receipt.schema_version -eq "awoooi_db_bounded_executor_receipt_v1" -and + [string]$Receipt.command_id -eq $CommandId -and + [string]$Receipt.canonical_asset -eq $CanonicalAsset -and + [string]$Receipt.domain -eq "database" -and + [string]$Receipt.executor -eq "db_bounded_executor" -and + [string]$Receipt.verifier -eq "db_independent_verifier" -and + [string]$Receipt.mode -eq $ExpectedStage -and + [string]$Receipt.trace_id -eq $TraceId -and + [string]$Receipt.run_id -eq $RunId -and + [string]$Receipt.work_item_id -eq $WorkItemId -and + [string]$Receipt.source_revision -eq $SourceRevision -and + [string]$Receipt.migration_sha256 -eq $MigrationSha256 -and + [string]$Receipt.normalized_sql_sha256 -eq $NormalizedSqlSha256 -and + [string]$Receipt.pod_name -eq $ExpectedPod -and + [string]$Receipt.pod_name -match $ApiPodPattern -and + [string]$Receipt.process_identity_sha256 -match "^[0-9a-f]{64}$" -and + $Receipt.raw_sql_emitted -eq $false -and + $Receipt.secret_value_read -eq $false -and + $Receipt.arbitrary_command_allowed -eq $false -and + $Receipt.cross_domain_fallback_allowed -eq $false + ) +} + +function Invoke-Agent99ReadyApiPodReadback { + $remoteCommand = ( + "ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes " + + "-o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $RemoteUser " + + "$ControlPlaneHost sudo -n $Kubectl get pods " + + "--namespace $Namespace --selector '$PodSelector' " + + "--field-selector 'status.phase=Running' " + + "--output 'custom-columns=NAME:.metadata.name,PHASE:.status.phase,READY:.status.conditions[?(@.type==`"Ready`")].status,DELETING:.metadata.deletionTimestamp' " + + "--no-headers" + ) + $arguments = @( + "-i", $IdentityFile, + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=8", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=3", + "-l", $RemoteUser, + $JumpHost, + $remoteCommand + ) + + $token = [Guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-db-pods-$token.out" + $stderrPath = Join-Path $env:TEMP "agent99-db-pods-$token.err" + $script:TransientTransportOutputUsed = $true + try { + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments ` + -NoNewWindow -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath -PassThru + if (-not $process.WaitForExit(120 * 1000)) { + try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {} + return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_timeout"; pods = @() } + } + $process.WaitForExit() | Out-Null + if ($process.ExitCode -ne 0) { + return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_read_failed"; pods = @() } + } + + $readyPods = @{} + foreach ($line in @(Get-Content -LiteralPath $stdoutPath -ErrorAction Stop)) { + $trimmed = [string]$line.Trim() + if (-not $trimmed) { continue } + $columns = @($trimmed -split "\s+") + if ($columns.Count -ne 4 -or [string]$columns[0] -notmatch $ApiPodPattern -or [string]$columns[1] -ne "Running") { + return [pscustomobject]@{ ok = $false; errorCode = "pod_inventory_contract_failed"; pods = @() } + } + if ([string]$columns[2] -eq "true" -and [string]$columns[3] -eq "") { + $podName = [string]$columns[0] + $readyPods[$podName] = $true + } + } + $pods = @($readyPods.Keys | Sort-Object) + if ($pods.Count -lt 2) { + return [pscustomobject]@{ ok = $false; errorCode = "two_ready_api_pods_required"; pods = $pods } + } + return [pscustomobject]@{ ok = $true; errorCode = ""; pods = $pods } + } finally { + Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue + if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) { + $script:TransportCleanupSucceeded = $false + } + } +} + +function Invoke-Agent99FixedDbStage { + param( + [ValidateSet("check", "apply", "verify")][string]$Stage, + [string]$PodName + ) + + if ($PodName -notmatch $ApiPodPattern) { + return [pscustomobject]@{ ok = $false; errorCode = "target_pod_contract_failed"; receipt = $null } + } + $remoteCommand = ( + "ssh -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes " + + "-o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $RemoteUser " + + "$ControlPlaneHost sudo -n $Kubectl exec " + + "--namespace $Namespace --container $Container $PodName -- " + + "python -B -m src.services.db_bounded_executor " + + "--mode $Stage --command-id $CommandId --canonical-asset $CanonicalAsset " + + "--migration-sha256 $MigrationSha256 --normalized-sql-sha256 $NormalizedSqlSha256 " + + "--trace-id $TraceId --run-id $RunId --work-item-id $WorkItemId --source-revision $SourceRevision" + ) + $arguments = @( + "-i", $IdentityFile, + "-o", "BatchMode=yes", + "-o", "IdentitiesOnly=yes", + "-o", "StrictHostKeyChecking=yes", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=8", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=3", + "-l", $RemoteUser, + $JumpHost, + $remoteCommand + ) + + $token = [Guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-db-$token.out" + $stderrPath = Join-Path $env:TEMP "agent99-db-$token.err" + $script:TransientTransportOutputUsed = $true + try { + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $arguments ` + -NoNewWindow -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath -PassThru + $timeoutSeconds = if ($Stage -eq "apply") { 900 } else { 120 } + if (-not $process.WaitForExit($timeoutSeconds * 1000)) { + try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {} + return [pscustomobject]@{ ok = $false; errorCode = "dispatch_timeout"; receipt = $null } + } + $process.WaitForExit() | Out-Null + try { + $receipt = Get-Content -LiteralPath $stdoutPath -Raw | ConvertFrom-Json + } catch { + return [pscustomobject]@{ ok = $false; errorCode = "receipt_parse_failed"; receipt = $null } + } + if (-not (Test-Agent99DbReceipt $receipt $Stage $PodName)) { + return [pscustomobject]@{ ok = $false; errorCode = "receipt_contract_failed"; receipt = $null } + } + if ($process.ExitCode -ne 0) { + return [pscustomobject]@{ ok = $false; errorCode = "fixed_remote_stage_failed"; receipt = $receipt } + } + return [pscustomobject]@{ ok = $true; errorCode = ""; receipt = $receipt } + } finally { + Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue + if ((Test-Path -LiteralPath $stdoutPath) -or (Test-Path -LiteralPath $stderrPath)) { + $script:TransportCleanupSucceeded = $false + } + } +} + +foreach ($identity in @($TraceId, $RunId, $WorkItemId)) { + if ($identity -notmatch $SafeIdPattern) { + Write-Agent99DbTerminal "blocked" "identity_missing_or_invalid" $null $null $null $false + exit 64 + } +} +$SourceRevision = $SourceRevision.ToLowerInvariant() +if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { + Write-Agent99DbTerminal "blocked" "source_revision_missing_or_invalid" $null $null $null $false + exit 64 +} +if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { + Write-Agent99DbTerminal "blocked" "dedicated_identity_unavailable" $null $null $null $false + exit 65 +} +$podReadback = Invoke-Agent99ReadyApiPodReadback +if (-not $podReadback.ok) { + Write-Agent99DbTerminal "blocked" $podReadback.errorCode $null $null $null $false + exit 65 +} +$ExecutorPod = [string]$podReadback.pods[0] +$VerifierPod = [string]$podReadback.pods[1] +if ($ExecutorPod -notmatch $ApiPodPattern -or $VerifierPod -notmatch $ApiPodPattern -or $ExecutorPod -eq $VerifierPod) { + Write-Agent99DbTerminal "blocked" "independent_verifier_pod_contract_failed" $null $null $null $false + exit 65 +} + +if ($Mode -eq "Check") { + $check = Invoke-Agent99FixedDbStage "check" $ExecutorPod + if (-not $check.ok -or [string]$check.receipt.terminal -ne "check_ready") { + Write-Agent99DbTerminal "blocked" $(if ($check.errorCode) { $check.errorCode } else { "precheck_failed" }) $check.receipt $null $null $false + exit 1 + } + Write-Agent99DbTerminal "check_ready" "" $check.receipt $null $null $false + exit 0 +} + +if ($Mode -eq "Verify") { + $verify = Invoke-Agent99FixedDbStage "verify" $VerifierPod + if (-not $verify.ok -or [string]$verify.receipt.terminal -ne "verified_healthy") { + Write-Agent99DbTerminal "verification_failed" $(if ($verify.errorCode) { $verify.errorCode } else { "catalog_verification_failed" }) $null $null $verify.receipt $false + exit 1 + } + Write-Agent99DbTerminal "verified_healthy" "" $null $null $verify.receipt $false + exit 0 +} + +# Apply is the only write path and always includes no-write precheck plus a +# second Ready API pod verifier under the same trace/run/work-item identity. +$check = Invoke-Agent99FixedDbStage "check" $ExecutorPod +if (-not $check.ok -or [string]$check.receipt.terminal -ne "check_ready") { + Write-Agent99DbTerminal "blocked" $(if ($check.errorCode) { $check.errorCode } else { "precheck_failed" }) $check.receipt $null $null $false + exit 1 +} +$apply = Invoke-Agent99FixedDbStage "apply" $ExecutorPod +$applyWriteState = if ($apply.receipt -and $apply.receipt.writes_performed -eq $true) { + $true +} elseif ($apply.receipt -and $apply.receipt.write_attempted -eq $true -and $null -eq $apply.receipt.writes_performed) { + $null +} elseif (-not $apply.ok -and -not $apply.receipt) { + # A transport loss after dispatch cannot prove whether PostgreSQL accepted + # the concurrent index build. Preserve unknown instead of false-green. + $null +} else { + $false +} +if (-not $apply.ok -or [string]$apply.receipt.terminal -ne "apply_verified_local") { + Write-Agent99DbTerminal "apply_failed" $(if ($apply.errorCode) { $apply.errorCode } else { "bounded_apply_failed" }) $check.receipt $apply.receipt $null $applyWriteState + exit 1 +} +$verify = Invoke-Agent99FixedDbStage "verify" $VerifierPod +$independent = [bool]( + $verify.ok -and + [string]$verify.receipt.terminal -eq "verified_healthy" -and + [string]$apply.receipt.pod_name -ne [string]$verify.receipt.pod_name -and + [string]$apply.receipt.process_identity_sha256 -ne [string]$verify.receipt.process_identity_sha256 +) +if (-not $independent) { + Write-Agent99DbTerminal "verification_failed" $(if ($verify.errorCode) { $verify.errorCode } else { "independent_verifier_failed" }) $check.receipt $apply.receipt $verify.receipt $applyWriteState + exit 1 +} +Write-Agent99DbTerminal "verified_healthy" "" $check.receipt $apply.receipt $verify.receipt $applyWriteState +exit 0 diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 41d7f1ed8..e24c2432d 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -33,6 +33,7 @@ $runtimeFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-db-bounded-executor.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index e1a1d09f1..3b7b2f49f 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -50,6 +50,7 @@ from src.services.agent99_enterprise_ai_automation_work_items import ( ) from src.services.agent99_telegram_lifecycle import ( deliver_agent99_telegram_lifecycle, + reconcile_agent99_telegram_lifecycle, ) from src.services.agent_market_governance_snapshot import ( load_latest_agent_market_governance_snapshot, @@ -1355,7 +1356,10 @@ async def post_agent99_telegram_lifecycle( request_id=payload.delivery_id, ) try: - receipt = await deliver_agent99_telegram_lifecycle(payload) + if payload.reconcile_receipt is not None: + receipt = await reconcile_agent99_telegram_lifecycle(payload) + else: + receipt = await deliver_agent99_telegram_lifecycle(payload) finally: clear_project_context(context_tokens) if receipt.get("ok") is not True: diff --git a/apps/api/src/models/agent99_completion.py b/apps/api/src/models/agent99_completion.py index c115ae871..ac8244215 100644 --- a/apps/api/src/models/agent99_completion.py +++ b/apps/api/src/models/agent99_completion.py @@ -124,6 +124,27 @@ class Agent99TelegramVisual(BaseModel): return base64.b64decode(self.content_base64, validate=True) +class Agent99TelegramLifecycleReconcileReceipt(BaseModel): + """Bound one retry to the exact provider delivery returned by a prior 202.""" + + model_config = ConfigDict(extra="forbid", str_strip_whitespace=True) + + schema_version: Literal["agent99_telegram_lifecycle_reconcile_v1"] + delivery_id: str = Field( + min_length=12, + max_length=180, + pattern=r"^[A-Za-z0-9_.:-]+$", + ) + incident_id: str = Field( + min_length=3, + max_length=180, + pattern=r"^[A-Za-z0-9_.:-]+$", + ) + state_key: str = Field(min_length=3, max_length=96) + provider_message_id: str = Field(pattern=r"^[1-9][0-9]{0,31}$") + destination_binding: str = Field(pattern=r"^[0-9a-f]{64}$") + + class Agent99TelegramLifecycleRequest(BaseModel): """Bounded Agent99 event card handed to the canonical Telegram gateway.""" @@ -169,4 +190,18 @@ class Agent99TelegramLifecycleRequest(BaseModel): next_update: str = Field(min_length=1, max_length=300) reply_to_message_id: int | None = Field(default=None, ge=1) visual: Agent99TelegramVisual | None = None + reconcile_receipt: Agent99TelegramLifecycleReconcileReceipt | None = None occurred_at: datetime + + @model_validator(mode="after") + def validate_reconcile_identity(self) -> Agent99TelegramLifecycleRequest: + receipt = self.reconcile_receipt + if receipt is None: + return self + if ( + receipt.delivery_id != self.delivery_id + or receipt.incident_id != self.incident_id + or receipt.state_key != self.state_key + ): + raise ValueError("agent99_lifecycle_reconcile_identity_mismatch") + return self diff --git a/apps/api/src/services/agent99_telegram_lifecycle.py b/apps/api/src/services/agent99_telegram_lifecycle.py index c2c0d4ae0..8351b59e4 100644 --- a/apps/api/src/services/agent99_telegram_lifecycle.py +++ b/apps/api/src/services/agent99_telegram_lifecycle.py @@ -28,6 +28,26 @@ _SEVERITY_LABELS = { _DURABLE_RECONCILE_DELAYS_SECONDS = (0.25, 0.5, 1.0) +def _bounded_reconcile_receipt( + payload: Agent99TelegramLifecycleRequest, + *, + provider_message_id: str, + destination_binding: str, +) -> dict[str, str] | None: + """Return the only client input allowed to resume a pending DB ack.""" + + if not provider_message_id or not destination_binding: + return None + return { + "schema_version": "agent99_telegram_lifecycle_reconcile_v1", + "delivery_id": payload.delivery_id, + "incident_id": payload.incident_id, + "state_key": payload.state_key, + "provider_message_id": provider_message_id, + "destination_binding": destination_binding, + } + + def _format_lifecycle_card(payload: Agent99TelegramLifecycleRequest) -> str: def esc(value: object) -> str: return html.escape(str(value or "")) @@ -133,6 +153,9 @@ async def deliver_agent99_telegram_lifecycle( provider_destination_verified = bool( initial_delivery_context.get("destination_binding_verified") is True ) + initial_destination_binding = str( + initial_delivery_context.get("destination_binding") or "" + ) reconciliation_attempts = 0 # A provider send can finish before the durable outbox finalizer becomes @@ -156,6 +179,9 @@ async def deliver_agent99_telegram_lifecycle( expected_provider_message_id=( provider_message_id_hint or None ), + expected_destination_binding=( + initial_destination_binding or None + ), ) result = response.get("result") if isinstance(response, dict) else None @@ -175,6 +201,11 @@ async def deliver_agent99_telegram_lifecycle( if isinstance(response, dict) else "" ) + destination_binding = str( + delivery_context.get("destination_binding") + or initial_destination_binding + or "" + ) durable_ack = bool( isinstance(response, dict) and response.get("_awooop_outbound_mirror_acknowledged") is True @@ -205,10 +236,23 @@ async def deliver_agent99_telegram_lifecycle( "event_type": payload.event_type, "lifecycle": payload.lifecycle, "provider_message_id": message_id or None, + "destination_binding": destination_binding or None, "delivery_status": delivery_status or "unknown", "durable_outbound_acknowledged": durable_ack, "destination_binding_verified": destination_verified, "provider_send_performed": provider_send_performed, + "reconcile_receipt": ( + _bounded_reconcile_receipt( + payload, + provider_message_id=message_id, + destination_binding=destination_binding, + ) + if ( + delivery_status == "pending_unknown" + and provider_send_performed + ) + else None + ), "durable_reconciliation_attempts": reconciliation_attempts, "visual_requested": visual_requested, "visual_sent": visual_sent, @@ -222,3 +266,95 @@ async def deliver_agent99_telegram_lifecycle( "raw_response_stored": False, "raw_visual_stored": False, } + + +async def reconcile_agent99_telegram_lifecycle( + payload: Agent99TelegramLifecycleRequest, +) -> dict[str, Any]: + """Finalize/read the exact prior 202 outbox row without provider egress.""" + + receipt = payload.reconcile_receipt + if receipt is None: + raise ValueError("agent99_lifecycle_reconcile_receipt_missing") + + gateway = get_telegram_gateway() + response = await gateway.reconcile_agent99_lifecycle_receipt( + delivery_id=payload.delivery_id, + incident_id=payload.incident_id, + state_key=payload.state_key, + project_id=payload.project_id, + expected_provider_message_id=receipt.provider_message_id, + expected_destination_binding=receipt.destination_binding, + ) + result = response.get("result") if isinstance(response, dict) else None + provider_message_id = ( + str(result.get("message_id") or "") + if isinstance(result, dict) + else "" + ) or receipt.provider_message_id + delivery_status = ( + str(response.get("_awooop_delivery_status") or "") + if isinstance(response, dict) + else "" + ) + delivery_context = ( + response.get("_awooop_delivery_context") + if isinstance(response, dict) + and isinstance(response.get("_awooop_delivery_context"), dict) + else {} + ) + destination_binding = str( + delivery_context.get("destination_binding") or "" + ) + durable_ack = bool( + isinstance(response, dict) + and response.get("_awooop_outbound_mirror_acknowledged") is True + ) + destination_verified = bool( + delivery_context.get("destination_binding_verified") is True + and destination_binding == receipt.destination_binding + ) + ok = bool( + durable_ack + and destination_verified + and provider_message_id == receipt.provider_message_id + and delivery_status == "sent_reused" + ) + visual_requested = payload.visual is not None + visual_sent = bool( + ok + and visual_requested + and isinstance(response, dict) + and response.get("_awooop_visual_delivery_verified") is True + ) + return { + "schema_version": "agent99_telegram_lifecycle_receipt_v1", + "ok": ok, + "delivery_id": payload.delivery_id, + "incident_id": payload.incident_id, + "event_type": payload.event_type, + "lifecycle": payload.lifecycle, + "provider_message_id": provider_message_id or None, + "destination_binding": destination_binding or None, + "delivery_status": delivery_status or "unknown", + "durable_outbound_acknowledged": durable_ack, + "destination_binding_verified": destination_verified, + "provider_send_performed": False, + "reconcile_receipt": ( + receipt.model_dump(mode="json") + if delivery_status == "pending_unknown" + else None + ), + "durable_reconciliation_attempts": 1, + "visual_requested": visual_requested, + "visual_sent": visual_sent, + "visual_delivery_status": ( + delivery_status if visual_requested else "not_requested" + ), + "visual_sha256": ( + payload.visual.sha256 if payload.visual is not None else None + ), + "stores_secret": False, + "raw_response_stored": False, + "raw_visual_stored": False, + } diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 64a62556e..16dcd2fb6 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -605,7 +605,7 @@ _CATALOG: tuple[dict[str, Any], ...] = ( "approval_required": False, "risk_level": "medium", "canonical_asset_id": "ai-provider:ollama_local", - "catalog_revision": "2026-07-16-host111-launchagent-k3s-path-v1", + "catalog_revision": "2026-07-16-host111-launchagent-k3s-path-v2", }, { "catalog_id": "ansible:110-host-pressure-readonly", diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 18f9db704..131321520 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -774,6 +774,7 @@ def _resolve_playbook_path(playbook_root: Path, playbook_path: str) -> Path: def _ansible_command_env( playbook_root: Path, *, + known_hosts_path: Path, allow_become_password_file: bool = False, ) -> dict[str, str]: roles_path = str((playbook_root / "roles").resolve()) @@ -785,6 +786,15 @@ def _ansible_command_env( "ANSIBLE_HOST_KEY_CHECKING": "true", "ANSIBLE_RETRY_FILES_ENABLED": "false", "ANSIBLE_ROLES_PATH": roles_path, + # Keep transport safety in ssh_args so host-specific + # ansible_ssh_common_args (notably Host111's fixed ProxyJump) remain + # effective. Passing ansible_ssh_common_args through --extra-vars + # would override the inventory route entirely. + "ANSIBLE_SSH_ARGS": ( + f"-o UserKnownHostsFile={known_hosts_path} " + "-o StrictHostKeyChecking=yes " + "-o IdentitiesOnly=yes -o BatchMode=yes -o ConnectTimeout=10" + ), } become_password_file = Path( settings.AWOOOP_ANSIBLE_BECOME_PASSWORD_FILE @@ -820,13 +830,8 @@ def build_ansible_check_mode_command( playbook_abs = _resolve_playbook_path(root, playbook_path) ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path() known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path() - ssh_common_args = ( - f"-o UserKnownHostsFile={known_hosts_path} " - "-o IdentitiesOnly=yes -o BatchMode=yes" - ) extra_vars = { "ansible_ssh_private_key_file": str(ssh_key_path), - "ansible_ssh_common_args": ssh_common_args, } command = [ "ansible-playbook", @@ -842,6 +847,7 @@ def build_ansible_check_mode_command( ] env = _ansible_command_env( root, + known_hosts_path=known_hosts_path, allow_become_password_file=( playbook_path == "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" @@ -879,13 +885,8 @@ def build_ansible_apply_command( playbook_abs = _resolve_playbook_path(root, playbook_path) ssh_key_path = check_mode_ssh_key_path or _check_mode_ssh_key_path() known_hosts_path = check_mode_known_hosts_path or _check_mode_known_hosts_path() - ssh_common_args = ( - f"-o UserKnownHostsFile={known_hosts_path} " - "-o IdentitiesOnly=yes -o BatchMode=yes" - ) extra_vars = { "ansible_ssh_private_key_file": str(ssh_key_path), - "ansible_ssh_common_args": ssh_common_args, } command = [ "ansible-playbook", @@ -900,6 +901,7 @@ def build_ansible_apply_command( ] env = _ansible_command_env( root, + known_hosts_path=known_hosts_path, allow_become_password_file=( playbook_path == "infra/ansible/playbooks/wazuh-alertmanager-integration.yml" diff --git a/apps/api/src/services/awooop_ansible_post_verifier.py b/apps/api/src/services/awooop_ansible_post_verifier.py index 547dd6dbb..4854014cb 100644 --- a/apps/api/src/services/awooop_ansible_post_verifier.py +++ b/apps/api/src/services/awooop_ansible_post_verifier.py @@ -21,6 +21,9 @@ VERIFIER_ID = "asset_specific_read_only_host_postconditions" INDEPENDENT_SOURCE = "broker_ssh_host_runtime_readback" _SAFE_HOST_ALIAS_RE = re.compile(r"^[A-Za-z0-9_.-]+$") _SAFE_REMOTE_ID_RE = re.compile(r"^[A-Za-z0-9_.-]+$") +_FIXED_PROXY_JUMP_BY_HOST = { + "host_111": "wooo@192.168.0.110", +} @dataclass(frozen=True) @@ -465,6 +468,9 @@ def _load_inventory_hosts(inventory_path: Path) -> dict[str, dict[str, str]]: hosts[str(alias)] = { "ansible_host": str(config.get("ansible_host") or ""), "ansible_user": str(config.get("ansible_user") or ""), + "ansible_ssh_common_args": str( + config.get("ansible_ssh_common_args") or "" + ), } return hosts @@ -484,7 +490,7 @@ def build_read_only_probe_command( user = str(target.get("ansible_user") or "") if not _SAFE_REMOTE_ID_RE.fullmatch(host) or not _SAFE_REMOTE_ID_RE.fullmatch(user): raise ValueError("postcondition_inventory_target_missing") - return [ + command = [ "ssh", "-i", str(ssh_key_path), @@ -498,9 +504,17 @@ def build_read_only_probe_command( "BatchMode=yes", "-o", "ConnectTimeout=10", - f"{user}@{host}", - condition.probe, ] + proxy_jump = _FIXED_PROXY_JUMP_BY_HOST.get(condition.inventory_host) + if proxy_jump is not None: + expected_common_args = ( + f"-o ProxyJump={proxy_jump} -o StrictHostKeyChecking=yes" + ) + if target.get("ansible_ssh_common_args") != expected_common_args: + raise ValueError("postcondition_proxy_jump_policy_mismatch") + command.extend(["-o", f"ProxyJump={proxy_jump}"]) + command.extend([f"{user}@{host}", condition.probe]) + return command async def _run_read_only_probe( diff --git a/apps/api/src/services/db_bounded_executor.py b/apps/api/src/services/db_bounded_executor.py new file mode 100644 index 000000000..1fc9ed8f0 --- /dev/null +++ b/apps/api/src/services/db_bounded_executor.py @@ -0,0 +1,594 @@ +"""Fixed database executor for the Telegram receipt hot-path index. + +This module is intentionally not a generic migration runner. It exposes one +command, one canonical table and one normalized SQL digest. Agent99 may +dispatch the command, but the routed domain remains ``database`` and the +actual check/apply/verify process runs inside allowlisted AWOOOI API pods. +""" + +from __future__ import annotations + +import argparse +import asyncio +import hashlib +import json +import os +import re +from collections.abc import Mapping +from typing import Any + +from sqlalchemy import text +from sqlalchemy.exc import SQLAlchemyError +from sqlalchemy.exc import TimeoutError as SQLAlchemyTimeoutError +from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine + +from src.db.base import get_engine + +SCHEMA_VERSION = "awoooi_db_bounded_executor_receipt_v1" +COMMAND_ID = "telegram_receipt_index_apply_v1" +CANONICAL_ASSET = "public.awooop_outbound_message" +DOMAIN = "database" +EXECUTOR = "db_bounded_executor" +VERIFIER = "db_independent_verifier" +INDEX_NAME = "idx_awooop_outbound_msg_telegram_receipt_latest" +MIGRATION_FILE = ( + "awooop_outbound_message_telegram_receipt_hot_path_index_2026-07-16.sql" +) +MIGRATION_SHA256 = "50aa05dea1718475051bdf4d45b1361291ec2335512b4d5c7d0a7eb77f935f5e" + +CREATE_INDEX_SQL = """\ +CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_awooop_outbound_msg_telegram_receipt_latest + ON awooop_outbound_message (project_id, queued_at DESC) + INCLUDE (send_status) + WHERE channel_type = 'telegram' + AND ( + source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result' + OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent' + ); +""" +NORMALIZED_SQL = " ".join(CREATE_INDEX_SQL.split()) +NORMALIZED_SQL_SHA256 = ( + "7e2f18b618c5c97bf7b428e1b224d4ef117c5551d5dc66b84838b14e546bf63a" +) + +_SAFE_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$") +_SOURCE_REVISION = re.compile(r"^[0-9a-f]{40}$") +_API_POD = re.compile(r"^awoooi-api-[a-z0-9][a-z0-9-]{0,126}$") +_LOCK_SQL = """ +SELECT pg_try_advisory_lock( + hashtextextended('awoooi/db/telegram_receipt_index_apply_v1', 0) +) AS acquired +""" +_UNLOCK_SQL = """ +SELECT pg_advisory_unlock( + hashtextextended('awoooi/db/telegram_receipt_index_apply_v1', 0) +) AS released +""" +_SESSION_GUARDS_SQL = ( + "SET search_path TO pg_catalog, public", + "SET lock_timeout TO '5s'", + "SET statement_timeout TO '10min'", +) +_CATALOG_SQL = f""" +WITH target AS ( + SELECT + to_regclass('{CANONICAL_ASSET}')::oid AS table_oid, + ( + SELECT c.relkind + FROM pg_class c + WHERE c.oid = to_regclass('{CANONICAL_ASSET}') + ) AS table_relkind, + ( + SELECT c.relowner + FROM pg_class c + WHERE c.oid = to_regclass('{CANONICAL_ASSET}') + ) AS table_owner_oid +), index_catalog AS ( + SELECT + i.indisready, + i.indisvalid, + i.indisunique, + i.indnkeyatts, + i.indnatts, + am.amname AS access_method, + pg_get_indexdef(i.indexrelid, 1, true) AS key_1, + pg_get_indexdef(i.indexrelid, 2, true) AS key_2, + pg_get_indexdef(i.indexrelid, 3, true) AS include_1, + pg_get_expr(i.indpred, i.indrelid) AS predicate + FROM target t + JOIN pg_index i ON i.indrelid = t.table_oid + JOIN pg_class index_class ON index_class.oid = i.indexrelid + JOIN pg_am am ON am.oid = index_class.relam + WHERE index_class.relname = '{INDEX_NAME}' +) +SELECT + t.table_oid IS NOT NULL AS table_exists, + COALESCE(( + SELECT count(*) = 5 + FROM pg_attribute a + WHERE a.attrelid = t.table_oid + AND a.attnum > 0 + AND NOT a.attisdropped + AND a.attname IN ( + 'project_id', 'queued_at', 'send_status', + 'channel_type', 'source_envelope' + ) + ), false) AS columns_complete, + COALESCE(t.table_relkind = 'r', false) AS table_kind_supported, + NOT pg_is_in_recovery() AS primary_database, + current_setting('transaction_read_only') = 'off' AS database_writable, + has_schema_privilege(current_user, 'public', 'USAGE') AS schema_usage, + has_schema_privilege(current_user, 'public', 'CREATE') AS schema_create, + -- The executor never issues SET ROLE. USAGE therefore proves that the + -- table owner's privileges are effective for current_user right now; + -- MEMBER alone would only prove that SET ROLE might be possible. + COALESCE( + pg_has_role(current_user, t.table_owner_oid, 'USAGE'), + false + ) AS table_owner_privilege_inherited, + COALESCE( + has_table_privilege(current_user, '{CANONICAL_ASSET}', 'SELECT'), + false + ) AS table_select, + (SELECT count(*) FROM index_catalog) AS index_count, + COALESCE((SELECT indisready FROM index_catalog), false) AS index_ready, + COALESCE((SELECT indisvalid FROM index_catalog), false) AS index_valid, + COALESCE((SELECT NOT indisunique FROM index_catalog), false) + AS index_non_unique, + COALESCE((SELECT indnkeyatts = 2 FROM index_catalog), false) + AS key_count_exact, + COALESCE((SELECT indnatts = 3 FROM index_catalog), false) + AS total_attribute_count_exact, + COALESCE((SELECT access_method = 'btree' FROM index_catalog), false) + AS access_method_exact, + COALESCE((SELECT key_1 FROM index_catalog), '') AS key_1, + COALESCE((SELECT key_2 FROM index_catalog), '') AS key_2, + COALESCE((SELECT include_1 FROM index_catalog), '') AS include_1, + COALESCE((SELECT predicate FROM index_catalog), '') AS predicate +FROM target t +""" + + +class BoundedExecutorContractError(ValueError): + """A caller or runtime tried to widen the fixed executor contract.""" + + +def _normalize_catalog_expression(value: str) -> str: + normalized = str(value or "").lower().replace('"', "") + normalized = normalized.replace("::text[]", "").replace("::text", "") + return re.sub(r"[\s()]", "", normalized) + + +_EXPECTED_PREDICATE = _normalize_catalog_expression( + """ + channel_type = 'telegram' + AND ( + source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result' + OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent' + ) + """ +) + + +def validate_execution_contract( + *, + mode: str, + command_id: str, + canonical_asset: str, + migration_sha256: str, + normalized_sql_sha256: str, + trace_id: str, + run_id: str, + work_item_id: str, + source_revision: str, + runtime_source_revision: str, + pod_name: str, +) -> None: + """Fail closed before opening a database connection.""" + + exact = { + "command_id": (command_id, COMMAND_ID), + "canonical_asset": (canonical_asset, CANONICAL_ASSET), + "migration_sha256": (migration_sha256, MIGRATION_SHA256), + "normalized_sql_sha256": ( + normalized_sql_sha256, + NORMALIZED_SQL_SHA256, + ), + } + for field, (actual, expected) in exact.items(): + if actual != expected: + raise BoundedExecutorContractError(f"{field}_contract_mismatch") + if mode not in {"check", "apply", "verify"}: + raise BoundedExecutorContractError("mode_not_allowlisted") + for name, value in { + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": work_item_id, + }.items(): + if not _SAFE_ID.fullmatch(value): + raise BoundedExecutorContractError(f"{name}_invalid") + if not _SOURCE_REVISION.fullmatch(source_revision): + raise BoundedExecutorContractError("source_revision_invalid") + if runtime_source_revision != source_revision: + raise BoundedExecutorContractError("runtime_source_revision_mismatch") + if not _API_POD.fullmatch(pod_name): + raise BoundedExecutorContractError("runtime_not_allowlisted_api_pod") + if hashlib.sha256(NORMALIZED_SQL.encode()).hexdigest() != NORMALIZED_SQL_SHA256: + raise BoundedExecutorContractError("embedded_sql_digest_mismatch") + + +def evaluate_catalog_snapshot(row: Mapping[str, Any]) -> dict[str, Any]: + """Convert a catalog-only row into a public-safe exact contract verdict.""" + + index_count = int(row.get("index_count") or 0) + index_exists = index_count == 1 + predicate_exact = ( + _normalize_catalog_expression(str(row.get("predicate") or "")) + == _EXPECTED_PREDICATE + ) + index_shape_exact = bool( + index_exists + and row.get("index_ready") is True + and row.get("index_valid") is True + and row.get("index_non_unique") is True + and row.get("key_count_exact") is True + and row.get("total_attribute_count_exact") is True + and row.get("access_method_exact") is True + and _normalize_catalog_expression(str(row.get("key_1") or "")) == "project_id" + and _normalize_catalog_expression(str(row.get("key_2") or "")) + == "queued_atdesc" + and _normalize_catalog_expression(str(row.get("include_1") or "")) + == "send_status" + and predicate_exact + ) + base_ready = bool( + row.get("table_exists") is True + and row.get("columns_complete") is True + and row.get("table_kind_supported") is True + and row.get("primary_database") is True + and row.get("database_writable") is True + and row.get("schema_usage") is True + and row.get("table_owner_privilege_inherited") is True + ) + drift = bool(index_count > 1 or (index_exists and not index_shape_exact)) + apply_ready = bool( + base_ready + and not drift + and (index_shape_exact or row.get("schema_create") is True) + ) + return { + "table_exists": row.get("table_exists") is True, + "columns_complete": row.get("columns_complete") is True, + "table_kind_supported": row.get("table_kind_supported") is True, + "primary_database": row.get("primary_database") is True, + "database_writable": row.get("database_writable") is True, + "schema_usage": row.get("schema_usage") is True, + "schema_create": row.get("schema_create") is True, + "table_owner_privilege_inherited": ( + row.get("table_owner_privilege_inherited") is True + ), + "table_select": row.get("table_select") is True, + "index_count": index_count, + "index_exists": index_exists, + "index_ready": row.get("index_ready") is True, + "index_valid": row.get("index_valid") is True, + "index_shape_exact": index_shape_exact, + "predicate_exact": predicate_exact, + "catalog_drift": drift, + "apply_ready": apply_ready, + } + + +async def _catalog_snapshot(connection: AsyncConnection) -> dict[str, Any]: + result = await connection.execute(text(_CATALOG_SQL)) + row = result.mappings().one() + return evaluate_catalog_snapshot(row) + + +def _receipt( + *, + mode: str, + trace_id: str, + run_id: str, + work_item_id: str, + source_revision: str, + pod_name: str, + terminal: str, + catalog: Mapping[str, Any], + writes_performed: bool | None, + write_attempted: bool = False, + error_code: str | None = None, +) -> dict[str, Any]: + role = { + "check": "precheck_executor", + "apply": "bounded_executor", + "verify": "independent_catalog_verifier", + }[mode] + process_identity = hashlib.sha256( + f"{pod_name}|{os.getpid()}|{run_id}|{mode}".encode() + ).hexdigest() + return { + "schema_version": SCHEMA_VERSION, + "command_id": COMMAND_ID, + "canonical_asset": CANONICAL_ASSET, + "domain": DOMAIN, + "executor": EXECUTOR, + "verifier": VERIFIER, + "mode": mode, + "executor_role": role, + "trace_id": trace_id, + "run_id": run_id, + "work_item_id": work_item_id, + "source_revision": source_revision, + "pod_name": pod_name, + "process_identity_sha256": process_identity, + "migration_file": MIGRATION_FILE, + "migration_sha256": MIGRATION_SHA256, + "normalized_sql_sha256": NORMALIZED_SQL_SHA256, + "terminal": terminal, + "catalog": dict(catalog), + "writes_performed": writes_performed, + "write_attempted": write_attempted, + "raw_sql_emitted": False, + "secret_value_read": False, + "arbitrary_command_allowed": False, + "cross_domain_fallback_allowed": False, + "rollback": "no_write_or_create_index_concurrently_only", + "error_code": error_code, + } + + +async def execute_bounded_db_command( + *, + mode: str, + command_id: str, + canonical_asset: str, + migration_sha256: str, + normalized_sql_sha256: str, + trace_id: str, + run_id: str, + work_item_id: str, + source_revision: str, + runtime_source_revision: str, + pod_name: str, + engine: AsyncEngine | None = None, +) -> dict[str, Any]: + """Execute the one fixed command with transactionless DDL semantics.""" + + validate_execution_contract( + mode=mode, + command_id=command_id, + canonical_asset=canonical_asset, + migration_sha256=migration_sha256, + normalized_sql_sha256=normalized_sql_sha256, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + runtime_source_revision=runtime_source_revision, + pod_name=pod_name, + ) + db_engine = engine + writes_performed = False + write_attempted = False + lock_acquired = False + try: + if db_engine is None: + db_engine = get_engine() + async with db_engine.connect() as raw_connection: + connection = await raw_connection.execution_options( + isolation_level="AUTOCOMMIT" + ) + for session_guard_sql in _SESSION_GUARDS_SQL: + await connection.execute(text(session_guard_sql)) + before = await _catalog_snapshot(connection) + if mode == "check": + terminal = "check_ready" if before["apply_ready"] else "blocked" + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal=terminal, + catalog=before, + writes_performed=False, + error_code=None if terminal == "check_ready" else "precheck_failed", + ) + if mode == "verify": + verified = before["index_shape_exact"] and not before["catalog_drift"] + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="verified_healthy" if verified else "verification_failed", + catalog=before, + writes_performed=False, + error_code=None if verified else "catalog_verification_failed", + ) + if not before["apply_ready"]: + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="blocked", + catalog=before, + writes_performed=False, + error_code="precheck_failed", + ) + lock_result = await connection.execute(text(_LOCK_SQL)) + lock_acquired = lock_result.scalar_one() is True + if not lock_acquired: + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="blocked", + catalog=before, + writes_performed=False, + error_code="single_flight_lock_unavailable", + ) + try: + locked = await _catalog_snapshot(connection) + if not locked["apply_ready"]: + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="blocked", + catalog=locked, + writes_performed=False, + error_code="locked_precheck_failed", + ) + if not locked["index_shape_exact"]: + write_attempted = True + await connection.execute(text(CREATE_INDEX_SQL)) + writes_performed = True + after = await _catalog_snapshot(connection) + verified = after["index_shape_exact"] and not after["catalog_drift"] + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal=( + "apply_verified_local" if verified else "verification_failed" + ), + catalog=after, + writes_performed=writes_performed, + write_attempted=write_attempted, + error_code=( + None if verified else "local_catalog_verification_failed" + ), + ) + finally: + try: + await connection.execute(text(_UNLOCK_SQL)) + except SQLAlchemyError: + pass + except (SQLAlchemyTimeoutError, TimeoutError): + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="failed", + catalog={}, + writes_performed=None if write_attempted else writes_performed, + write_attempted=write_attempted, + error_code="database_timeout", + ) + except SQLAlchemyError: + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="failed", + catalog={}, + writes_performed=None if write_attempted else writes_performed, + write_attempted=write_attempted, + error_code="database_operation_failed", + ) + except Exception: + return _receipt( + mode=mode, + trace_id=trace_id, + run_id=run_id, + work_item_id=work_item_id, + source_revision=source_revision, + pod_name=pod_name, + terminal="failed", + catalog={}, + writes_performed=None if write_attempted else writes_performed, + write_attempted=write_attempted, + error_code="executor_internal_error", + ) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Fixed AWOOOI DB executor") + parser.add_argument("--mode", required=True, choices=("check", "apply", "verify")) + parser.add_argument("--command-id", required=True) + parser.add_argument("--canonical-asset", required=True) + parser.add_argument("--migration-sha256", required=True) + parser.add_argument("--normalized-sql-sha256", required=True) + parser.add_argument("--trace-id", required=True) + parser.add_argument("--run-id", required=True) + parser.add_argument("--work-item-id", required=True) + parser.add_argument("--source-revision", required=True) + return parser + + +async def _async_main() -> int: + args = _parser().parse_args() + runtime_source_revision = os.getenv("AWOOOI_BUILD_COMMIT_SHA", "").lower() + pod_name = os.getenv("HOSTNAME", "") + try: + result = await execute_bounded_db_command( + mode=args.mode, + command_id=args.command_id, + canonical_asset=args.canonical_asset, + migration_sha256=args.migration_sha256, + normalized_sql_sha256=args.normalized_sql_sha256, + trace_id=args.trace_id, + run_id=args.run_id, + work_item_id=args.work_item_id, + source_revision=args.source_revision.lower(), + runtime_source_revision=runtime_source_revision, + pod_name=pod_name, + ) + except BoundedExecutorContractError as exc: + result = { + "schema_version": SCHEMA_VERSION, + "command_id": COMMAND_ID, + "canonical_asset": CANONICAL_ASSET, + "domain": DOMAIN, + "executor": EXECUTOR, + "verifier": VERIFIER, + "mode": args.mode, + "trace_id": args.trace_id, + "run_id": args.run_id, + "work_item_id": args.work_item_id, + "terminal": "blocked", + "writes_performed": False, + "write_attempted": False, + "secret_value_read": False, + "arbitrary_command_allowed": False, + "cross_domain_fallback_allowed": False, + "error_code": str(exc), + } + print(json.dumps(result, sort_keys=True, separators=(",", ":"))) + return ( + 0 + if result.get("terminal") + in { + "check_ready", + "apply_verified_local", + "verified_healthy", + } + else 1 + ) + + +if __name__ == "__main__": + raise SystemExit(asyncio.run(_async_main())) diff --git a/apps/api/src/services/paid_provider_canary_validation.py b/apps/api/src/services/paid_provider_canary_validation.py index b87607a60..49529aa86 100644 --- a/apps/api/src/services/paid_provider_canary_validation.py +++ b/apps/api/src/services/paid_provider_canary_validation.py @@ -14,9 +14,13 @@ import json import re from collections.abc import Awaitable, Callable from typing import Any -from uuid import uuid4 +from uuid import UUID, uuid4 + +from sqlalchemy import select from src.core.config import settings +from src.db.awooop_models import AwoooPRunState +from src.db.base import get_db_context from src.models.ai import OpenClawDecision, SuggestedAction from src.repositories.alert_operation_log_repository import ( AlertOperationLogRepository, @@ -35,6 +39,7 @@ from src.services.cloud_transport_receipt import ( resolve_gcp_transport_endpoint, ) from src.services.paid_provider_canary_authorization import ( + PAID_CANARY_PROJECT_ID, paid_provider_canary_bucket, paid_provider_canary_run_selected, ) @@ -404,7 +409,7 @@ async def _run_budget_readback(*, run_id: str) -> dict[str, Any]: } -async def run_paid_provider_canary_validation( +async def _run_paid_provider_canary_validation_core( *, run_ref: str, operator_id: str, @@ -417,6 +422,7 @@ async def run_paid_provider_canary_validation( operation_repository: AlertOperationLogRepository | None = None, cloud_transport_receipt_issuer: Callable[..., Awaitable[dict[str, Any]]] | None = None, + verified_authorization_receipt: dict[str, Any] | None = None, ) -> dict[str, Any]: """Run one five-lane comparison and a bounded paid-provider canary.""" @@ -435,9 +441,11 @@ async def run_paid_provider_canary_validation( # constructing provider clients, touching runtime control or making any # network call. The verifier deliberately returns no operator identity or # raw authorization payload. - authorization_receipt = await _require_paid_canary_authorization( - clean_authorization_ref - ) + authorization_receipt = verified_authorization_receipt + if authorization_receipt is None: + authorization_receipt = await _require_paid_canary_authorization( + clean_authorization_ref + ) clean_run_ref = str(run_ref or "").strip() if not _SAFE_RUN_REF.fullmatch(clean_run_ref): raise ValueError("invalid_paid_provider_canary_run_ref") @@ -1135,3 +1143,266 @@ async def run_paid_provider_canary_validation( else "rollback_unverified" ), ) + + +async def _read_paid_canary_gate5_terminal( + run_id: UUID, + *, + expected_state: str, + expected_trace_id: str, + expected_error_code: str | None, +) -> dict[str, Any]: + """Read back the exact non-sensitive Gate 5 terminal identity from PG.""" + + async with get_db_context(PAID_CANARY_PROJECT_ID) as db: + result = await db.execute( + select( + AwoooPRunState.run_id, + AwoooPRunState.project_id, + AwoooPRunState.state, + AwoooPRunState.completed_at, + AwoooPRunState.trace_id, + AwoooPRunState.error_code, + ).where( + AwoooPRunState.run_id == run_id, + AwoooPRunState.project_id == PAID_CANARY_PROJECT_ID, + ) + ) + row = result.one_or_none() + if row is None: + raise RuntimeError("paid_provider_canary_gate5_terminal_readback_missing") + + checks = { + "run_id_exact": row[0] == run_id, + "project_exact": row[1] == PAID_CANARY_PROJECT_ID, + "state_exact": row[2] == expected_state, + "completed_at_present": row[3] is not None, + "trace_id_exact": row[4] == expected_trace_id, + "error_code_exact": row[5] == expected_error_code, + } + failed_checks = [name for name, passed in checks.items() if passed is not True] + if failed_checks: + raise RuntimeError( + "paid_provider_canary_gate5_terminal_readback_failed:" + + ",".join(failed_checks) + ) + return { + "schema_version": "paid_provider_canary_gate5_terminal_readback_v1", + "run_id": str(row[0]), + "project_id": row[1], + "trace_id": row[4], + "state": row[2], + "completed_at": row[3].isoformat(), + "error_code": row[5], + "checks": checks, + "durable_readback_verified": True, + "same_run_trace_preserved": True, + } + + +async def _terminalize_paid_canary_gate5_run( + run_id: str, + *, + expected_trace_id: str, + succeeded: bool, + error_code: str | None = None, +) -> dict[str, Any]: + """Terminalize the exact authorized Gate 5 run without changing its trace.""" + + from src.services.run_state_machine import transition + + run_uuid = UUID(run_id) + expected_state = "completed" if succeeded else "failed" + await transition( + run_uuid, + PAID_CANARY_PROJECT_ID, + expected_state, + error_code=None if succeeded else error_code, + ) + return await _read_paid_canary_gate5_terminal( + run_uuid, + expected_state=expected_state, + expected_trace_id=expected_trace_id, + expected_error_code=None if succeeded else error_code, + ) + + +async def _shielded_terminalize_paid_canary_gate5_run( + run_id: str, + *, + expected_trace_id: str, + succeeded: bool, + error_code: str | None = None, +) -> dict[str, Any]: + """Finish the durable FSM write even when the caller is cancelled.""" + + task = asyncio.create_task( + _terminalize_paid_canary_gate5_run( + run_id, + expected_trace_id=expected_trace_id, + succeeded=succeeded, + error_code=error_code, + ) + ) + try: + return await asyncio.shield(task) + except BaseException: + return await task + + +async def _append_paid_canary_gate5_terminal_receipt( + repository: AlertOperationLogRepository, + *, + operator_id: str, + terminal_readback: dict[str, Any], +) -> tuple[bool, str | None]: + """Best-effort immutable projection; it never reopens a terminal FSM run.""" + + try: + row = await repository.append( + "CHANGE_APPLIED", + actor=f"awoooi_paid_provider_canary:{operator_id}"[:100], + action_detail="paid_provider_canary_gate5_terminal_readback"[:200], + success=terminal_readback.get("durable_readback_verified") is True, + context={ + "schema_version": "paid_provider_canary_gate5_terminal_receipt_v1", + "trace_id": terminal_readback.get("trace_id"), + "run_id": terminal_readback.get("run_id"), + "work_item_id": _WORK_ITEM_ID, + "terminal_readback": terminal_readback, + "raw_prompt_persisted": False, + "raw_response_persisted": False, + "secret_value_persisted_or_returned": False, + }, + project_id=PAID_CANARY_PROJECT_ID, + ) + except BaseException: + return False, None + return row is not None, str(getattr(row, "id", "")) or None + + +async def run_paid_provider_canary_validation( + *, + run_ref: str, + operator_id: str, + authorization_ref: str, + ollama_gcp_a_provider: Any | None = None, + ollama_gcp_b_provider: Any | None = None, + ollama_local_provider: Any | None = None, + claude_provider: Any | None = None, + gemini_provider: Any | None = None, + operation_repository: AlertOperationLogRepository | None = None, + cloud_transport_receipt_issuer: Callable[..., Awaitable[dict[str, Any]]] + | None = None, +) -> dict[str, Any]: + """Run the canary and close the same durable Gate 5 FSM identity.""" + + clean_authorization_ref = str(authorization_ref or "").strip() + # Establish that this is the exact approved Gate 5 row before any failure + # handler is allowed to mutate its FSM state. The core receives this receipt + # so the single-use check is not widened by a second authorization read. + authorization_receipt = await _require_paid_canary_authorization( + clean_authorization_ref + ) + repository = operation_repository or get_alert_operation_log_repository() + try: + receipt = await _run_paid_provider_canary_validation_core( + run_ref=run_ref, + operator_id=operator_id, + authorization_ref=clean_authorization_ref, + ollama_gcp_a_provider=ollama_gcp_a_provider, + ollama_gcp_b_provider=ollama_gcp_b_provider, + ollama_local_provider=ollama_local_provider, + claude_provider=claude_provider, + gemini_provider=gemini_provider, + operation_repository=repository, + cloud_transport_receipt_issuer=cloud_transport_receipt_issuer, + verified_authorization_receipt=authorization_receipt, + ) + except BaseException as exc: + # A held claim means a different caller owns the same exact run. Never + # terminalize underneath that caller; its single-use start receipt and + # terminal wrapper remain authoritative. + if isinstance(exc, RuntimeError) and exc.args == ( + "paid_canary_authorization_execution_claim_held", + ): + raise + + # The core performs owner-bound compensation whenever activation was + # attempted. This independent readback must happen before the FSM is + # marked failed, including failures before provider construction. + disabled_verified = False + try: + control_readback = await asyncio.wait_for( + _paid_canary_control_readback(), + timeout=5.0, + ) + disabled_verified = control_readback.get("disabled_verified") is True + except BaseException: + disabled_verified = False + try: + terminal_readback = await _shielded_terminalize_paid_canary_gate5_run( + clean_authorization_ref, + expected_trace_id=str(authorization_receipt.get("trace_id") or ""), + succeeded=False, + error_code=( + "E-PAID-CANARY-EXECUTION" + if disabled_verified + else "E-PAID-CANARY-ROLLBACK" + ), + ) + except BaseException as terminal_exc: + raise RuntimeError( + "paid_provider_canary_gate5_failure_terminalization_failed" + ) from terminal_exc + await _append_paid_canary_gate5_terminal_receipt( + repository, + operator_id=operator_id, + terminal_readback=terminal_readback, + ) + raise + + rollback_terminal = receipt.get("rollback_or_no_write_terminal") or {} + final_control_readback = (receipt.get("source_of_truth_diff") or {}).get( + "final_control_readback" + ) or {} + rollback_verified = rollback_terminal.get("rollback_verified") is True + disabled_verified = final_control_readback.get("disabled_verified") is True + verifier_passed = (receipt.get("independent_verifier") or {}).get("passed") is True + gate_completed = bool(verifier_passed and rollback_verified and disabled_verified) + terminal_error_code = None + if not gate_completed: + terminal_error_code = ( + "E-PAID-CANARY-VERIFY" + if rollback_verified and disabled_verified + else "E-PAID-CANARY-ROLLBACK" + ) + try: + terminal_readback = await _shielded_terminalize_paid_canary_gate5_run( + clean_authorization_ref, + expected_trace_id=str(authorization_receipt.get("trace_id") or ""), + succeeded=gate_completed, + error_code=terminal_error_code, + ) + except BaseException as exc: + raise RuntimeError("paid_provider_canary_gate5_terminalization_failed") from exc + + ( + terminal_aol_recorded, + terminal_aol_record_id, + ) = await _append_paid_canary_gate5_terminal_receipt( + repository, + operator_id=operator_id, + terminal_readback=terminal_readback, + ) + + receipt["gate5_run_terminal"] = { + **terminal_readback, + "rollback_verified_before_terminal": rollback_verified, + "disabled_readback_verified_before_terminal": disabled_verified, + "immutable_aol_terminal_receipt_recorded": terminal_aol_recorded, + "immutable_aol_terminal_receipt_record_id": terminal_aol_record_id, + } + if not gate_completed: + receipt["independent_verifier"]["passed"] = False + return receipt diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 734f0e758..da5548d1b 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -3644,6 +3644,14 @@ def _merge_outbound_source_envelope_extra( if isinstance(notification_policy, dict): envelope["notification_policy"] = notification_policy + canonical_route_receipt = extra.get("canonical_route_receipt") + if isinstance(canonical_route_receipt, dict): + envelope["canonical_route_receipt"] = canonical_route_receipt + + visual_delivery = extra.get("visual_delivery") + if isinstance(visual_delivery, dict): + envelope["visual_delivery"] = visual_delivery + incident_lifecycle_parent_receipt = extra.get( "incident_lifecycle_parent_receipt" ) @@ -5849,6 +5857,9 @@ class TelegramGateway: message_id::text AS message_id, send_status, provider_message_id, + source_envelope #>> + '{canonical_route_receipt,destination_binding}' + AS destination_binding, ( source_envelope #>> '{visual_delivery,requested}' @@ -6040,7 +6051,7 @@ class TelegramGateway: existing_row.get("send_status") == "sent" and provider_message_id ): - return { + sent_receipt: dict[str, object] = { "status": "sent", "run_id": str(delivery_run_id), "message_id": str(existing_row["message_id"]), @@ -6049,6 +6060,12 @@ class TelegramGateway: existing_row.get("visual_requested") is True ), } + destination_binding = str( + existing_row.get("destination_binding") or "" + ) + if destination_binding: + sent_receipt["destination_binding"] = destination_binding + return sent_receipt return { "status": "pending_unknown", "run_id": str(delivery_run_id), @@ -6081,6 +6098,7 @@ class TelegramGateway: identity: dict[str, str], reservation: dict[str, object], provider_message_id: str, + expected_destination_binding: str | None = None, ) -> bool: """Finalize only the reserved row, retrying DB acknowledgement only.""" from sqlalchemy import text @@ -6137,6 +6155,12 @@ class TelegramGateway: provider_message_id IS NULL OR provider_message_id = :provider_message_id ) + AND ( + :expected_destination_binding = '' + OR source_envelope #>> + '{canonical_route_receipt,destination_binding}' + = :expected_destination_binding + ) RETURNING send_status, provider_message_id """), { @@ -6144,6 +6168,9 @@ class TelegramGateway: "message_id": message_id, "run_id": run_id, "provider_message_id": provider_message_id, + "expected_destination_binding": str( + expected_destination_binding or "" + ), "triggered_by_state": delivery_kind, }, ) @@ -6188,6 +6215,9 @@ class TelegramGateway: message_id::text AS message_id, send_status, provider_message_id, + source_envelope #>> + '{canonical_route_receipt,destination_binding}' + AS destination_binding, ( source_envelope #>> '{visual_delivery,requested}' @@ -6246,13 +6276,19 @@ class TelegramGateway: ) send_status = str(readback.get("send_status") or "") if send_status == "sent" and provider_message_id: - return { + sent_receipt = { "status": "sent", "run_id": str(delivery_run_id), "message_id": message_id, "provider_message_id": provider_message_id, "visual_requested": readback.get("visual_requested") is True, } + destination_binding = str( + readback.get("destination_binding") or "" + ) + if destination_binding: + sent_receipt["destination_binding"] = destination_binding + return sent_receipt if send_status == "shadow": return { "status": "suppressed", @@ -6260,11 +6296,17 @@ class TelegramGateway: "message_id": message_id, } if send_status == "pending" and message_id: - return { + pending_receipt = { "status": "pending_unknown", "run_id": str(delivery_run_id), "message_id": message_id, } + destination_binding = str( + readback.get("destination_binding") or "" + ) + if destination_binding: + pending_receipt["destination_binding"] = destination_binding + return pending_receipt return { "status": "durable_state_invalid", "run_id": str(delivery_run_id), @@ -6276,12 +6318,27 @@ class TelegramGateway: *, identity: dict[str, str], expected_provider_message_id: str | None = None, + expected_destination_binding: str | None = None, ) -> dict[str, object]: """Finalize/read one existing row; never reserve or contact Telegram.""" readback = await self._read_controlled_apply_result_outbound( identity=identity ) provider_message_id = str(expected_provider_message_id or "").strip() + destination_binding = str( + expected_destination_binding or "" + ).strip() + durable_destination_binding = str( + readback.get("destination_binding") or "" + ).strip() + if ( + destination_binding + and durable_destination_binding != destination_binding + ): + return { + **readback, + "status": "destination_binding_mismatch", + } if readback.get("status") == "sent": durable_provider_message_id = str( readback.get("provider_message_id") or "" @@ -6302,6 +6359,7 @@ class TelegramGateway: identity=identity, reservation=readback, provider_message_id=provider_message_id, + expected_destination_binding=destination_binding or None, ) terminal_readback = await self._read_controlled_apply_result_outbound( identity=identity @@ -6675,6 +6733,9 @@ class TelegramGateway: identity=controlled_delivery_identity, reservation=controlled_reservation, provider_message_id=provider_message_id, + expected_destination_binding=( + route_destination_binding + ), ) ) result["_awooop_outbound_mirror_acknowledged"] = finalized @@ -11936,6 +11997,7 @@ class TelegramGateway: state_key: str, project_id: str = "awoooi", expected_provider_message_id: str | None = None, + expected_destination_binding: str | None = None, ) -> dict[str, object]: """Read/finalize one lifecycle outbox row without provider egress.""" identity = { @@ -11948,12 +12010,24 @@ class TelegramGateway: readback = await self._reconcile_controlled_apply_result_outbound( identity=identity, expected_provider_message_id=expected_provider_message_id, + expected_destination_binding=expected_destination_binding, ) delivery_status = str(readback.get("status") or "unknown") provider_message_id = str( readback.get("provider_message_id") or "" ) + destination_binding = str( + readback.get("destination_binding") or "" + ) sent = delivery_status == "sent" and bool(provider_message_id) + destination_binding_verified = bool( + sent + and destination_binding + and ( + not expected_destination_binding + or destination_binding == expected_destination_binding + ) + ) return { "ok": sent, "result": ( @@ -11968,7 +12042,8 @@ class TelegramGateway: sent and readback.get("visual_requested") is True ), "_awooop_delivery_context": { - "destination_binding_verified": sent, + "destination_binding": destination_binding, + "destination_binding_verified": destination_binding_verified, "durable_exact_row_readback": True, }, "_awooop_durable_readback_performed": True, diff --git a/apps/api/tests/test_agent99_telegram_lifecycle_api.py b/apps/api/tests/test_agent99_telegram_lifecycle_api.py index 20f1fe3a4..dfdbbd5c4 100644 --- a/apps/api/tests/test_agent99_telegram_lifecycle_api.py +++ b/apps/api/tests/test_agent99_telegram_lifecycle_api.py @@ -5,6 +5,7 @@ import base64 import hashlib import os import struct +from pathlib import Path from unittest.mock import AsyncMock os.environ.setdefault("DATABASE_URL", "postgresql+asyncpg://test:test@localhost/test") @@ -70,6 +71,36 @@ def app_client() -> TestClient: return TestClient(app) +def test_agent99_client_retries_one_202_with_exact_db_only_receipt() -> None: + source = ( + Path(__file__).resolve().parents[3] / "agent99-control-plane.ps1" + ).read_text(encoding="utf-8-sig") + send_section = source.split("function Send-AgentTelegram {", 1)[1].split( + "function Record-AgentEvent {", + 1, + )[0] + + assert send_section.count("Invoke-AgentCanonicalLifecycleIngress ") == 2 + assert 'if ([int]$ingress.statusCode -eq 202)' in send_section + assert '"agent99_telegram_lifecycle_reconcile_v1"' in send_section + assert '$payload["reconcile_receipt"] = $reconcileReceipt' in send_section + assert "Start-Sleep -Seconds ([int]$ingress.retryAfterSeconds)" in send_section + assert "[int]$ingress.retryAfterSeconds -le 2" in send_section + assert "canonicalIngressAttemptCount = 2" in send_section + assert "reconcileProviderSendPerformed" in send_section + assert "provider_message_id" in send_section + assert "destination_binding" in send_section + assert "X-Agent99-Lifecycle-Token" not in send_section + selftest_section = source.split( + "function Invoke-AgentTelegramDeliverySelfTest {", + 1, + )[1].split("function Invoke-AgentSensorGateSelfTest {", 1)[0] + assert 'id = $testId' in selftest_section + assert selftest_section.count( + 'Send-AgentTelegram "warning" "telegram_delivery_test"' + ) == 1 + + def test_lifecycle_payload_rejects_unbounded_content() -> None: with pytest.raises(ValidationError): Agent99TelegramLifecycleRequest.model_validate( @@ -90,6 +121,22 @@ def test_lifecycle_payload_rejects_unbounded_content() -> None: "visual": {**visual_payload(), "sha256": "0" * 64}, } ) + with pytest.raises(ValidationError, match="reconcile_identity_mismatch"): + Agent99TelegramLifecycleRequest.model_validate( + { + **payload(), + "reconcile_receipt": { + "schema_version": ( + "agent99_telegram_lifecycle_reconcile_v1" + ), + "delivery_id": payload()["delivery_id"], + "incident_id": "INC-WRONG", + "state_key": payload()["state_key"], + "provider_message_id": "8123", + "destination_binding": "a" * 64, + }, + } + ) def test_visual_caption_stays_within_telegram_limit_without_broken_html() -> None: @@ -208,6 +255,91 @@ def test_lifecycle_endpoint_returns_same_delivery_receipt(monkeypatch) -> None: assert response.json()["provider_message_id"] == "8123" +def test_lifecycle_endpoint_202_retry_is_db_only_and_returns_200( + monkeypatch, +) -> None: + monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected") + destination_binding = "a" * 64 + calls = {"deliver": 0, "reconcile": 0} + + async def pending_delivery(request): + calls["deliver"] += 1 + assert request.reconcile_receipt is None + return { + "schema_version": "agent99_telegram_lifecycle_receipt_v1", + "ok": False, + "delivery_id": request.delivery_id, + "incident_id": request.incident_id, + "delivery_status": "pending_unknown", + "provider_message_id": "8124", + "destination_binding": destination_binding, + "provider_send_performed": True, + "reconcile_receipt": { + "schema_version": "agent99_telegram_lifecycle_reconcile_v1", + "delivery_id": request.delivery_id, + "incident_id": request.incident_id, + "state_key": request.state_key, + "provider_message_id": "8124", + "destination_binding": destination_binding, + }, + } + + async def durable_reconcile(request): + calls["reconcile"] += 1 + assert request.reconcile_receipt is not None + assert request.reconcile_receipt.provider_message_id == "8124" + assert ( + request.reconcile_receipt.destination_binding + == destination_binding + ) + return { + "schema_version": "agent99_telegram_lifecycle_receipt_v1", + "ok": True, + "delivery_id": request.delivery_id, + "incident_id": request.incident_id, + "delivery_status": "sent_reused", + "provider_message_id": "8124", + "destination_binding": destination_binding, + "durable_outbound_acknowledged": True, + "destination_binding_verified": True, + "provider_send_performed": False, + } + + monkeypatch.setattr( + agents_api, + "deliver_agent99_telegram_lifecycle", + pending_delivery, + ) + monkeypatch.setattr( + agents_api, + "reconcile_agent99_telegram_lifecycle", + durable_reconcile, + ) + client = app_client() + first = client.post( + "/api/v1/agents/agent99/telegram-lifecycle", + headers={"X-Agent99-Lifecycle-Token": "expected"}, + json=payload(), + ) + + assert first.status_code == 202 + assert first.headers["retry-after"] == "2" + retry_payload = { + **payload(), + "reconcile_receipt": first.json()["reconcile_receipt"], + } + second = client.post( + "/api/v1/agents/agent99/telegram-lifecycle", + headers={"X-Agent99-Lifecycle-Token": "expected"}, + json=retry_payload, + ) + + assert second.status_code == 200 + assert second.json()["delivery_status"] == "sent_reused" + assert second.json()["provider_send_performed"] is False + assert calls == {"deliver": 1, "reconcile": 1} + + def test_lifecycle_outbox_identity_is_stable_and_separate() -> None: base = { "callback_reply": { @@ -249,15 +381,42 @@ def test_lifecycle_outbox_identity_is_stable_and_separate() -> None: ) != gateway_service._controlled_apply_result_delivery_run_id(controlled_identity) +def test_lifecycle_outbox_persists_destination_and_visual_bindings() -> None: + destination_binding = "a" * 64 + + envelope = gateway_service._merge_outbound_source_envelope_extra( + {"adapter": "legacy_telegram_gateway"}, + { + "canonical_route_receipt": { + "destination_binding": destination_binding, + }, + "visual_delivery": { + "requested": True, + "raw_visual_stored": False, + }, + }, + ) + + assert envelope["canonical_route_receipt"] == { + "destination_binding": destination_binding, + } + assert envelope["visual_delivery"] == { + "requested": True, + "raw_visual_stored": False, + } + + @pytest.mark.asyncio async def test_lifecycle_durable_readback_binds_exact_identity_without_egress( monkeypatch, ) -> None: gateway = object.__new__(gateway_service.TelegramGateway) + destination_binding = "a" * 64 reconcile = AsyncMock( return_value={ "status": "sent", "provider_message_id": "8124", + "destination_binding": destination_binding, "visual_requested": True, } ) @@ -272,12 +431,18 @@ async def test_lifecycle_durable_readback_binds_exact_identity_without_egress( incident_id=payload()["incident_id"], state_key=payload()["state_key"], expected_provider_message_id="8124", + expected_destination_binding=destination_binding, ) assert result["ok"] is True assert result["_awooop_delivery_status"] == "sent_reused" assert result["_awooop_provider_send_performed"] is False assert result["_awooop_durable_readback_performed"] is True + assert result["_awooop_delivery_context"] == { + "destination_binding": destination_binding, + "destination_binding_verified": True, + "durable_exact_row_readback": True, + } reconcile.assert_awaited_once_with( identity={ "project_id": "awoooi", @@ -287,9 +452,116 @@ async def test_lifecycle_durable_readback_binds_exact_identity_without_egress( "delivery_kind": "agent99_lifecycle", }, expected_provider_message_id="8124", + expected_destination_binding=destination_binding, ) +@pytest.mark.asyncio +async def test_lifecycle_reconcile_destination_mismatch_never_finalizes( + monkeypatch, +) -> None: + gateway = object.__new__(gateway_service.TelegramGateway) + readback = AsyncMock( + return_value={ + "status": "pending_unknown", + "run_id": "00000000-0000-0000-0000-000000000201", + "message_id": "00000000-0000-0000-0000-000000000202", + "destination_binding": "a" * 64, + } + ) + finalize = AsyncMock(return_value=True) + monkeypatch.setattr( + gateway, + "_read_controlled_apply_result_outbound", + readback, + ) + monkeypatch.setattr( + gateway, + "_finalize_controlled_apply_result_outbound", + finalize, + ) + + result = await gateway._reconcile_controlled_apply_result_outbound( + identity={ + "project_id": "awoooi", + "automation_run_id": payload()["delivery_id"], + "incident_id": payload()["incident_id"], + "apply_op_id": payload()["state_key"], + "delivery_kind": "agent99_lifecycle", + }, + expected_provider_message_id="8124", + expected_destination_binding="b" * 64, + ) + + assert result["status"] == "destination_binding_mismatch" + finalize.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_lifecycle_reconcile_service_never_enters_provider_sender( + monkeypatch, +) -> None: + destination_binding = "a" * 64 + calls: list[dict] = [] + + class Gateway: + async def send_agent99_lifecycle_receipt(self, **_kwargs): + raise AssertionError("provider sender must not run during reconcile") + + async def reconcile_agent99_lifecycle_receipt(self, **kwargs): + calls.append(kwargs) + return { + "ok": True, + "result": {"message_id": "8124"}, + "_awooop_outbound_mirror_acknowledged": True, + "_awooop_delivery_status": "sent_reused", + "_awooop_provider_send_performed": False, + "_awooop_delivery_context": { + "destination_binding": destination_binding, + "destination_binding_verified": True, + "durable_exact_row_readback": True, + }, + } + + monkeypatch.setattr( + lifecycle_service, + "get_telegram_gateway", + lambda: Gateway(), + ) + request = Agent99TelegramLifecycleRequest.model_validate( + { + **payload(), + "reconcile_receipt": { + "schema_version": "agent99_telegram_lifecycle_reconcile_v1", + "delivery_id": payload()["delivery_id"], + "incident_id": payload()["incident_id"], + "state_key": payload()["state_key"], + "provider_message_id": "8124", + "destination_binding": destination_binding, + }, + } + ) + + receipt = await lifecycle_service.reconcile_agent99_telegram_lifecycle( + request + ) + + assert receipt["ok"] is True + assert receipt["delivery_status"] == "sent_reused" + assert receipt["provider_send_performed"] is False + assert receipt["destination_binding_verified"] is True + assert calls == [ + { + "delivery_id": payload()["delivery_id"], + "incident_id": payload()["incident_id"], + "state_key": payload()["state_key"], + "project_id": "awoooi", + "expected_provider_message_id": "8124", + "expected_destination_binding": destination_binding, + } + ] + + @pytest.mark.asyncio async def test_lifecycle_service_uses_durable_canonical_sender(monkeypatch) -> None: calls: list[dict] = [] @@ -343,6 +615,7 @@ async def test_lifecycle_reconciles_pending_send_without_duplicate_provider_mess "_awooop_visual_delivery_verified": True, "_awooop_delivery_context": { "destination_binding_verified": True, + "destination_binding": "a" * 64, }, } reconciled_response = { @@ -395,6 +668,7 @@ async def test_lifecycle_reconciles_pending_send_without_duplicate_provider_mess "state_key": payload()["state_key"], "project_id": "awoooi", "expected_provider_message_id": "9126", + "expected_destination_binding": "a" * 64, } @@ -449,6 +723,7 @@ async def test_lifecycle_follower_only_reads_inflight_durable_delivery( assert len(send_calls) == 1 assert len(reconcile_calls) == 1 assert reconcile_calls[0]["expected_provider_message_id"] is None + assert reconcile_calls[0]["expected_destination_binding"] is None @pytest.mark.asyncio @@ -470,6 +745,7 @@ async def test_lifecycle_reconciliation_fails_closed_after_bounded_readbacks( "_awooop_provider_send_performed": True, "_awooop_delivery_context": { "destination_binding_verified": True, + "destination_binding": "a" * 64, }, } @@ -511,6 +787,10 @@ async def test_lifecycle_reconciliation_fails_closed_after_bounded_readbacks( call["expected_provider_message_id"] == "9127" for call in reconcile_calls ) + assert all( + call["expected_destination_binding"] == "a" * 64 + for call in reconcile_calls + ) @pytest.mark.asyncio diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 583fd7920..dbda93b3b 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -1736,7 +1736,10 @@ def test_ansible_check_mode_command_uses_check_diff_and_selected_ssh_transport(t assert "host_188" in spec.command assert "ansible_ssh_private_key_file" in spec.command[-1] assert str(repair_key) in spec.command[-1] - assert str(known_hosts) in spec.command[-1] + assert str(known_hosts) in spec.env["ANSIBLE_SSH_ARGS"] + assert "StrictHostKeyChecking=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "BatchMode=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "ansible_ssh_common_args" not in spec.command[-1] assert "apply" not in " ".join(spec.command) assert str((playbook_root / "roles").resolve()) in spec.env[ "ANSIBLE_ROLES_PATH" @@ -1770,7 +1773,10 @@ def test_ansible_apply_command_uses_controlled_apply_without_check(tmp_path: Pat assert "host_188" in spec.command assert "ansible_ssh_private_key_file" in spec.command[-1] assert str(repair_key) in spec.command[-1] - assert str(known_hosts) in spec.command[-1] + assert str(known_hosts) in spec.env["ANSIBLE_SSH_ARGS"] + assert "StrictHostKeyChecking=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "BatchMode=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "ansible_ssh_common_args" not in spec.command[-1] assert str((playbook_root / "roles").resolve()) in spec.env[ "ANSIBLE_ROLES_PATH" ].split(os.pathsep) diff --git a/apps/api/tests/test_db_bounded_executor.py b/apps/api/tests/test_db_bounded_executor.py new file mode 100644 index 000000000..71eea5519 --- /dev/null +++ b/apps/api/tests/test_db_bounded_executor.py @@ -0,0 +1,353 @@ +from __future__ import annotations + +# ruff: noqa: E402 +import hashlib +import json +import os +from pathlib import Path + +import pytest +from sqlalchemy.exc import SQLAlchemyError + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://test:test@localhost/test", +) + +from src.services.db_bounded_executor import ( + CANONICAL_ASSET, + COMMAND_ID, + CREATE_INDEX_SQL, + MIGRATION_FILE, + MIGRATION_SHA256, + NORMALIZED_SQL, + NORMALIZED_SQL_SHA256, + BoundedExecutorContractError, + evaluate_catalog_snapshot, + execute_bounded_db_command, + validate_execution_contract, +) + +ROOT = Path(__file__).resolve().parents[1] +SOURCE_REVISION = "a" * 40 +IDENTITY = { + "trace_id": "trace-AIA-SRE-013-db-index", + "run_id": "run-AIA-SRE-013-db-index", + "work_item_id": "AIA-SRE-013", +} + + +def _normalized_migration_sql() -> str: + source = (ROOT / "migrations" / MIGRATION_FILE).read_text() + sql_only = " ".join( + line for line in source.splitlines() if not line.lstrip().startswith("--") + ) + return " ".join(sql_only.split()) + + +def _absent_catalog() -> dict: + return { + "table_exists": True, + "columns_complete": True, + "table_kind_supported": True, + "primary_database": True, + "database_writable": True, + "schema_usage": True, + "schema_create": True, + "table_owner_privilege_inherited": True, + "table_select": True, + "index_count": 0, + "index_ready": False, + "index_valid": False, + "index_non_unique": False, + "key_count_exact": False, + "total_attribute_count_exact": False, + "access_method_exact": False, + "key_1": "", + "key_2": "", + "include_1": "", + "predicate": "", + } + + +def _exact_catalog() -> dict: + return { + **_absent_catalog(), + "index_count": 1, + "index_ready": True, + "index_valid": True, + "index_non_unique": True, + "key_count_exact": True, + "total_attribute_count_exact": True, + "access_method_exact": True, + "key_1": "project_id", + "key_2": "queued_at DESC", + "include_1": "send_status", + "predicate": """ + ((channel_type = 'telegram'::text) AND + (((source_envelope #>> '{callback_reply,action}'::text[]) = + 'controlled_apply_result'::text) OR + ((source_envelope #>> '{alert_notification,status}'::text[]) = + 'alert_notification_sent'::text))) + """, + } + + +def _contract(mode: str = "check") -> dict: + return { + "mode": mode, + "command_id": COMMAND_ID, + "canonical_asset": CANONICAL_ASSET, + "migration_sha256": MIGRATION_SHA256, + "normalized_sql_sha256": NORMALIZED_SQL_SHA256, + **IDENTITY, + "source_revision": SOURCE_REVISION, + "runtime_source_revision": SOURCE_REVISION, + "pod_name": "awoooi-api-7bc9867b77-alpha1", + } + + +def test_fixed_migration_and_embedded_sql_digests_match() -> None: + migration = (ROOT / "migrations" / MIGRATION_FILE).read_bytes() + + assert hashlib.sha256(migration).hexdigest() == MIGRATION_SHA256 + assert _normalized_migration_sql() == NORMALIZED_SQL + assert hashlib.sha256(NORMALIZED_SQL.encode()).hexdigest() == NORMALIZED_SQL_SHA256 + + +def test_embedded_sql_is_one_fixed_transactionless_create_only() -> None: + normalized = CREATE_INDEX_SQL.upper() + + assert normalized.count("CREATE INDEX CONCURRENTLY IF NOT EXISTS") == 1 + assert normalized.count(";") == 1 + for forbidden in ( + "DROP ", + "TRUNCATE ", + "DELETE ", + "UPDATE ", + "INSERT ", + "ALTER ", + "BEGIN", + "COMMIT", + ): + assert forbidden not in normalized + + +def test_precheck_requires_immediately_effective_owner_and_plain_table() -> None: + source = (ROOT / "src/services/db_bounded_executor.py").read_text() + + assert "pg_has_role(current_user, t.table_owner_oid, 'USAGE')" in source + assert "pg_has_role(current_user, t.table_owner_oid, 'MEMBER')" not in source + assert "COALESCE(t.table_relkind = 'r', false)" in source + + +@pytest.mark.parametrize( + ("field", "value", "error"), + ( + ("command_id", "arbitrary_sql_v1", "command_id_contract_mismatch"), + ("canonical_asset", "public.users", "canonical_asset_contract_mismatch"), + ("migration_sha256", "0" * 64, "migration_sha256_contract_mismatch"), + ( + "normalized_sql_sha256", + "f" * 64, + "normalized_sql_sha256_contract_mismatch", + ), + ("mode", "rollback", "mode_not_allowlisted"), + ("pod_name", "awoooi-worker-1", "runtime_not_allowlisted_api_pod"), + ( + "runtime_source_revision", + "b" * 40, + "runtime_source_revision_mismatch", + ), + ), +) +def test_contract_rejects_any_scope_or_runtime_drift( + field: str, + value: str, + error: str, +) -> None: + payload = _contract() + payload[field] = value + + with pytest.raises(BoundedExecutorContractError, match=error): + validate_execution_contract(**payload) + + +def test_catalog_precheck_allows_absent_index_but_rejects_drift() -> None: + absent = evaluate_catalog_snapshot(_absent_catalog()) + exact = evaluate_catalog_snapshot(_exact_catalog()) + drift_row = {**_exact_catalog(), "key_2": "queued_at"} + drift = evaluate_catalog_snapshot(drift_row) + non_owner = evaluate_catalog_snapshot( + {**_absent_catalog(), "table_owner_privilege_inherited": False} + ) + select_revoked = evaluate_catalog_snapshot( + {**_absent_catalog(), "table_select": False} + ) + partitioned_parent = evaluate_catalog_snapshot( + {**_absent_catalog(), "table_kind_supported": False} + ) + + assert absent["apply_ready"] is True + assert absent["index_exists"] is False + assert exact["index_shape_exact"] is True + assert exact["apply_ready"] is True + assert drift["catalog_drift"] is True + assert drift["apply_ready"] is False + assert non_owner["apply_ready"] is False + # CREATE INDEX requires table ownership, not a separate SELECT grant. + assert select_revoked["apply_ready"] is True + assert partitioned_parent["apply_ready"] is False + + +class _Result: + def __init__(self, *, row: dict | None = None, scalar: bool | None = None): + self.row = row + self.scalar = scalar + + def mappings(self): + return self + + def one(self): + assert self.row is not None + return self.row + + def scalar_one(self): + return self.scalar + + +class _Connection: + def __init__(self, catalog_rows: list[dict]): + self.catalog_rows = list(catalog_rows) + self.statements: list[str] = [] + self.isolation_level = "" + + async def execution_options(self, *, isolation_level: str): + self.isolation_level = isolation_level + return self + + async def execute(self, statement): + sql = str(statement) + self.statements.append(sql) + if "WITH target AS" in sql: + return _Result(row=self.catalog_rows.pop(0)) + if sql.startswith("SET "): + return _Result() + if "pg_try_advisory_lock" in sql: + return _Result(scalar=True) + if "pg_advisory_unlock" in sql: + return _Result(scalar=True) + if "CREATE INDEX CONCURRENTLY IF NOT EXISTS" in sql: + return _Result() + raise AssertionError(f"unexpected SQL: {sql}") + + +class _ConnectionContext: + def __init__(self, connection: _Connection): + self.connection = connection + + async def __aenter__(self): + return self.connection + + async def __aexit__(self, *_args): + return False + + +class _Engine: + def __init__(self, connection: _Connection): + self.connection = connection + self.connect_count = 0 + + def connect(self): + self.connect_count += 1 + return _ConnectionContext(self.connection) + + +class _CreateFailureConnection(_Connection): + async def execute(self, statement): + sql = str(statement) + if "CREATE INDEX CONCURRENTLY IF NOT EXISTS" in sql: + self.statements.append(sql) + raise SQLAlchemyError("sensitive backend detail must not escape") + return await super().execute(statement) + + +@pytest.mark.asyncio +async def test_apply_uses_autocommit_fixed_sql_and_same_session_lock_release() -> None: + connection = _Connection([_absent_catalog(), _absent_catalog(), _exact_catalog()]) + engine = _Engine(connection) + + receipt = await execute_bounded_db_command( + **_contract("apply"), + engine=engine, + ) + + assert receipt["terminal"] == "apply_verified_local" + assert receipt["writes_performed"] is True + assert receipt["domain"] == "database" + assert receipt["cross_domain_fallback_allowed"] is False + assert receipt["secret_value_read"] is False + assert connection.isolation_level == "AUTOCOMMIT" + assert engine.connect_count == 1 + assert connection.statements[:3] == [ + "SET search_path TO pg_catalog, public", + "SET lock_timeout TO '5s'", + "SET statement_timeout TO '10min'", + ] + assert sum("CREATE INDEX CONCURRENTLY" in sql for sql in connection.statements) == 1 + assert connection.statements.index(CREATE_INDEX_SQL) < next( + index + for index, sql in enumerate(connection.statements) + if "pg_advisory_unlock" in sql + ) + + +@pytest.mark.asyncio +async def test_apply_is_idempotent_when_exact_index_already_exists() -> None: + connection = _Connection([_exact_catalog(), _exact_catalog(), _exact_catalog()]) + engine = _Engine(connection) + + receipt = await execute_bounded_db_command( + **_contract("apply"), + engine=engine, + ) + + assert receipt["terminal"] == "apply_verified_local" + assert receipt["writes_performed"] is False + assert all("CREATE INDEX CONCURRENTLY" not in sql for sql in connection.statements) + + +@pytest.mark.asyncio +async def test_verify_is_catalog_only_and_never_takes_the_apply_lock() -> None: + connection = _Connection([_exact_catalog()]) + engine = _Engine(connection) + + receipt = await execute_bounded_db_command( + **_contract("verify"), + engine=engine, + ) + + assert receipt["terminal"] == "verified_healthy" + assert receipt["executor_role"] == "independent_catalog_verifier" + assert receipt["writes_performed"] is False + assert all("advisory_lock" not in sql for sql in connection.statements) + + +@pytest.mark.asyncio +async def test_failed_create_reports_unknown_write_state_without_backend_detail() -> ( + None +): + connection = _CreateFailureConnection([_absent_catalog(), _absent_catalog()]) + engine = _Engine(connection) + + receipt = await execute_bounded_db_command( + **_contract("apply"), + engine=engine, + ) + + assert receipt["terminal"] == "failed" + assert receipt["write_attempted"] is True + assert receipt["writes_performed"] is None + assert receipt["error_code"] == "database_operation_failed" + assert "sensitive backend detail" not in json.dumps(receipt) + assert any("pg_advisory_unlock" in sql for sql in connection.statements) diff --git a/apps/api/tests/test_host111_ollama_ansible_catalog.py b/apps/api/tests/test_host111_ollama_ansible_catalog.py index bb6964fb7..d9afc3ae1 100644 --- a/apps/api/tests/test_host111_ollama_ansible_catalog.py +++ b/apps/api/tests/test_host111_ollama_ansible_catalog.py @@ -1,5 +1,6 @@ from __future__ import annotations +import json from pathlib import Path import yaml @@ -8,7 +9,14 @@ from src.services.awooop_ansible_audit_service import ( _catalog_hints, get_ansible_catalog_item, ) -from src.services.awooop_ansible_post_verifier import postconditions_for_catalog +from src.services.awooop_ansible_check_mode_service import ( + build_ansible_apply_command, + build_ansible_check_mode_command, +) +from src.services.awooop_ansible_post_verifier import ( + build_read_only_probe_command, + postconditions_for_catalog, +) from src.services.controlled_alert_target_router import resolve_typed_alert_target ROOT = Path(__file__).resolve().parents[3] @@ -21,6 +29,18 @@ EXPECTED_ALLOWLIST = ( "127.0.0.1/32,192.168.0.111/32,192.168.0.188/32," "192.168.0.120/32,192.168.0.121/32" ) +EXPECTED_PROXY_JUMP = "wooo@192.168.0.110" + + +def _flatten_tasks(tasks: list[dict]) -> list[dict]: + flattened: list[dict] = [] + for task in tasks: + flattened.append(task) + for section in ("block", "rescue", "always"): + nested = task.get(section) + if isinstance(nested, list): + flattened.extend(_flatten_tasks(nested)) + return flattened def _incident(alertname: str = "OllamaLocalUnavailable") -> dict: @@ -94,7 +114,7 @@ def test_host111_catalog_and_postconditions_cover_all_runtime_origins() -> None: assert catalog["inventory_hosts"] == ["host_111", "host_120", "host_121"] assert catalog["domains"] == ["ai_provider", "ollama", "host_launchagent"] assert catalog["catalog_revision"] == ( - "2026-07-16-host111-launchagent-k3s-path-v1" + "2026-07-16-host111-launchagent-k3s-path-v2" ) assert {condition.condition_id for condition in conditions} == { @@ -134,9 +154,14 @@ def test_host111_playbook_is_launchagent_bounded_and_check_mode_safe() -> None: assert "systemctl" not in source assert "docker restart" not in source - tasks = {task["name"]: task for task in play["tasks"]} + tasks = {task["name"]: task for task in _flatten_tasks(play["tasks"])} assert tasks["Ollama111 | 讀取現行 allowlist"]["check_mode"] is False assert tasks["Ollama111 | 讀回生效 allowlist"]["check_mode"] is False + converge = tasks["Ollama111 | 收斂 K3s direct provider allowlist"] + assert converge["when"] == [ + "not ansible_check_mode", + "current_allowlist.stdout != allowed_cidrs", + ] check_receipt = tasks["Ollama111 | check-mode 發布精確 source diff"] assert check_receipt["when"] == "ansible_check_mode" assert check_receipt["ansible.builtin.debug"]["msg"]["persistent_writes_performed"] == 0 @@ -147,6 +172,85 @@ def test_host111_playbook_is_launchagent_bounded_and_check_mode_safe() -> None: assert path_readback["changed_when"] is False assert path_readback["ansible.builtin.uri"]["status_code"] == 200 + rollback = tasks["Ollama111 | rollback 恢復原 allowlist"] + assert rollback["when"] == "not ansible_check_mode" + assert rollback["ansible.builtin.command"]["argv"][2].endswith( + "{{ current_allowlist.stdout }}" + ) + assert tasks["Ollama111 | rollback 重新啟動 allow proxy"]["when"] == ( + "not ansible_check_mode" + ) + assert tasks["Ollama111 | rollback 讀回原 allowlist"]["check_mode"] is False + assert tasks["Ollama111 | rollback 驗證原 allowlist 已恢復"][ + "ansible.builtin.assert" + ]["that"] == ["rollback_allowlist.stdout == current_allowlist.stdout"] + assert tasks["Ollama111 | check-mode failure 維持 no-write"][ + "ansible.builtin.debug" + ]["msg"]["persistent_writes_performed"] == 0 + assert "ansible.builtin.fail" in tasks[ + "Ollama111 | apply 或 verifier 失敗後保持 failed terminal" + ] + + +def test_host111_broker_transport_preserves_fixed_proxyjump_and_strict_safety( + tmp_path: Path, +) -> None: + inventory = yaml.safe_load( + (ROOT / "infra" / "ansible" / "inventory" / "hosts.yml").read_text( + encoding="utf-8" + ) + ) + host111 = inventory["all"]["children"]["ollama_fallback"]["hosts"][ + "host_111" + ] + assert host111["ansible_ssh_common_args"] == ( + f"-o ProxyJump={EXPECTED_PROXY_JUMP} -o StrictHostKeyChecking=yes" + ) + assert "accept-new" not in host111["ansible_ssh_common_args"] + + key = tmp_path / "ssh_mcp_key" + known_hosts = tmp_path / "known_hosts" + for builder in (build_ansible_check_mode_command, build_ansible_apply_command): + spec = builder( + playbook_path="infra/ansible/playbooks/111-ollama-fallback.yml", + inventory_hosts=("host_111", "host_120", "host_121"), + playbook_root=ROOT / "infra" / "ansible", + check_mode_ssh_key_path=key, + check_mode_known_hosts_path=known_hosts, + ) + extra_vars = json.loads(spec.command[-1]) + assert extra_vars == {"ansible_ssh_private_key_file": str(key)} + assert "ansible_ssh_common_args" not in extra_vars + assert f"UserKnownHostsFile={known_hosts}" in spec.env["ANSIBLE_SSH_ARGS"] + assert "StrictHostKeyChecking=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "BatchMode=yes" in spec.env["ANSIBLE_SSH_ARGS"] + assert "ProxyJump" not in spec.env["ANSIBLE_SSH_ARGS"] + + +def test_host111_independent_verifier_uses_same_fixed_proxyjump( + tmp_path: Path, +) -> None: + condition = next( + row + for row in postconditions_for_catalog(CATALOG_ID) + if row.inventory_host == "host_111" + ) + key = tmp_path / "ssh_mcp_key" + known_hosts = tmp_path / "known_hosts" + + command = build_read_only_probe_command( + condition, + inventory_path=ROOT / "infra" / "ansible" / "inventory" / "hosts.yml", + ssh_key_path=key, + known_hosts_path=known_hosts, + ) + + assert f"ProxyJump={EXPECTED_PROXY_JUMP}" in command + assert "StrictHostKeyChecking=yes" in command + assert "BatchMode=yes" in command + assert "ooo@192.168.0.111" in command + assert command[-1] == condition.probe + def test_monitoring_registry_uses_launchagent_and_exact_catalog_action() -> None: payload = yaml.safe_load(MONITORING_REGISTRY.read_text(encoding="utf-8")) diff --git a/apps/api/tests/test_paid_provider_canary_runner.py b/apps/api/tests/test_paid_provider_canary_runner.py index dae215a26..bf990cf2d 100644 --- a/apps/api/tests/test_paid_provider_canary_runner.py +++ b/apps/api/tests/test_paid_provider_canary_runner.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import importlib.util import json import os @@ -7,10 +8,20 @@ import subprocess import sys from pathlib import Path +import pytest + REPO_ROOT = Path(__file__).resolve().parents[3] RUNNER = REPO_ROOT / "scripts" / "ops" / "run-paid-provider-canary.py" +def _load_runner_module(name: str): + spec = importlib.util.spec_from_file_location(name, RUNNER) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + def test_runner_help_bootstraps_api_import_from_repo_root() -> None: env = dict(os.environ) env["PYTHONDONTWRITEBYTECODE"] = "1" @@ -30,14 +41,59 @@ def test_runner_help_bootstraps_api_import_from_repo_root() -> None: assert "ModuleNotFoundError" not in result.stderr +def test_runner_initializes_and_closes_redis_around_validation(monkeypatch) -> None: + module = _load_runner_module("paid_canary_runner_lifecycle") + events: list[object] = [] + + async def _init() -> None: + events.append("init") + + async def _execute(**kwargs): + events.append(("execute", kwargs)) + return {"status": "ok"} + + async def _close() -> None: + events.append("close") + + monkeypatch.setattr(module, "_init_runtime", _init) + monkeypatch.setattr(module, "_execute_validation", _execute) + monkeypatch.setattr(module, "_close_runtime", _close) + + result = asyncio.run(module._run_validation(run_ref="source-sha")) + + assert result == {"status": "ok"} + assert events == ["init", ("execute", {"run_ref": "source-sha"}), "close"] + + +def test_runner_closes_redis_when_validation_fails(monkeypatch) -> None: + module = _load_runner_module("paid_canary_runner_failure_lifecycle") + events: list[str] = [] + + async def _init() -> None: + events.append("init") + + async def _execute(**_kwargs): + events.append("execute") + raise RuntimeError("bounded_failure") + + async def _close() -> None: + events.append("close") + + monkeypatch.setattr(module, "_init_runtime", _init) + monkeypatch.setattr(module, "_execute_validation", _execute) + monkeypatch.setattr(module, "_close_runtime", _close) + + with pytest.raises(RuntimeError, match="bounded_failure"): + asyncio.run(module._run_validation(run_ref="source-sha")) + + assert events == ["init", "execute", "close"] + + def test_runner_redacts_unexpected_exception_text( monkeypatch, capsys, ) -> None: - spec = importlib.util.spec_from_file_location("paid_canary_runner", RUNNER) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + module = _load_runner_module("paid_canary_runner") leak = "http://192.168.0.111:11434/?key=secret-like-value" @@ -69,10 +125,7 @@ def test_runner_returns_nonzero_for_partial_five_lane_validation( monkeypatch, capsys, ) -> None: - spec = importlib.util.spec_from_file_location("paid_canary_runner_partial", RUNNER) - assert spec is not None and spec.loader is not None - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) + module = _load_runner_module("paid_canary_runner_partial") async def _partial(**_kwargs): return { @@ -99,7 +152,5 @@ def test_runner_returns_nonzero_for_partial_five_lane_validation( assert module.main() == 1 payload = json.loads(capsys.readouterr().out) - assert payload["independent_verifier"][ - "paid_activation_verifier_passed" - ] is True + assert payload["independent_verifier"]["paid_activation_verifier_passed"] is True assert payload["independent_verifier"]["five_lane_verifier_passed"] is False diff --git a/apps/api/tests/test_paid_provider_canary_validation.py b/apps/api/tests/test_paid_provider_canary_validation.py index d88cd39a6..bed2c403f 100644 --- a/apps/api/tests/test_paid_provider_canary_validation.py +++ b/apps/api/tests/test_paid_provider_canary_validation.py @@ -4,7 +4,10 @@ from __future__ import annotations import asyncio import json import os +from contextlib import asynccontextmanager +from datetime import UTC, datetime from types import SimpleNamespace +from uuid import UUID import pytest @@ -17,9 +20,7 @@ from src.services.paid_provider_canary_validation import ( ) _AUTHORIZATION_REF = "267a3d26-3c29-470f-8fe3-388a2f408f39" -_AUTHORIZATION_TRACE_ID = ( - "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" -) +_AUTHORIZATION_TRACE_ID = "00-0123456789abcdef0123456789abcdef-0123456789abcdef-01" class _Provider: @@ -119,6 +120,7 @@ def _configure_runtime_control(monkeypatch) -> dict[str, object]: "active_completion_record_id": None, "lease_ttl": -2, "execution_claim_owner": None, + "gate5_terminal_transitions": [], } def _readback( @@ -277,6 +279,46 @@ def _configure_runtime_control(monkeypatch) -> dict[str, object]: "terminal_verified": True, } + async def _terminalize_gate5( + run_id: str, + *, + expected_trace_id: str, + succeeded: bool, + error_code: str | None = None, + ) -> dict: + assert run_id == _AUTHORIZATION_REF + assert expected_trace_id == _AUTHORIZATION_TRACE_ID + transitions = state["gate5_terminal_transitions"] + assert isinstance(transitions, list) + transitions.append( + { + "run_id": run_id, + "succeeded": succeeded, + "error_code": error_code, + "paid_routes_disabled": bool(state["claude"] and state["gemini"]), + "activation_lock_absent": state["lock_owner"] is None, + } + ) + return { + "schema_version": "paid_provider_canary_gate5_terminal_readback_v1", + "run_id": run_id, + "project_id": "awoooi", + "trace_id": expected_trace_id, + "state": "completed" if succeeded else "failed", + "completed_at": "2026-07-16T00:10:00", + "error_code": None if succeeded else error_code, + "checks": { + "run_id_exact": True, + "project_exact": True, + "state_exact": True, + "completed_at_present": True, + "trace_id_exact": True, + "error_code_exact": True, + }, + "durable_readback_verified": True, + "same_run_trace_preserved": True, + } + async def _authorization_readback(authorization_ref: str) -> dict: assert authorization_ref == _AUTHORIZATION_REF return { @@ -365,6 +407,11 @@ def _configure_runtime_control(monkeypatch) -> dict[str, object]: monkeypatch.setattr(module, "_rollback_paid_canary", _rollback) monkeypatch.setattr(module, "_generation_receipt_readback", _receipt_readback) monkeypatch.setattr(module, "_run_budget_readback", _run_budget_readback) + monkeypatch.setattr( + module, + "_terminalize_paid_canary_gate5_run", + _terminalize_gate5, + ) return state @@ -376,6 +423,154 @@ def test_canary_run_selection_is_deterministic_and_inside_percent() -> None: assert first[1] < 5 +@pytest.mark.asyncio +async def test_gate5_terminal_readback_requires_exact_durable_state_and_trace( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + + completed_at = datetime(2026, 7, 16, 0, 10, tzinfo=UTC) + + class _Result: + @staticmethod + def one_or_none(): + return ( + UUID(_AUTHORIZATION_REF), + "awoooi", + "completed", + completed_at, + _AUTHORIZATION_TRACE_ID, + None, + ) + + class _Db: + @staticmethod + async def execute(_statement): + return _Result() + + @asynccontextmanager + async def _db_context(project_id: str): + assert project_id == "awoooi" + yield _Db() + + monkeypatch.setattr(module, "get_db_context", _db_context) + + receipt = await module._read_paid_canary_gate5_terminal( + UUID(_AUTHORIZATION_REF), + expected_state="completed", + expected_trace_id=_AUTHORIZATION_TRACE_ID, + expected_error_code=None, + ) + + assert receipt["durable_readback_verified"] is True + assert receipt["state"] == "completed" + assert receipt["completed_at"] == completed_at.isoformat() + assert receipt["trace_id"] == _AUTHORIZATION_TRACE_ID + assert all(receipt["checks"].values()) + + with pytest.raises( + RuntimeError, + match="paid_provider_canary_gate5_terminal_readback_failed:trace_id_exact", + ): + await module._read_paid_canary_gate5_terminal( + UUID(_AUTHORIZATION_REF), + expected_state="completed", + expected_trace_id=( + "00-aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa-bbbbbbbbbbbbbbbb-01" + ), + expected_error_code=None, + ) + + +@pytest.mark.asyncio +async def test_gate5_terminal_transition_is_followed_by_exact_readback( + monkeypatch, +) -> None: + from src.services import paid_provider_canary_validation as module + from src.services import run_state_machine + + events: list[tuple] = [] + + async def _transition( + run_id: UUID, + project_id: str, + to_state: str, + *, + error_code: str | None = None, + ) -> None: + events.append(("transition", run_id, project_id, to_state, error_code)) + + async def _readback( + run_id: UUID, + *, + expected_state: str, + expected_trace_id: str, + expected_error_code: str | None, + ) -> dict: + events.append( + ( + "readback", + run_id, + expected_state, + expected_trace_id, + expected_error_code, + ) + ) + return {"durable_readback_verified": True, "state": expected_state} + + monkeypatch.setattr(run_state_machine, "transition", _transition) + monkeypatch.setattr(module, "_read_paid_canary_gate5_terminal", _readback) + + receipt = await module._terminalize_paid_canary_gate5_run( + _AUTHORIZATION_REF, + expected_trace_id=_AUTHORIZATION_TRACE_ID, + succeeded=False, + error_code="E-PAID-CANARY-VERIFY", + ) + + assert receipt == {"durable_readback_verified": True, "state": "failed"} + assert events == [ + ( + "transition", + UUID(_AUTHORIZATION_REF), + "awoooi", + "failed", + "E-PAID-CANARY-VERIFY", + ), + ( + "readback", + UUID(_AUTHORIZATION_REF), + "failed", + _AUTHORIZATION_TRACE_ID, + "E-PAID-CANARY-VERIFY", + ), + ] + + +@pytest.mark.asyncio +async def test_optional_terminal_aol_projection_cannot_reopen_terminal_run() -> None: + from src.services import paid_provider_canary_validation as module + + class _FailingRepository: + @staticmethod + async def append(*_args, **_kwargs): + raise RuntimeError("synthetic_aol_unavailable") + + recorded, record_id = await module._append_paid_canary_gate5_terminal_receipt( + _FailingRepository(), + operator_id="test-operator", + terminal_readback={ + "run_id": _AUTHORIZATION_REF, + "trace_id": _AUTHORIZATION_TRACE_ID, + "state": "completed", + "durable_readback_verified": True, + }, + ) + + assert recorded is False + assert record_id is None + + @pytest.mark.asyncio async def test_durable_authorization_blocks_before_receipt_or_provider_call( monkeypatch, @@ -441,7 +636,7 @@ async def test_atomic_execution_claim_blocks_concurrent_authorization_reuse( ALERT_OLLAMA_MODEL="qwen3:14b", ), ) - _configure_runtime_control(monkeypatch) + state = _configure_runtime_control(monkeypatch) async def _claim_held(_run_id: str, _owner: str) -> bool: return False @@ -472,6 +667,7 @@ async def test_atomic_execution_claim_blocks_concurrent_authorization_reuse( assert repository.rows == [] assert [provider.calls for provider in providers] == [0, 0, 0, 0, 0] + assert state["gate5_terminal_transitions"] == [] @pytest.mark.asyncio @@ -574,6 +770,7 @@ async def test_canary_discards_content_and_writes_bounded_receipt(monkeypatch) - "PRE_FLIGHT_PASSED", "CHANGE_APPLIED", "EXECUTION_COMPLETED", + "CHANGE_APPLIED", ] assert claude.closed is True assert gemini.closed is True @@ -591,6 +788,38 @@ async def test_canary_discards_content_and_writes_bounded_receipt(monkeypatch) - "validated_comparison_rollback_verified" ) assert receipt["policy_decision"]["persistent_promotion_authorized"] is False + assert receipt["gate5_run_terminal"] == { + "schema_version": "paid_provider_canary_gate5_terminal_readback_v1", + "run_id": _AUTHORIZATION_REF, + "project_id": "awoooi", + "trace_id": _AUTHORIZATION_TRACE_ID, + "state": "completed", + "completed_at": "2026-07-16T00:10:00", + "error_code": None, + "checks": { + "run_id_exact": True, + "project_exact": True, + "state_exact": True, + "completed_at_present": True, + "trace_id_exact": True, + "error_code_exact": True, + }, + "durable_readback_verified": True, + "same_run_trace_preserved": True, + "rollback_verified_before_terminal": True, + "disabled_readback_verified_before_terminal": True, + "immutable_aol_terminal_receipt_recorded": True, + "immutable_aol_terminal_receipt_record_id": "row-6", + } + assert state["gate5_terminal_transitions"] == [ + { + "run_id": _AUTHORIZATION_REF, + "succeeded": True, + "error_code": None, + "paid_routes_disabled": True, + "activation_lock_absent": True, + } + ] @pytest.mark.asyncio @@ -634,6 +863,17 @@ async def test_canary_verifier_rejects_a_mutating_false_positive(monkeypatch) -> assert state["lock_owner"] is None assert state["active_owner"] is None assert state["execution_claim_owner"] is None + assert receipt["gate5_run_terminal"]["state"] == "failed" + assert receipt["gate5_run_terminal"]["error_code"] == ("E-PAID-CANARY-VERIFY") + assert state["gate5_terminal_transitions"] == [ + { + "run_id": _AUTHORIZATION_REF, + "succeeded": False, + "error_code": "E-PAID-CANARY-VERIFY", + "paid_routes_disabled": True, + "activation_lock_absent": True, + } + ] @pytest.mark.asyncio @@ -723,6 +963,15 @@ async def test_missing_paid_generation_receipt_forces_atomic_rollback( assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None + assert state["gate5_terminal_transitions"] == [ + { + "run_id": _AUTHORIZATION_REF, + "succeeded": False, + "error_code": "E-PAID-CANARY-VERIFY", + "paid_routes_disabled": True, + "activation_lock_absent": True, + } + ] @pytest.mark.asyncio @@ -766,6 +1015,15 @@ async def test_no_paid_or_baseline_call_without_durable_start_receipt( assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None + assert state["gate5_terminal_transitions"] == [ + { + "run_id": _AUTHORIZATION_REF, + "succeeded": False, + "error_code": "E-PAID-CANARY-EXECUTION", + "paid_routes_disabled": True, + "activation_lock_absent": True, + } + ] @pytest.mark.asyncio @@ -977,9 +1235,14 @@ async def test_cancelled_paid_call_runs_verified_compensation(monkeypatch) -> No assert state["gemini"] is True assert state["lock_owner"] is None assert state["active_owner"] is None - assert repository.rows[-1]["event_type"] == "EXECUTION_COMPLETED" - assert repository.rows[-1]["success"] is False - assert repository.rows[-1]["context"]["rollback_verified"] is True + assert repository.rows[-2]["event_type"] == "EXECUTION_COMPLETED" + assert repository.rows[-2]["success"] is False + assert repository.rows[-2]["context"]["rollback_verified"] is True + assert repository.rows[-1]["event_type"] == "CHANGE_APPLIED" + assert repository.rows[-1]["action_detail"] == ( + "paid_provider_canary_gate5_terminal_readback" + ) + assert repository.rows[-1]["context"]["terminal_readback"]["state"] == ("failed") @pytest.mark.asyncio diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index ba35e266c..07c89ce3d 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -134,7 +134,7 @@ def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None: assert "public-HTTP" in items["AIA-SRE-013"]["exit_condition"] assert "durable Gate5" in " ".join(items["AIA-SRE-013"]["runtime_gaps"]) assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"]) - assert "atomic 15-file bundle" in items["AIA-SRE-017"]["next_action"] + assert "atomic 16-file bundle" in items["AIA-SRE-017"]["next_action"] assert "gitea-native" in " ".join(items["AIA-SRE-018"]["runtime_gaps"]) assert payload["completion_contract"]["evidence_layers"] == { "source_test": "partial_source_evidence_only", diff --git a/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json index f121002b7..ced6b0e19 100644 --- a/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json +++ b/docs/operations/portfolio-infrastructure-asset-reconciliation.snapshot.json @@ -122,7 +122,7 @@ "next_action": "Resolve exact database/container/port identity, collect read-only health and backup freshness, then create bounded DB verifier work." }, "members": [ - {"canonical_id": "database:awoooi:prod", "inventory_labels": ["awoooi_prod"], "source_rows": ["L025"], "runtime_identity": "host188/postgresql/awoooi_prod", "reconciliation_state": "identity_scope_conflict", "findings": ["Inventory combines AWOOOI business database and K3s datastore into one purpose", "Canonical service registry names the container postgres, not awoooi_prod"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-awoooi.sh"], "product_id": "awoooi", "project_id": "awoooi", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-awoooi.sh", "freshness": "unverified"}}, + {"canonical_id": "database:awoooi:prod", "inventory_labels": ["awoooi_prod"], "source_rows": ["L025"], "runtime_identity": "host188/postgresql/awoooi_prod", "reconciliation_state": "identity_scope_conflict", "findings": ["Inventory combines AWOOOI business database and K3s datastore into one purpose", "Canonical service registry names the container postgres, not awoooi_prod", "The fixed telegram_receipt_index_apply_v1 DB executor is source-ready but has no production same-run apply and independent verifier receipt"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-awoooi.sh", "apps/api/src/services/db_bounded_executor.py", "agent99-db-bounded-executor.ps1"], "product_id": "awoooi", "project_id": "awoooi", "owner_lane": "database_ops", "domain_router": "database", "executor": "db_bounded_executor", "verifier": "db_independent_verifier", "bounded_command": {"command_id": "telegram_receipt_index_apply_v1", "canonical_asset": "public.awooop_outbound_message", "execution_hub": "host:192.168.0.99", "transport": "host99_to_host110_to_host120_fixed_sudo_kubectl_exec", "cross_domain_fallback_allowed": false, "runtime_status": "source_ready_runtime_pending"}, "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-awoooi.sh", "freshness": "unverified"}}, {"canonical_id": "database:momo-pro:postgres", "inventory_labels": ["momo-db"], "source_rows": ["L026"], "runtime_identity": "host188/docker/momo-db", "reconciliation_state": "matched_source_runtime_unverified", "findings": ["Exact exposed port and database name are not proven by the inventory row"], "source_refs": ["ops/config/service-registry.yaml", "scripts/backup/backup-momo.sh"], "product_id": "momo-pro", "project_id": "momo-pro-system", "owner_lane": "momo_product_ops", "telegram_destination": "blocked:momo_configured_chat_unresolved", "backup_restore": {"mode": "critical_read_only_default", "target": "scripts/backup/backup-momo.sh", "freshness": "unverified"}}, {"canonical_id": "database:vibework:postgres", "inventory_labels": ["vibework-production-postgres-1"], "source_rows": ["L027"], "runtime_identity": "host188/docker/vibework-production-postgres-1", "reconciliation_state": "missing_from_canonical_service_registry", "findings": ["No exact AWOOOI service registry entry or committed restore contract found"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "vibework", "project_id": "vibework", "owner_lane": "vibework_product_ops", "telegram_destination": "blocked:vibework_route_not_implemented"}, {"canonical_id": "database:2026fifa:timescaledb", "inventory_labels": ["current-fifa2026-postgres-1", "TimescaleDB current-fifa2026-postgres-1"], "source_rows": ["L028", "L043"], "runtime_identity": "host188/docker/current-fifa2026-postgres-1", "reconciliation_state": "duplicate_inventory_row_and_registry_gap", "findings": ["Same instance is counted once as PostgreSQL and again as TimescaleDB", "No exact restore drill is registered"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "product_id": "2026fifa", "project_id": "2026FIFAWorldCup", "owner_lane": "2026fifa_product_ops", "telegram_destination": "blocked:2026fifa_destination_unassigned"}, @@ -544,7 +544,7 @@ "next_action": "Add the canonical asset to the reconciled source registry and require same-run production readback." }, "members": [ - {"canonical_id": "windows-vmware:host_99", "inventory_labels": ["Agent99 Windows/VMware control plane"], "source_rows": [], "runtime_identity": "192.168.0.99/Agent99", "findings": ["Agent99 is required for Windows/VMware and control-plane recovery but absent from the imported inventory", "Agent99 also owns the read-only host110 Alertmanager poll/reduced relay coordination role; Linux remediation still routes only to the host Ansible executor", "The atomic 15-file runtime bundle includes the poller but production deployment/readback is pending"], "source_refs": ["docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json", "k8s/awoooi-prod/04-configmap.yaml", "agent99-alertmanager-alertchain-poll.ps1", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "Agent99", "domain_router": "windows_vmware", "executor": "Agent99", "verifier": "agent99_independent_runtime_verifier"}, + {"canonical_id": "windows-vmware:host_99", "inventory_labels": ["Agent99 Windows/VMware control plane"], "source_rows": [], "runtime_identity": "192.168.0.99/Agent99", "findings": ["Agent99 is required for Windows/VMware and control-plane recovery but absent from the imported inventory", "Agent99 also owns the read-only host110 Alertmanager poll/reduced relay coordination role; Linux remediation still routes only to the host Ansible executor", "The atomic 16-file runtime bundle includes the poller and fixed database dispatch entrypoint but production deployment/readback is pending"], "source_refs": ["docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json", "k8s/awoooi-prod/04-configmap.yaml", "agent99-alertmanager-alertchain-poll.ps1", "agent99-db-bounded-executor.ps1", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "Agent99", "domain_router": "windows_vmware", "executor": "Agent99", "verifier": "agent99_independent_runtime_verifier"}, {"canonical_id": "service:ollama:host110", "inventory_labels": ["retired Host110 Ollama tombstone"], "source_rows": [], "runtime_identity": "retired-tombstone:host110/ollama/no-endpoint", "source_truth_state": "retired_tombstone_runtime_absence_verified", "reconciliation_state": "retired_tombstone_runtime_absence_verified", "findings": ["Host110 Ollama is forbidden as a provider, proxy, primary, secondary or failover target", "A tombstone is retained so stale sensors and routes create drift instead of resurrecting the endpoint", "AIA-SRE-002 run-aia-sre-002-20260716-1752 removed the exact stale Nginx 11435 proxy with a root-only rollback backup", "Independent post-verifier proved listeners 11435/11434=0, Ollama containers and systemd units=0, stale config paths=0, Nginx active/config valid and Prometheus host110 Ollama targets=0"], "source_refs": ["ops/config/service-registry.yaml", "ops/monitoring/service-registry.yaml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "scripts/ops/retire-host110-ollama-proxy.sh"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "bounded_executor_receipt": "run-aia-sre-002-20260716-1752", "runtime_closure": {"status": "verified_absent", "trace_id": "trace-aia-sre-002-20260716-1752", "run_id": "run-aia-sre-002-20260716-1752", "work_item_id": "AIA-SRE-002", "post_verifier": "host110_ollama_runtime_and_monitoring_absence_verifier"}, "verifier": "host110_ollama_runtime_and_monitoring_absence_verifier", "monitoring": {"signals": ["runtime_endpoint_absence", "prometheus_target_absence", "route_reference_absence"], "coverage": "runtime_absence_verified_20260716"}, "next_action": "Preserve the tombstone and create a drift work item on any Host110 Ollama process, proxy, listener, target, rule or provider-route recurrence."}, {"canonical_id": "host:111", "inventory_labels": ["host111 local Ollama"], "source_rows": [], "runtime_identity": "192.168.0.111/macos-launchd/ollama", "findings": ["Approved third provider hop and Ansible host are absent from the imported inventory", "The runtime manager is a macOS LaunchAgent, not systemd", "The exact LaunchAgent playbook requires local verification plus independent origin probes from host120 and host121", "The canonical host111 Ollama sensor/rule has no fresh production series"], "source_refs": ["infra/ansible/inventory/hosts.yml", "infra/ansible/playbooks/111-ollama-fallback.yml", "ops/monitoring/service-registry.yaml", "ops/monitoring/alerts-unified.yml", "apps/api/src/services/awooop_ansible_post_verifier.py"], "owner_lane": "ai_team", "domain_router": "host_systemd", "executor": "host_ansible_executor", "verifier": "host111_launchagent_local_and_120_121_origin_verifier", "monitoring": {"signals": ["launchagent_state", "local_generation", "host120_origin_generation", "host121_origin_generation", "canonical_alert_series"], "coverage": "source_candidate_runtime_sensor_missing"}, "next_action": "Check/apply the exact LaunchAgent catalog, verify local plus host120/host121 origins, and require a fresh canonical sensor series before provider readiness."}, {"canonical_id": "ai-provider:ollama_gcp_a", "inventory_labels": ["GCP-A Ollama"], "source_rows": [], "runtime_identity": "34.143.170.20:11434", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Approved first provider hop is absent from the imported inventory", "Observed transport is public HTTP and is candidate-only until secure mesh/TLS promotion", "Only sanitized prompts may cross this boundary; unsanitized input and every tool loop fail closed before network access", "The exact Prometheus provider target is missing in production"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/monitoring/prometheus.yml", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json", "apps/api/src/services/ai_provider_policy.py"], "owner_lane": "ai_team", "domain_router": "ai_provider", "executor": "deterministic_provider_router", "verifier": "provider_health_generation_and_transport_receipt_verifier", "monitoring": {"signals": ["exact_target_up", "scrape_freshness", "sanitization_receipt", "tool_loop_denial", "transport_boundary"], "coverage": "source_candidate_runtime_target_missing"}, "transport_policy": {"observed": "public_http", "execution_scope": "sanitized_candidate_only", "tool_loop": "fail_closed", "promotion_requires": "secure_mesh_or_tls_plus_runtime_readback"}, "next_action": "Deploy the exact blackbox target, verify fresh series and sanitization/tool-loop denials, then retain candidate-only status until secure transport promotion."}, @@ -556,7 +556,7 @@ {"canonical_id": "k8s:awoooi-prod:web", "inventory_labels": ["AWOOOI production Web"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-web", "findings": ["Production Web is in the monitoring registry but absent from the imported infrastructure inventory"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/awoooi-prod"], "owner_lane": "frontend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, {"canonical_id": "k8s:awoooi-prod:worker", "inventory_labels": ["AWOOOI production Worker"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-worker", "findings": ["Worker is in the monitoring registry but absent from the imported infrastructure inventory"], "source_refs": ["ops/monitoring/service-registry.yaml", "k8s/awoooi-prod"], "owner_lane": "backend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, {"canonical_id": "k8s:awoooi-prod:broker", "inventory_labels": ["AWOOOI production Broker"], "source_rows": [], "runtime_identity": "k3s/awoooi-prod/deployment/awoooi-broker", "findings": ["Broker is part of production deployment/readback but absent from the imported inventory and monitoring service registry"], "source_refs": ["k8s/awoooi-prod", "docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json"], "owner_lane": "backend_team", "domain_router": "kubernetes_workload", "executor": "kubernetes_controlled_executor", "verifier": "kubernetes_rollout_verifier"}, - {"canonical_id": "signal:alertmanager-webhook-chain", "inventory_labels": ["AlertChainBroken_Alertmanager"], "source_rows": [], "runtime_identity": "alertmanager->awoooi-api-webhook->incident->telegram plus host99 Agent99 exact-host pull fallback", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Canonical signal is locked to container:host_110:alertmanager", "Rule, concise attribution card, exact bounded recovery and independent verifier are source-ready", "Host99 Agent99 independently polls only the exact active/critical AlertChain signal from host110 without a first-hop credential or raw-payload persistence", "The reduced event is relayed over HTTPS with stable dedupe while Linux execution remains in the host Ansible lane", "Production deployment, poll freshness/dedupe, current alert resolution and same-run learning acknowledgements remain pending"], "source_refs": ["k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-relay.ps1", "apps/api/src/api/v1/webhooks.py", "apps/api/src/services/controlled_alert_target_router.py", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "observability_ops", "verifier": "alert_delivery_chain_plus_agent99_pull_independent_verifier", "monitoring": {"signals": ["alertmanager_delivery_counters", "primary_webhook_health", "agent99_pull_freshness", "dedupe_receipt", "telegram_delivery_receipt"], "coverage": "source_primary_and_independent_pull_ready_runtime_pending"}, "next_action": "Deploy the receiver-scoped scrape/rule/playbook and the atomic Agent99 15-file bundle; run check/apply/verify under one trace/run/work item; then require Alertmanager resolution, Telegram and KM/RAG/MCP/PlayBook acknowledgements before closure."}, + {"canonical_id": "signal:alertmanager-webhook-chain", "inventory_labels": ["AlertChainBroken_Alertmanager"], "source_rows": [], "runtime_identity": "alertmanager->awoooi-api-webhook->incident->telegram plus host99 Agent99 exact-host pull fallback", "source_truth_state": "source_reconciled_runtime_pending", "findings": ["Canonical signal is locked to container:host_110:alertmanager", "Rule, concise attribution card, exact bounded recovery and independent verifier are source-ready", "Host99 Agent99 independently polls only the exact active/critical AlertChain signal from host110 without a first-hop credential or raw-payload persistence", "The reduced event is relayed over HTTPS with stable dedupe while Linux execution remains in the host Ansible lane", "Production deployment, poll freshness/dedupe, current alert resolution and same-run learning acknowledgements remain pending"], "source_refs": ["k8s/monitoring/alert-chain-monitor.yaml", "ops/alertmanager/alertmanager.yml", "agent99-alertmanager-alertchain-poll.ps1", "agent99-sre-alert-relay.ps1", "apps/api/src/api/v1/webhooks.py", "apps/api/src/services/controlled_alert_target_router.py", "infra/ansible/playbooks/110-alertmanager-delivery-recovery.yml", "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"], "owner_lane": "observability_ops", "verifier": "alert_delivery_chain_plus_agent99_pull_independent_verifier", "monitoring": {"signals": ["alertmanager_delivery_counters", "primary_webhook_health", "agent99_pull_freshness", "dedupe_receipt", "telegram_delivery_receipt"], "coverage": "source_primary_and_independent_pull_ready_runtime_pending"}, "next_action": "Deploy the receiver-scoped scrape/rule/playbook and the atomic Agent99 16-file bundle; run check/apply/verify under one trace/run/work item; then require Alertmanager resolution, Telegram and KM/RAG/MCP/PlayBook acknowledgements before closure."}, {"canonical_id": "control-plane:telegram-routing-registry", "inventory_labels": ["Canonical Telegram routing registry"], "source_rows": [], "runtime_identity": "docs/security/telegram-canonical-routing-registry.snapshot.json", "findings": ["Nine product routes are blocked/disabled/not implemented; scattering alerts is forbidden"], "source_refs": ["docs/security/telegram-canonical-routing-registry.snapshot.json"], "owner_lane": "NotificationGovernance", "verifier": "telegram_route_and_delivery_receipt_verifier"}, {"canonical_id": "control-plane:km-rag", "inventory_labels": ["Knowledge Base and RAG"], "source_rows": [], "runtime_identity": "awoooi-api/knowledge-rag", "findings": ["Required learning target is implemented in source but absent from the imported inventory and runtime closure is unverified"], "source_refs": ["apps/api/src/api/v1/knowledge.py", "apps/api/src/api/v1/rag.py"], "owner_lane": "LearningPlane", "verifier": "km_rag_durable_write_ack_verifier"}, {"canonical_id": "control-plane:mcp-gateway", "inventory_labels": ["Internal MCP Gateway"], "source_rows": [], "runtime_identity": "awoooi-api/mcp-control-plane", "findings": ["MCP catalog has broad product coverage but is absent from the imported inventory; write adapters must remain policy controlled"], "source_refs": ["docs/operations/mcp-control-plane-catalog.snapshot.json"], "owner_lane": "MCPControlPlane", "verifier": "mcp_gateway_audit_and_runtime_verifier"}, diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index a6649b673..3ffb4ad30 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -513,14 +513,30 @@ "ops/config/service-registry.yaml", "apps/api/src/db/base.py", "apps/api/src/core/unit_of_work.py", + "apps/api/src/services/db_bounded_executor.py", "apps/api/src/services/executor_trust_boundary_readback.py", + "agent99-db-bounded-executor.ps1", "scripts/ops/awoooi-workload-db-identity-bootstrap.sh" ], "executor": "db_bounded_executor", "verifier": "db_independent_verifier", "rollback": "transactional or compensating action only; destructive restore remains break-glass", "exit_condition": "DB incidents use DB-native preflight/postconditions and never generic host restart", - "next_action": "deploy then verify cap-2 global session budget, role headroom >= 8 and DB-native no-cross-domain receipts" + "runtime_entrypoint_contract": { + "status": "source_ready_runtime_not_applied", + "command_id": "telegram_receipt_index_apply_v1", + "canonical_asset": "public.awooop_outbound_message", + "execution_hub": "host:192.168.0.99", + "dispatch_role": "Agent99_dispatch_only", + "executor_domain": "database", + "runtime_transport": "host99_to_host110_to_host120_fixed_sudo_kubectl_exec", + "host120_installer_required": false, + "check_apply_pod_role": "first_ready_awoooi_api_pod", + "independent_verifier_role": "second_ready_awoooi_api_pod_second_process", + "arbitrary_sql_allowed": false, + "cross_domain_fallback_allowed": false + }, + "next_action": "deploy the fixed Agent99 dispatch through the atomic runtime bundle, then use the already allowlisted host120 sudo kubectl path for one same-trace check/apply/second-pod verify of telegram_receipt_index_apply_v1; separately retain cap-2 global session budget and role headroom >= 8 verification" }, { "id": "AIA-SRE-011", @@ -713,7 +729,7 @@ "the host99 Agent99 independent pull/reduced relay has no production deployment, freshness, dedupe or controlled-repair receipt", "production Alertmanager exposes integration-only notification counters; the bounded check now passes after adding a scrape-time receiver_contract label, but apply, alert resolution and same-run learning receipts remain pending" ], - "next_action": "deploy the production-schema-compatible Alertmanager runtime scrape/canonical rules and the host99 Agent99 exact-host poller through the atomic 15-file bundle; verify receiver-contract freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination" + "next_action": "deploy the production-schema-compatible Alertmanager runtime scrape/canonical rules, the host99 Agent99 exact-host poller and the fixed DB dispatch entrypoint through the atomic 16-file bundle; verify receiver-contract freshness, dedupe and the current firing alert resolution; then reconcile every project/product/alert class against its typed domain and canonical Telegram destination" }, { "id": "AIA-SRE-018", diff --git a/infra/ansible/inventory/hosts.yml b/infra/ansible/inventory/hosts.yml index 0e8e65a78..e52a3c952 100644 --- a/infra/ansible/inventory/hosts.yml +++ b/infra/ansible/inventory/hosts.yml @@ -26,7 +26,7 @@ all: ansible_host: 192.168.0.111 ansible_user: ooo ansible_ssh_private_key_file: "~/.ssh/id_rsa" - ansible_ssh_common_args: "-o ProxyJump=192.168.0.110 -o StrictHostKeyChecking=accept-new" + ansible_ssh_common_args: "-o ProxyJump=wooo@192.168.0.110 -o StrictHostKeyChecking=yes" k3s_masters: hosts: diff --git a/infra/ansible/playbooks/111-ollama-fallback.yml b/infra/ansible/playbooks/111-ollama-fallback.yml index faa29c0dd..618d2d5af 100644 --- a/infra/ansible/playbooks/111-ollama-fallback.yml +++ b/infra/ansible/playbooks/111-ollama-fallback.yml @@ -20,96 +20,176 @@ changed_when: false tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 收斂 K3s direct provider allowlist" - ansible.builtin.command: - cmd: "/usr/libexec/PlistBuddy -c 'Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}' {{ proxy_plist }}" - when: current_allowlist.stdout != allowed_cidrs - changed_when: true - notify: Restart ollama111 allow proxy - tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 受控 apply 與 verifier，失敗即 rollback" + block: + - name: "Ollama111 | 收斂 K3s direct provider allowlist" + ansible.builtin.command: + argv: + - /usr/libexec/PlistBuddy + - -c + - "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ allowed_cidrs }}" + - "{{ proxy_plist }}" + when: + - not ansible_check_mode + - current_allowlist.stdout != allowed_cidrs + changed_when: true + notify: Restart ollama111 allow proxy + tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 套用 allowlist handler 後再驗證" - ansible.builtin.meta: flush_handlers + - name: "Ollama111 | 套用 allowlist handler 後再驗證" + ansible.builtin.meta: flush_handlers + when: not ansible_check_mode - - name: "Ollama111 | 讀回生效 allowlist" - ansible.builtin.command: - cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}" - register: effective_allowlist - check_mode: false - changed_when: false - tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 讀回生效 allowlist" + ansible.builtin.command: + cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}" + register: effective_allowlist + check_mode: false + changed_when: false + tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 驗證 K3s origins 並排除 retired host110" - ansible.builtin.assert: - that: - - effective_allowlist.stdout == allowed_cidrs - - "'192.168.0.120/32' in effective_allowlist.stdout.split(',')" - - "'192.168.0.121/32' in effective_allowlist.stdout.split(',')" - fail_msg: ollama111_allowlist_runtime_identity_mismatch - changed_when: false - when: not ansible_check_mode - tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 驗證 K3s origins 並排除 retired host110" + ansible.builtin.assert: + that: + - effective_allowlist.stdout == allowed_cidrs + - "'192.168.0.120/32' in effective_allowlist.stdout.split(',')" + - "'192.168.0.121/32' in effective_allowlist.stdout.split(',')" + fail_msg: ollama111_allowlist_runtime_identity_mismatch + changed_when: false + when: not ansible_check_mode + tags: ["111", "ollama-fallback"] - - name: "Ollama111 | check-mode 發布精確 source diff" - ansible.builtin.debug: - msg: - schema_version: ollama111_allowlist_check_v1 - current_allowlist_sha_only: true - would_change: "{{ current_allowlist.stdout != allowed_cidrs }}" - expected_k3s_origins: - - host120 - - host121 - retired_host110_allowed_after_apply: false - persistent_writes_performed: 0 - when: ansible_check_mode - changed_when: current_allowlist.stdout != allowed_cidrs - tags: ["111", "ollama-fallback"] + - name: "Ollama111 | check-mode 發布精確 source diff" + ansible.builtin.debug: + msg: + schema_version: ollama111_allowlist_check_v1 + current_allowlist_sha_only: true + would_change: "{{ current_allowlist.stdout != allowed_cidrs }}" + expected_k3s_origins: + - host120 + - host121 + retired_host110_allowed_after_apply: false + persistent_writes_performed: 0 + when: ansible_check_mode + changed_when: current_allowlist.stdout != allowed_cidrs + tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 驗證本機 Ollama tag API" - ansible.builtin.uri: - url: http://127.0.0.1:11434/api/tags - method: GET - return_content: false - status_code: 200 - check_mode: false - changed_when: false - tags: ["111", "ollama-fallback"] + - name: "Ollama111 | 驗證本機 Ollama tag API" + ansible.builtin.uri: + url: http://127.0.0.1:11434/api/tags + method: GET + return_content: false + status_code: 200 + check_mode: false + changed_when: false + tags: ["111", "ollama-fallback"] - - name: "Ollama111 | 從每個 K3s node 驗證 direct provider path" - ansible.builtin.uri: - url: http://192.168.0.111:11434/api/tags - method: GET - return_content: false - status_code: 200 - timeout: 10 - delegate_to: "{{ item }}" - loop: - - host_120 - - host_121 - register: k3s_direct_provider_readback - check_mode: false - changed_when: false - when: >- - not ansible_check_mode - or current_allowlist.stdout == allowed_cidrs - tags: ["111", "ollama-fallback", "independent-verifier"] + - name: "Ollama111 | 從每個 K3s node 驗證 direct provider path" + ansible.builtin.uri: + url: http://192.168.0.111:11434/api/tags + method: GET + return_content: false + status_code: 200 + timeout: 10 + delegate_to: "{{ item }}" + loop: + - host_120 + - host_121 + register: k3s_direct_provider_readback + check_mode: false + changed_when: false + when: >- + not ansible_check_mode + or current_allowlist.stdout == allowed_cidrs + tags: ["111", "ollama-fallback", "independent-verifier"] - - name: "Ollama111 | 發布 K3s path verifier receipt" - ansible.builtin.debug: - msg: - schema_version: ollama111_k3s_path_verifier_v1 - canonical_asset_id: ai-provider:ollama_local - probe_origins: - - host120 - - host121 - http_statuses: >- - {{ k3s_direct_provider_readback.results - | map(attribute='status') | list }} - retired_host110_allowed: false - runtime_verified: true - changed_when: false - when: not ansible_check_mode - tags: ["111", "ollama-fallback", "independent-verifier"] + - name: "Ollama111 | 發布 K3s path verifier receipt" + ansible.builtin.debug: + msg: + schema_version: ollama111_k3s_path_verifier_v1 + canonical_asset_id: ai-provider:ollama_local + probe_origins: + - host120 + - host121 + http_statuses: >- + {{ k3s_direct_provider_readback.results + | map(attribute='status') | list }} + retired_host110_allowed: false + runtime_verified: true + changed_when: false + when: not ansible_check_mode + tags: ["111", "ollama-fallback", "independent-verifier"] + + rescue: + - name: "Ollama111 | check-mode failure 維持 no-write" + ansible.builtin.debug: + msg: + schema_version: ollama111_check_failure_no_write_v1 + failed_task: "{{ ansible_failed_task.name | default('unknown') }}" + rollback_performed: false + persistent_writes_performed: 0 + when: ansible_check_mode + changed_when: false + + - name: "Ollama111 | rollback 恢復原 allowlist" + ansible.builtin.command: + argv: + - /usr/libexec/PlistBuddy + - -c + - "Set :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS {{ current_allowlist.stdout }}" + - "{{ proxy_plist }}" + when: not ansible_check_mode + changed_when: true + tags: ["111", "ollama-fallback", "rollback"] + + - name: "Ollama111 | rollback 重新啟動 allow proxy" + ansible.builtin.shell: + cmd: | + set -e + launchctl enable gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true + launchctl bootout gui/{{ proxy_user_uid }}/{{ proxy_label }} 2>/dev/null || true + launchctl bootstrap gui/{{ proxy_user_uid }} {{ proxy_plist }} + when: not ansible_check_mode + changed_when: true + tags: ["111", "ollama-fallback", "rollback"] + + - name: "Ollama111 | rollback 讀回原 allowlist" + ansible.builtin.command: + cmd: "/usr/libexec/PlistBuddy -c 'Print :EnvironmentVariables:OLLAMA111_PROXY_ALLOWED_CIDRS' {{ proxy_plist }}" + register: rollback_allowlist + when: not ansible_check_mode + check_mode: false + changed_when: false + tags: ["111", "ollama-fallback", "rollback"] + + - name: "Ollama111 | rollback 驗證原 allowlist 已恢復" + ansible.builtin.assert: + that: + - rollback_allowlist.stdout == current_allowlist.stdout + fail_msg: ollama111_allowlist_rollback_readback_mismatch + when: not ansible_check_mode + changed_when: false + tags: ["111", "ollama-fallback", "rollback"] + + - name: "Ollama111 | 發布 rollback receipt" + ansible.builtin.debug: + msg: + schema_version: ollama111_allowlist_rollback_v1 + canonical_asset_id: ai-provider:ollama_local + failed_task: "{{ ansible_failed_task.name | default('unknown') }}" + original_allowlist_readback_verified: true + proxy_restarted: true + rollback_performed: true + when: not ansible_check_mode + changed_when: false + tags: ["111", "ollama-fallback", "rollback"] + + - name: "Ollama111 | apply 或 verifier 失敗後保持 failed terminal" + ansible.builtin.fail: + msg: >- + ollama111_controlled_lane_failed + task={{ ansible_failed_task.name | default('unknown') }} + rollback_performed={{ not ansible_check_mode }} handlers: - name: Restart ollama111 allow proxy diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 19d8b112d..7dde3fabd 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -372,6 +372,18 @@ def test_post_deploy_smoke_uses_workspace_playwright_dependency() -> None: assert "npx playwright" not in block +def test_post_deploy_pressure_wait_covers_bounded_backup_and_metric_freshness() -> None: + text = _workflow_text() + block = text.split("post-deploy-checks:", 1)[1] + block = block.split("- name: Get Commit Info", 1)[0] + + assert "timeout-minutes: 45" in block + assert '- name: Wait for Host Web Build Pressure' in block + assert 'HOST_WEB_BUILD_PRESSURE_ATTEMPTS: "120"' in block + assert 'HOST_WEB_BUILD_PRESSURE_WARN_ONLY: "1"' not in block + assert "run: bash scripts/ci/wait-host-web-build-pressure.sh" in block + + def test_harbor_login_has_public_route_retry_and_safe_secret_transport() -> None: text = _workflow_text() block = text.split("- name: Login to Harbor", 1)[1] diff --git a/scripts/ops/run-paid-provider-canary.py b/scripts/ops/run-paid-provider-canary.py index dbd55f3e1..2ea180a0b 100644 --- a/scripts/ops/run-paid-provider-canary.py +++ b/scripts/ops/run-paid-provider-canary.py @@ -16,7 +16,19 @@ if str(_API_ROOT) not in sys.path: sys.path.insert(0, str(_API_ROOT)) -async def _run_validation(**kwargs): +async def _init_runtime() -> None: + from src.core.redis_client import init_redis_pool + + await init_redis_pool() + + +async def _close_runtime() -> None: + from src.core.redis_client import close_redis_pool + + await close_redis_pool() + + +async def _execute_validation(**kwargs): from src.services.paid_provider_canary_validation import ( run_paid_provider_canary_validation, ) @@ -24,6 +36,16 @@ async def _run_validation(**kwargs): return await run_paid_provider_canary_validation(**kwargs) +async def _run_validation(**kwargs): + """Run with the same Redis lifecycle the API process normally provides.""" + + await _init_runtime() + try: + return await _execute_validation(**kwargs) + finally: + await _close_runtime() + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--run-ref", required=True) diff --git a/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py new file mode 100644 index 000000000..eaf160373 --- /dev/null +++ b/scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py @@ -0,0 +1,118 @@ +from __future__ import annotations + +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +AGENT99_ENTRYPOINT = ROOT / "agent99-db-bounded-executor.ps1" + + +def test_agent99_reads_and_validates_two_ready_api_pods_locally() -> None: + source = AGENT99_ENTRYPOINT.read_text() + + assert '$Kubectl = "/usr/local/bin/kubectl"' in source + assert '$Namespace = "awoooi-prod"' in source + assert '$PodSelector = "app=awoooi-api,environment=prod,system=awoooi"' in source + assert '$Container = "api"' in source + assert "function Invoke-Agent99ReadyApiPodReadback" in source + assert '"$ControlPlaneHost sudo -n $Kubectl get pods "' in source + assert "\"--field-selector 'status.phase=Running' \"" in source + assert '.status.conditions[?(@.type==`"Ready`")].status' in source + assert "$columns.Count -ne 4" in source + assert '$columns[2] -eq "true"' in source + assert '$columns[3] -eq ""' in source + assert "$pods.Count -lt 2" in source + assert "$ExecutorPod = [string]$podReadback.pods[0]" in source + assert "$VerifierPod = [string]$podReadback.pods[1]" in source + assert "$ExecutorPod -eq $VerifierPod" in source + + +def test_agent99_is_dispatch_only_and_apply_requires_independent_verifier() -> None: + source = AGENT99_ENTRYPOINT.read_text() + + for fixed in ( + '$CommandId = "telegram_receipt_index_apply_v1"', + '$CanonicalAsset = "public.awooop_outbound_message"', + '$JumpHost = "192.168.0.110"', + '$ControlPlaneHost = "192.168.0.120"', + '$Kubectl = "/usr/local/bin/kubectl"', + 'domain = "database"', + 'executor = "db_bounded_executor"', + "dispatchOnly = $true", + "windowsVmwareMutationPerformed = $false", + "arbitraryCommandAllowed = $false", + "crossDomainFallbackAllowed = $false", + ): + assert fixed in source + + assert '"$ControlPlaneHost sudo -n $Kubectl exec "' in source + assert '"--namespace $Namespace --container $Container $PodName -- "' in source + assert '"python -B -m src.services.db_bounded_executor "' in source + assert 'Invoke-Agent99FixedDbStage "check" $ExecutorPod' in source + assert 'Invoke-Agent99FixedDbStage "apply" $ExecutorPod' in source + assert 'Invoke-Agent99FixedDbStage "verify" $VerifierPod' in source + apply_flow = source[source.index("# Apply is the only write path") :] + assert apply_flow.index('Invoke-Agent99FixedDbStage "check"') < apply_flow.index( + 'Invoke-Agent99FixedDbStage "apply"' + ) + assert apply_flow.index('Invoke-Agent99FixedDbStage "apply"') < apply_flow.index( + 'Invoke-Agent99FixedDbStage "verify"' + ) + assert ( + "[string]$apply.receipt.pod_name -ne [string]$verify.receipt.pod_name" + in apply_flow + ) + assert "Invoke-Expression" not in source + assert "-EncodedCommand" not in source + assert "/usr/local/sbin/awoooi-db-bounded-executor" not in source + assert "--sql" not in source + assert "rawTransportOutputPersisted" in source + assert "$script:TransportCleanupSucceeded = $false" in source + stage = source[source.index("function Invoke-Agent99FixedDbStage") :] + assert stage.index("ConvertFrom-Json") < stage.index("if ($process.ExitCode -ne 0)") + assert "$applyWriteState" in source + assert "$null -eq $apply.receipt.writes_performed" in source + assert "-not $apply.ok -and -not $apply.receipt" in source + assert "$terminalAllowed" in source + assert "$writeContract" in source + assert "$catalogContract" in source + assert ( + "sql" + not in " ".join( + line for line in source.splitlines() if line.lstrip().startswith("param(") + ).lower() + ) + + +def test_static_k3s_contract_supports_second_pod_verifier() -> None: + deployment = (ROOT / "k8s/awoooi-prod/06-deployment-api.yaml").read_text() + hpa = (ROOT / "k8s/awoooi-prod/12-hpa.yaml").read_text() + dockerfile = (ROOT / "apps/api/Dockerfile").read_text() + + assert "replicas: 2" in deployment + assert "app: awoooi-api" in deployment + assert "environment: prod" in deployment + assert "system: awoooi" in deployment + assert "- name: api" in deployment + assert "minReplicas: 2" in hpa + assert "COPY --chown=appuser:appuser apps/api/src/ ./src/" in dockerfile + assert "ENV AWOOOI_BUILD_COMMIT_SHA=$CACHE_BUST" in dockerfile + + +def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None: + deploy = (ROOT / "agent99-deploy.ps1").read_text() + bootstrap = (ROOT / "agent99-bootstrap.ps1").read_text() + contract = (ROOT / "agent99-contract-check.ps1").read_text() + transport = ( + ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" + ).read_text() + receiver = ( + ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1" + ).read_text() + + for source in (deploy, bootstrap, contract, transport, receiver): + assert '"agent99-db-bounded-executor.ps1"' in source + assert 'Add-Check "database:fixed_bounded_executor"' in contract + assert 'expectedRuntimeFileCount": 16' in transport + assert "expectedRuntimeFileCount=16" in transport + assert "expectedRuntimeFileCount -ne 16" in receiver + assert "fileCount = 16" in receiver diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index b9790a3ff..b8d172fc0 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -22,6 +22,7 @@ $FixedRuntimeFiles = @( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-db-bounded-executor.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -717,7 +718,7 @@ try { } $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant() if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" } - if ([int]$envelope.expectedRuntimeFileCount -ne 15) { throw "invalid_expected_runtime_file_count" } + if ([int]$envelope.expectedRuntimeFileCount -ne 16) { throw "invalid_expected_runtime_file_count" } $traceId = [string]$envelope.traceId $runId = [string]$envelope.runId @@ -752,7 +753,7 @@ try { $sourceManifest = [pscustomobject]@{ schemaVersion = "agent99_remote_source_manifest_v1" sourceRevision = $sourceRevision - fileCount = 15 + fileCount = 16 manifestSha256 = $manifestDigest files = @($FixedRuntimeFiles | ForEach-Object { [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() } @@ -789,7 +790,7 @@ try { status = "check_ready" mode = "check" sourceRevision = $sourceRevision - expectedRuntimeFileCount = 15 + expectedRuntimeFileCount = 16 manifestSha256 = $manifestDigest plannedStagingPath = $stagePath runtimeManifest = $currentManifest @@ -876,7 +877,7 @@ try { if ( [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or - [int]$existingSourceManifest.fileCount -ne 15 -or + [int]$existingSourceManifest.fileCount -ne 16 -or [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest ) { throw "existing_staging_manifest_mismatch" } } catch { @@ -900,7 +901,7 @@ try { [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and $live.runtimeMatched -and $live.sourceRevision -eq $sourceRevision -and - $live.fileCount -eq 15 -and + $live.fileCount -eq 16 -and $live.mismatchCount -eq 0 -and $liveLauncherContract.ok -and [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256 @@ -1136,7 +1137,7 @@ try { $runtimeManifest.exists -and $runtimeManifest.runtimeMatched -and $runtimeManifest.sourceRevision -eq $sourceRevision -and - $runtimeManifest.fileCount -eq 15 -and + $runtimeManifest.fileCount -eq 16 -and $runtimeManifest.mismatchCount -eq 0 -and $launcherContract.ok -and [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 278c1560d..a217f4b6d 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -17,6 +17,7 @@ RUNTIME_FILES=( "agent99-bootstrap.ps1" "agent99-contract-check.ps1" "agent99-control-plane.ps1" + "agent99-db-bounded-executor.ps1" "agent99-deploy.ps1" "agent99-register-tasks.ps1" "agent99-alertmanager-alertchain-poll.ps1" @@ -235,7 +236,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7] output_path = Path(sys.argv[7]) runtime_names = sys.argv[8:] -if len(runtime_names) != 15 or len(set(runtime_names)) != 15: +if len(runtime_names) != 16 or len(set(runtime_names)) != 16: raise SystemExit("fixed_runtime_file_contract_failed") files: list[dict[str, str]] = [] @@ -274,7 +275,7 @@ envelope = { "runId": run_id, "workItemId": work_item_id, "sourceRevision": source_revision, - "expectedRuntimeFileCount": 15, + "expectedRuntimeFileCount": 16, "manifestSha256": manifest_sha256, "files": files, "livePreflight": { @@ -426,14 +427,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20] script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue' $a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json' $r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''} -if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 15-and$m.mismatchCount-eq 0-and$rows.Count-eq 15-and$seen.Count-eq 15-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} +if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 16-and$m.mismatchCount-eq 0-and$rows.Count-eq 16-and$seen.Count-eq 16-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}} $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} $ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) $status=if($ready){'check_ready'}else{'check_failed_no_apply'} $next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=15;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=16;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 7c9e23707..3eee4a24e 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -23,6 +23,7 @@ EXPECTED_RUNTIME_FILES = ( "agent99-bootstrap.ps1", "agent99-contract-check.ps1", "agent99-control-plane.ps1", + "agent99-db-bounded-executor.ps1", "agent99-deploy.ps1", "agent99-register-tasks.ps1", "agent99-alertmanager-alertchain-poll.ps1", @@ -78,7 +79,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None assert sender_files == EXPECTED_RUNTIME_FILES assert receiver_files == EXPECTED_RUNTIME_FILES assert deployer_files == EXPECTED_RUNTIME_FILES - assert "expectedRuntimeFileCount\": 15" in sender + assert "expectedRuntimeFileCount\": 16" in sender assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver assert "required_runtime_file_missing" in receiver assert "duplicate_runtime_file" in receiver @@ -231,7 +232,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier() assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source assert "$livePreflight.sourceRevision -eq $sourceRevision" in source assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source - assert '$runtimeManifest.fileCount -eq 15' in source + assert '$runtimeManifest.fileCount -eq 16' in source assert '$runtimeManifest.mismatchCount -eq 0' in source assert 'status = "deployed_verified"' in source assert '$failurePhase = "deploy"' in source