fix(sre): close bounded runtime contract gaps

scripts/ops/run-paid-provider-canary.py

@@ -16,7 +16,19 @@ if str(_API_ROOT) not in sys.path:
     sys.path.insert(0, str(_API_ROOT))
 
 
-async def _run_validation(**kwargs):
+async def _init_runtime() -> None:
+    from src.core.redis_client import init_redis_pool
+
+    await init_redis_pool()
+
+
+async def _close_runtime() -> None:
+    from src.core.redis_client import close_redis_pool
+
+    await close_redis_pool()
+
+
+async def _execute_validation(**kwargs):
     from src.services.paid_provider_canary_validation import (
         run_paid_provider_canary_validation,
     )
@@ -24,6 +36,16 @@ async def _run_validation(**kwargs):
     return await run_paid_provider_canary_validation(**kwargs)
 
 
+async def _run_validation(**kwargs):
+    """Run with the same Redis lifecycle the API process normally provides."""
+
+    await _init_runtime()
+    try:
+        return await _execute_validation(**kwargs)
+    finally:
+        await _close_runtime()
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--run-ref", required=True)

scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+AGENT99_ENTRYPOINT = ROOT / "agent99-db-bounded-executor.ps1"
+
+
+def test_agent99_reads_and_validates_two_ready_api_pods_locally() -> None:
+    source = AGENT99_ENTRYPOINT.read_text()
+
+    assert '$Kubectl = "/usr/local/bin/kubectl"' in source
+    assert '$Namespace = "awoooi-prod"' in source
+    assert '$PodSelector = "app=awoooi-api,environment=prod,system=awoooi"' in source
+    assert '$Container = "api"' in source
+    assert "function Invoke-Agent99ReadyApiPodReadback" in source
+    assert '"$ControlPlaneHost sudo -n $Kubectl get pods "' in source
+    assert "\"--field-selector 'status.phase=Running' \"" in source
+    assert '.status.conditions[?(@.type==`"Ready`")].status' in source
+    assert "$columns.Count -ne 4" in source
+    assert '$columns[2] -eq "true"' in source
+    assert '$columns[3] -eq "<none>"' in source
+    assert "$pods.Count -lt 2" in source
+    assert "$ExecutorPod = [string]$podReadback.pods[0]" in source
+    assert "$VerifierPod = [string]$podReadback.pods[1]" in source
+    assert "$ExecutorPod -eq $VerifierPod" in source
+
+
+def test_agent99_is_dispatch_only_and_apply_requires_independent_verifier() -> None:
+    source = AGENT99_ENTRYPOINT.read_text()
+
+    for fixed in (
+        '$CommandId = "telegram_receipt_index_apply_v1"',
+        '$CanonicalAsset = "public.awooop_outbound_message"',
+        '$JumpHost = "192.168.0.110"',
+        '$ControlPlaneHost = "192.168.0.120"',
+        '$Kubectl = "/usr/local/bin/kubectl"',
+        'domain = "database"',
+        'executor = "db_bounded_executor"',
+        "dispatchOnly = $true",
+        "windowsVmwareMutationPerformed = $false",
+        "arbitraryCommandAllowed = $false",
+        "crossDomainFallbackAllowed = $false",
+    ):
+        assert fixed in source
+
+    assert '"$ControlPlaneHost sudo -n $Kubectl exec "' in source
+    assert '"--namespace $Namespace --container $Container $PodName -- "' in source
+    assert '"python -B -m src.services.db_bounded_executor "' in source
+    assert 'Invoke-Agent99FixedDbStage "check" $ExecutorPod' in source
+    assert 'Invoke-Agent99FixedDbStage "apply" $ExecutorPod' in source
+    assert 'Invoke-Agent99FixedDbStage "verify" $VerifierPod' in source
+    apply_flow = source[source.index("# Apply is the only write path") :]
+    assert apply_flow.index('Invoke-Agent99FixedDbStage "check"') < apply_flow.index(
+        'Invoke-Agent99FixedDbStage "apply"'
+    )
+    assert apply_flow.index('Invoke-Agent99FixedDbStage "apply"') < apply_flow.index(
+        'Invoke-Agent99FixedDbStage "verify"'
+    )
+    assert (
+        "[string]$apply.receipt.pod_name -ne [string]$verify.receipt.pod_name"
+        in apply_flow
+    )
+    assert "Invoke-Expression" not in source
+    assert "-EncodedCommand" not in source
+    assert "/usr/local/sbin/awoooi-db-bounded-executor" not in source
+    assert "--sql" not in source
+    assert "rawTransportOutputPersisted" in source
+    assert "$script:TransportCleanupSucceeded = $false" in source
+    stage = source[source.index("function Invoke-Agent99FixedDbStage") :]
+    assert stage.index("ConvertFrom-Json") < stage.index("if ($process.ExitCode -ne 0)")
+    assert "$applyWriteState" in source
+    assert "$null -eq $apply.receipt.writes_performed" in source
+    assert "-not $apply.ok -and -not $apply.receipt" in source
+    assert "$terminalAllowed" in source
+    assert "$writeContract" in source
+    assert "$catalogContract" in source
+    assert (
+        "sql"
+        not in " ".join(
+            line for line in source.splitlines() if line.lstrip().startswith("param(")
+        ).lower()
+    )
+
+
+def test_static_k3s_contract_supports_second_pod_verifier() -> None:
+    deployment = (ROOT / "k8s/awoooi-prod/06-deployment-api.yaml").read_text()
+    hpa = (ROOT / "k8s/awoooi-prod/12-hpa.yaml").read_text()
+    dockerfile = (ROOT / "apps/api/Dockerfile").read_text()
+
+    assert "replicas: 2" in deployment
+    assert "app: awoooi-api" in deployment
+    assert "environment: prod" in deployment
+    assert "system: awoooi" in deployment
+    assert "- name: api" in deployment
+    assert "minReplicas: 2" in hpa
+    assert "COPY --chown=appuser:appuser apps/api/src/ ./src/" in dockerfile
+    assert "ENV AWOOOI_BUILD_COMMIT_SHA=$CACHE_BUST" in dockerfile
+
+
+def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
+    deploy = (ROOT / "agent99-deploy.ps1").read_text()
+    bootstrap = (ROOT / "agent99-bootstrap.ps1").read_text()
+    contract = (ROOT / "agent99-contract-check.ps1").read_text()
+    transport = (
+        ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
+    ).read_text()
+    receiver = (
+        ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
+    ).read_text()
+
+    for source in (deploy, bootstrap, contract, transport, receiver):
+        assert '"agent99-db-bounded-executor.ps1"' in source
+    assert 'Add-Check "database:fixed_bounded_executor"' in contract
+    assert 'expectedRuntimeFileCount": 16' in transport
+    assert "expectedRuntimeFileCount=16" in transport
+    assert "expectedRuntimeFileCount -ne 16" in receiver
+    assert "fileCount = 16" in receiver

scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1

@@ -22,6 +22,7 @@ $FixedRuntimeFiles = @(
   "agent99-bootstrap.ps1",
   "agent99-contract-check.ps1",
   "agent99-control-plane.ps1",
+  "agent99-db-bounded-executor.ps1",
   "agent99-deploy.ps1",
   "agent99-register-tasks.ps1",
   "agent99-alertmanager-alertchain-poll.ps1",
@@ -717,7 +718,7 @@ try {
   }
   $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
   if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
-  if ([int]$envelope.expectedRuntimeFileCount -ne 15) { throw "invalid_expected_runtime_file_count" }
+  if ([int]$envelope.expectedRuntimeFileCount -ne 16) { throw "invalid_expected_runtime_file_count" }
 
   $traceId = [string]$envelope.traceId
   $runId = [string]$envelope.runId
@@ -752,7 +753,7 @@ try {
   $sourceManifest = [pscustomobject]@{
     schemaVersion = "agent99_remote_source_manifest_v1"
     sourceRevision = $sourceRevision
-    fileCount = 15
+    fileCount = 16
     manifestSha256 = $manifestDigest
     files = @($FixedRuntimeFiles | ForEach-Object {
       [pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
@@ -789,7 +790,7 @@ try {
       status = "check_ready"
       mode = "check"
       sourceRevision = $sourceRevision
-      expectedRuntimeFileCount = 15
+      expectedRuntimeFileCount = 16
       manifestSha256 = $manifestDigest
       plannedStagingPath = $stagePath
       runtimeManifest = $currentManifest
@@ -876,7 +877,7 @@ try {
       if (
         [string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
         [string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
-        [int]$existingSourceManifest.fileCount -ne 15 -or
+        [int]$existingSourceManifest.fileCount -ne 16 -or
         [string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
       ) { throw "existing_staging_manifest_mismatch" }
     } catch {
@@ -900,7 +901,7 @@ try {
     [string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
     $live.runtimeMatched -and
     $live.sourceRevision -eq $sourceRevision -and
-    $live.fileCount -eq 15 -and
+    $live.fileCount -eq 16 -and
     $live.mismatchCount -eq 0 -and
     $liveLauncherContract.ok -and
     [string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
@@ -1136,7 +1137,7 @@ try {
           $runtimeManifest.exists -and
           $runtimeManifest.runtimeMatched -and
           $runtimeManifest.sourceRevision -eq $sourceRevision -and
-          $runtimeManifest.fileCount -eq 15 -and
+          $runtimeManifest.fileCount -eq 16 -and
           $runtimeManifest.mismatchCount -eq 0 -and
           $launcherContract.ok -and
           [string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and

scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh

@@ -17,6 +17,7 @@ RUNTIME_FILES=(
   "agent99-bootstrap.ps1"
   "agent99-contract-check.ps1"
   "agent99-control-plane.ps1"
+  "agent99-db-bounded-executor.ps1"
   "agent99-deploy.ps1"
   "agent99-register-tasks.ps1"
   "agent99-alertmanager-alertchain-poll.ps1"
@@ -235,7 +236,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
 output_path = Path(sys.argv[7])
 runtime_names = sys.argv[8:]
 
-if len(runtime_names) != 15 or len(set(runtime_names)) != 15:
+if len(runtime_names) != 16 or len(set(runtime_names)) != 16:
     raise SystemExit("fixed_runtime_file_contract_failed")
 
 files: list[dict[str, str]] = []
@@ -274,7 +275,7 @@ envelope = {
     "runId": run_id,
     "workItemId": work_item_id,
     "sourceRevision": source_revision,
-    "expectedRuntimeFileCount": 15,
+    "expectedRuntimeFileCount": 16,
     "manifestSha256": manifest_sha256,
     "files": files,
     "livePreflight": {
@@ -426,14 +427,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
 script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
 $a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
 $r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
-if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 15-and$m.mismatchCount-eq 0-and$rows.Count-eq 15-and$seen.Count-eq 15-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
+if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 16-and$m.mismatchCount-eq 0-and$rows.Count-eq 16-and$seen.Count-eq 16-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
 $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
 try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
 $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
 $ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
 $status=if($ready){'check_ready'}else{'check_failed_no_apply'}
 $next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
-[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=15;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
+[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=16;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
 if(-not$ready){exit 65}'''
 script = (
     script.replace("__SOURCE_REVISION__", source_revision)

scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py

@@ -23,6 +23,7 @@ EXPECTED_RUNTIME_FILES = (
     "agent99-bootstrap.ps1",
     "agent99-contract-check.ps1",
     "agent99-control-plane.ps1",
+    "agent99-db-bounded-executor.ps1",
     "agent99-deploy.ps1",
     "agent99-register-tasks.ps1",
     "agent99-alertmanager-alertchain-poll.ps1",
@@ -78,7 +79,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
     assert sender_files == EXPECTED_RUNTIME_FILES
     assert receiver_files == EXPECTED_RUNTIME_FILES
     assert deployer_files == EXPECTED_RUNTIME_FILES
-    assert "expectedRuntimeFileCount\": 15" in sender
+    assert "expectedRuntimeFileCount\": 16" in sender
     assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
     assert "required_runtime_file_missing" in receiver
     assert "duplicate_runtime_file" in receiver
@@ -231,7 +232,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
     assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
     assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
     assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
-    assert '$runtimeManifest.fileCount -eq 15' in source
+    assert '$runtimeManifest.fileCount -eq 16' in source
     assert '$runtimeManifest.mismatchCount -eq 0' in source
     assert 'status = "deployed_verified"' in source
     assert '$failurePhase = "deploy"' in source