fix(sre): close bounded runtime contract gaps

scripts/ops/run-paid-provider-canary.py

@@ -16,7 +16,19 @@ if str(_API_ROOT) not in sys.path:
     sys.path.insert(0, str(_API_ROOT))
 
 
-async def _run_validation(**kwargs):
+async def _init_runtime() -> None:
+    from src.core.redis_client import init_redis_pool
+
+    await init_redis_pool()
+
+
+async def _close_runtime() -> None:
+    from src.core.redis_client import close_redis_pool
+
+    await close_redis_pool()
+
+
+async def _execute_validation(**kwargs):
     from src.services.paid_provider_canary_validation import (
         run_paid_provider_canary_validation,
     )
@@ -24,6 +36,16 @@ async def _run_validation(**kwargs):
     return await run_paid_provider_canary_validation(**kwargs)
 
 
+async def _run_validation(**kwargs):
+    """Run with the same Redis lifecycle the API process normally provides."""
+
+    await _init_runtime()
+    try:
+        return await _execute_validation(**kwargs)
+    finally:
+        await _close_runtime()
+
+
 def main() -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument("--run-ref", required=True)

scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py

@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[3]
+AGENT99_ENTRYPOINT = ROOT / "agent99-db-bounded-executor.ps1"
+
+
+def test_agent99_reads_and_validates_two_ready_api_pods_locally() -> None:
+    source = AGENT99_ENTRYPOINT.read_text()
+
+    assert '$Kubectl = "/usr/local/bin/kubectl"' in source
+    assert '$Namespace = "awoooi-prod"' in source
+    assert '$PodSelector = "app=awoooi-api,environment=prod,system=awoooi"' in source
+    assert '$Container = "api"' in source
+    assert "function Invoke-Agent99ReadyApiPodReadback" in source
+    assert '"$ControlPlaneHost sudo -n $Kubectl get pods "' in source
+    assert "\"--field-selector 'status.phase=Running' \"" in source
+    assert '.status.conditions[?(@.type==`"Ready`")].status' in source
+    assert "$columns.Count -ne 4" in source
+    assert '$columns[2] -eq "true"' in source
+    assert '$columns[3] -eq "<none>"' in source
+    assert "$pods.Count -lt 2" in source
+    assert "$ExecutorPod = [string]$podReadback.pods[0]" in source
+    assert "$VerifierPod = [string]$podReadback.pods[1]" in source
+    assert "$ExecutorPod -eq $VerifierPod" in source
+
+
+def test_agent99_is_dispatch_only_and_apply_requires_independent_verifier() -> None:
+    source = AGENT99_ENTRYPOINT.read_text()
+
+    for fixed in (
+        '$CommandId = "telegram_receipt_index_apply_v1"',
+        '$CanonicalAsset = "public.awooop_outbound_message"',
+        '$JumpHost = "192.168.0.110"',
+        '$ControlPlaneHost = "192.168.0.120"',
+        '$Kubectl = "/usr/local/bin/kubectl"',
+        'domain = "database"',
+        'executor = "db_bounded_executor"',
+        "dispatchOnly = $true",
+        "windowsVmwareMutationPerformed = $false",
+        "arbitraryCommandAllowed = $false",
+        "crossDomainFallbackAllowed = $false",
+    ):
+        assert fixed in source
+
+    assert '"$ControlPlaneHost sudo -n $Kubectl exec "' in source
+    assert '"--namespace $Namespace --container $Container $PodName -- "' in source
+    assert '"python -B -m src.services.db_bounded_executor "' in source
+    assert 'Invoke-Agent99FixedDbStage "check" $ExecutorPod' in source
+    assert 'Invoke-Agent99FixedDbStage "apply" $ExecutorPod' in source
+    assert 'Invoke-Agent99FixedDbStage "verify" $VerifierPod' in source
+    apply_flow = source[source.index("# Apply is the only write path") :]
+    assert apply_flow.index('Invoke-Agent99FixedDbStage "check"') < apply_flow.index(
+        'Invoke-Agent99FixedDbStage "apply"'
+    )
+    assert apply_flow.index('Invoke-Agent99FixedDbStage "apply"') < apply_flow.index(
+        'Invoke-Agent99FixedDbStage "verify"'
+    )
+    assert (
+        "[string]$apply.receipt.pod_name -ne [string]$verify.receipt.pod_name"
+        in apply_flow
+    )
+    assert "Invoke-Expression" not in source
+    assert "-EncodedCommand" not in source
+    assert "/usr/local/sbin/awoooi-db-bounded-executor" not in source
+    assert "--sql" not in source
+    assert "rawTransportOutputPersisted" in source
+    assert "$script:TransportCleanupSucceeded = $false" in source
+    stage = source[source.index("function Invoke-Agent99FixedDbStage") :]
+    assert stage.index("ConvertFrom-Json") < stage.index("if ($process.ExitCode -ne 0)")
+    assert "$applyWriteState" in source
+    assert "$null -eq $apply.receipt.writes_performed" in source
+    assert "-not $apply.ok -and -not $apply.receipt" in source
+    assert "$terminalAllowed" in source
+    assert "$writeContract" in source
+    assert "$catalogContract" in source
+    assert (
+        "sql"
+        not in " ".join(
+            line for line in source.splitlines() if line.lstrip().startswith("param(")
+        ).lower()
+    )
+
+
+def test_static_k3s_contract_supports_second_pod_verifier() -> None:
+    deployment = (ROOT / "k8s/awoooi-prod/06-deployment-api.yaml").read_text()
+    hpa = (ROOT / "k8s/awoooi-prod/12-hpa.yaml").read_text()
+    dockerfile = (ROOT / "apps/api/Dockerfile").read_text()
+
+    assert "replicas: 2" in deployment
+    assert "app: awoooi-api" in deployment
+    assert "environment: prod" in deployment
+    assert "system: awoooi" in deployment
+    assert "- name: api" in deployment
+    assert "minReplicas: 2" in hpa
+    assert "COPY --chown=appuser:appuser apps/api/src/ ./src/" in dockerfile
+    assert "ENV AWOOOI_BUILD_COMMIT_SHA=$CACHE_BUST" in dockerfile
+
+
+def test_agent99_atomic_bundle_owns_the_dispatch_entrypoint() -> None:
+    deploy = (ROOT / "agent99-deploy.ps1").read_text()
+    bootstrap = (ROOT / "agent99-bootstrap.ps1").read_text()
+    contract = (ROOT / "agent99-contract-check.ps1").read_text()
+    transport = (
+        ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
+    ).read_text()
+    receiver = (
+        ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
+    ).read_text()
+
+    for source in (deploy, bootstrap, contract, transport, receiver):
+        assert '"agent99-db-bounded-executor.ps1"' in source
+    assert 'Add-Check "database:fixed_bounded_executor"' in contract
+    assert 'expectedRuntimeFileCount": 16' in transport
+    assert "expectedRuntimeFileCount=16" in transport
+    assert "expectedRuntimeFileCount -ne 16" in receiver
+    assert "fileCount = 16" in receiver