fix(security): isolate workload database identities

apps/api/tests/test_workload_database_identity_projection.py

@@ -0,0 +1,133 @@
+from pathlib import Path
+
+import yaml
+
+REPO_ROOT = Path(__file__).resolve().parents[3]
+
+
+def _deployment_container(path: str, container_name: str) -> dict:
+    documents = yaml.safe_load_all((REPO_ROOT / path).read_text())
+    deployment = next(
+        document
+        for document in documents
+        if document and document.get("kind") == "Deployment"
+    )
+    return next(
+        container
+        for container in deployment["spec"]["template"]["spec"]["containers"]
+        if container["name"] == container_name
+    )
+
+
+def _env_by_name(container: dict) -> dict[str, dict]:
+    return {entry["name"]: entry for entry in container.get("env", [])}
+
+
+def test_runtime_workloads_use_distinct_database_secret_projections() -> None:
+    workload_specs = {
+        "api": (
+            "k8s/awoooi-prod/06-deployment-api.yaml",
+            "api",
+            "awoooi-api-db",
+        ),
+        "worker": (
+            "k8s/awoooi-prod/08-deployment-worker.yaml",
+            "worker",
+            "awoooi-worker-db",
+        ),
+        "broker": (
+            "k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml",
+            "broker",
+            "awoooi-broker-db",
+        ),
+    }
+    observed_database_secret_refs: set[str] = set()
+
+    for workload_id, (path, container_name, expected_secret) in workload_specs.items():
+        container = _deployment_container(path, container_name)
+        secret_env_from = [
+            source
+            for source in container.get("envFrom", [])
+            if source.get("secretRef")
+        ]
+        assert secret_env_from == [], workload_id
+
+        env_names = [entry["name"] for entry in container.get("env", [])]
+        assert len(env_names) == len(set(env_names)), workload_id
+        env = _env_by_name(container)
+        database_ref = env["DATABASE_URL"]["valueFrom"]["secretKeyRef"]
+        assert database_ref == {
+            "name": expected_secret,
+            "key": "DATABASE_URL",
+        }
+        observed_database_secret_refs.add(database_ref["name"])
+        assert env["DATABASE_NULL_POOL"]["value"] == "true"
+
+    assert observed_database_secret_refs == {
+        "awoooi-api-db",
+        "awoooi-worker-db",
+        "awoooi-broker-db",
+    }
+
+
+def test_api_and_worker_project_shared_secrets_explicitly() -> None:
+    for path, container_name in (
+        ("k8s/awoooi-prod/06-deployment-api.yaml", "api"),
+        ("k8s/awoooi-prod/08-deployment-worker.yaml", "worker"),
+    ):
+        env = _env_by_name(_deployment_container(path, container_name))
+        for env_name in (
+            "REDIS_URL",
+            "GEMINI_API_KEY",
+            "OPENCLAW_TG_BOT_TOKEN",
+            "SRE_GROUP_CHAT_ID",
+            "WEBHOOK_HMAC_SECRET",
+            "AWOOOP_OPERATOR_API_KEY",
+        ):
+            secret_ref = env[env_name]["valueFrom"]["secretKeyRef"]
+            assert secret_ref["name"] == "awoooi-secrets"
+            assert secret_ref["key"] == env_name
+            assert secret_ref["optional"] is True
+
+
+def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None:
+    script = (
+        REPO_ROOT / "scripts/ops/awoooi-workload-db-identity-bootstrap.sh"
+    ).read_text()
+
+    assert "set +x" in script
+    assert "openssl rand -hex 32" in script
+    assert "--from-file=DATABASE_URL=/dev/stdin" in script
+    assert "--from-literal=DATABASE_URL" not in script
+    assert "docker exec -i -u postgres" in script
+    assert "table_privilege_gap_count" in script
+    assert "sequence_privilege_gap_count" in script
+    assert '"incidents"' in script
+    assert '"knowledge_entries"' in script
+    assert '"awooop_run_state"' in script
+    assert "ALTER ROLE %I NOLOGIN" in script
+    assert "DROP ROLE" not in script.upper()
+    assert "base64 -d" not in script
+    assert "GITHUB_RUN_ID" not in script
+    assert 'check|apply|verify|disable' in script
+    assert '"secret_value_exposed": False' in script
+    assert '"secret_value_logged": False' in script
+    assert "secret_value_read" not in script
+
+
+def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None:
+    workflow = (REPO_ROOT / ".gitea/workflows/cd.yaml").read_text()
+
+    assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow
+    assert "WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" in workflow
+    assert "restore_workload_database_projection" in workflow
+    assert "bash -s -- apply" in workflow
+    assert "bash -s -- verify" in workflow
+    assert workflow.index("bash -s -- apply") < workflow.index(
+        "bash -s -- verify"
+    )
+    assert '"spec": item["spec"]' in workflow
+    assert '"data": item' not in workflow
+    assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow
+    assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow
+    assert "HEAD^" not in workflow