From 262c9c8a7e2e65047fa79798e2af955d65914316 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 04:20:08 +0800 Subject: [PATCH] fix(security): isolate workload database identities --- .gitea/workflows/cd.yaml | 226 ++++++++- .../awoooi_priority_work_order_readback.py | 4 +- .../executor_trust_boundary_readback.py | 34 +- ...awoooi_priority_work_order_readback_api.py | 7 + .../test_executor_trust_boundary_readback.py | 49 +- ..._signal_worker_ansible_executor_binding.py | 19 +- ...t_workload_database_identity_projection.py | 133 +++++ k8s/awoooi-prod/06-deployment-api.yaml | 79 ++- ...08-deployment-ansible-executor-broker.yaml | 2 +- k8s/awoooi-prod/08-deployment-worker.yaml | 76 ++- .../awoooi-workload-db-identity-bootstrap.sh | 458 ++++++++++++++++++ 11 files changed, 1062 insertions(+), 25 deletions(-) create mode 100644 apps/api/tests/test_workload_database_identity_projection.py create mode 100755 scripts/ops/awoooi-workload-db-identity-bootstrap.sh diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index bc40e5dbe..58d9ed40c 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -2145,6 +2145,22 @@ jobs: echo "✅ 所有 Secrets 注入完成" SECRETS + # AIA-P0-011: record only Secret object pre-existence, never Secret data. + # The bootstrap streams CSPRNG credentials directly to PostgreSQL and + # kubectl stdin, then proves each identity with a no-write preflight. + WORKLOAD_DB_SECRET_PREEXISTENCE_FILE="/tmp/awoooi-workload-db-secret-preexistence" + : > "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" + for secret_name in awoooi-api-db awoooi-worker-db awoooi-broker-db; do + secret_existed=$(ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; if \$KUBECTL get secret '${secret_name}' -n awoooi-prod >/dev/null 2>&1; then printf 1; else printf 0; fi") + printf '%s=%s\n' "$secret_name" "$secret_existed" \ + >> "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" + done + cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \ + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' AWOOOI_RUN_ID='${GITEA_RUN_NUMBER:-cd}' bash -s -- apply" + echo "✅ workload DB identities, Secret projections, and preflights ready" + # 2026-04-11 Claude Sonnet 4.6 (Sprint B-3 ADR-069): # Deploy 改為 ArgoCD GitOps 模式:更新 kustomization.yaml → git push [skip ci] → ArgoCD sync # 舊做法 (kubectl set image) 與 ArgoCD selfHeal 衝突 — ArgoCD 會 revert 任何直接 kubectl 操作 @@ -2187,6 +2203,70 @@ jobs: HARBOR=192.168.0.110:5000 # ─── Step 1: Apply NetworkPolicy + ConfigMap + ServiceRegistry ─── + # Preserve only live Deployment specs and Secret object existence. + # Secret data is never read or copied into the rollback artifact. + WORKLOAD_DB_ROLLBACK_FILE="/tmp/awoooi-workload-db-deployment-rollback.yaml" + WORKLOAD_DB_SECRET_PREEXISTENCE_FILE="/tmp/awoooi-workload-db-secret-preexistence" + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get deployment awoooi-api awoooi-worker awoooi-ansible-executor-broker -n awoooi-prod -o json" \ + | python3 -c ' + import json + import sys + + source = json.load(sys.stdin) + items = [ + { + "apiVersion": "apps/v1", + "kind": "Deployment", + "metadata": { + "name": item["metadata"]["name"], + "namespace": item["metadata"]["namespace"], + "labels": item["metadata"].get("labels", {}), + }, + "spec": item["spec"], + } + for item in source["items"] + ] + json.dump({"apiVersion": "v1", "kind": "List", "items": items}, sys.stdout) + ' > "$WORKLOAD_DB_ROLLBACK_FILE" + test -s "$WORKLOAD_DB_ROLLBACK_FILE" + if [ ! -s "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" ]; then + : > "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" + for secret_name in awoooi-api-db awoooi-worker-db awoooi-broker-db; do + secret_existed=$(ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; if \$KUBECTL get secret '${secret_name}' -n awoooi-prod >/dev/null 2>&1; then printf 1; else printf 0; fi") + printf '%s=%s\n' "$secret_name" "$secret_existed" \ + >> "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" + done + fi + restore_workload_database_projection() { + echo "⏪ restoring previous AWOOOI workload database projection" + cat "$WORKLOAD_DB_ROLLBACK_FILE" | \ + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -" + for deployment_name in awoooi-api awoooi-worker awoooi-ansible-executor-broker; do + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL rollout status deployment/${deployment_name} -n awoooi-prod --timeout=180s" + done + + disable_workloads=() + while IFS='=' read -r secret_name secret_existed; do + [ "$secret_existed" = "0" ] || continue + case "$secret_name" in + awoooi-api-db) disable_workloads+=(api) ;; + awoooi-worker-db) disable_workloads+=(worker) ;; + awoooi-broker-db) disable_workloads+=(broker) ;; + esac + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL delete secret '${secret_name}' -n awoooi-prod --ignore-not-found=true" + done < "$WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" + if [ "${#disable_workloads[@]}" -gt 0 ]; then + cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \ + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' bash -s -- disable ${disable_workloads[*]}" + fi + } + # NetworkPolicy is intentionally outside kustomize because commonLabels # mutates nested selectors. Capture production truth before apply; the # checkout is intentionally depth=1, so parent Git history is not a @@ -2238,13 +2318,26 @@ jobs: "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL delete networkpolicy '$BROKER_POLICY_NAME' -n awoooi-prod --ignore-not-found=true" fi } + restore_deploy_controls() { + set +e + restore_workload_database_projection + workload_restore_exit=$? + restore_network_policy + network_restore_exit=$? + set -e + if [ "$workload_restore_exit" -ne 0 ] \ + || [ "$network_restore_exit" -ne 0 ]; then + echo "❌ deploy rollback incomplete: workload=${workload_restore_exit} network=${network_restore_exit}" >&2 + return 1 + fi + } cat k8s/awoooi-prod/02-network-policy.yaml | \ ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply --dry-run=server -f - >/dev/null" if ! cat k8s/awoooi-prod/02-network-policy.yaml | \ ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ "KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"; then - restore_network_policy + restore_deploy_controls || true exit 1 fi echo "✅ NetworkPolicy 已受控更新" @@ -2730,10 +2823,55 @@ jobs: WHERE rolname = current_user """)) row = result.mappings().one() + for table_name in ( + "incidents", + "knowledge_entries", + "awooop_run_state", + ): + await db.execute( + text(f'SELECT count(*) FROM "{table_name}"') + ) + table_gap_count = int((await db.execute(text(""" + SELECT count(*) + FROM pg_class relation + JOIN pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'public' + AND relation.relkind IN ('r', 'p', 'v', 'm') + AND ( + (has_table_privilege('awoooi', relation.oid, 'SELECT') + AND NOT has_table_privilege(current_user, relation.oid, 'SELECT')) + OR (has_table_privilege('awoooi', relation.oid, 'INSERT') + AND NOT has_table_privilege(current_user, relation.oid, 'INSERT')) + OR (has_table_privilege('awoooi', relation.oid, 'UPDATE') + AND NOT has_table_privilege(current_user, relation.oid, 'UPDATE')) + OR (has_table_privilege('awoooi', relation.oid, 'DELETE') + AND NOT has_table_privilege(current_user, relation.oid, 'DELETE')) + ) + """))).scalar_one()) + sequence_gap_count = int((await db.execute(text(""" + SELECT count(*) + FROM pg_class sequence + JOIN pg_namespace namespace + ON namespace.oid = sequence.relnamespace + WHERE namespace.nspname = 'public' + AND sequence.relkind = 'S' + AND ( + (has_sequence_privilege('awoooi', sequence.oid, 'USAGE') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'USAGE')) + OR (has_sequence_privilege('awoooi', sequence.oid, 'SELECT') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'SELECT')) + OR (has_sequence_privilege('awoooi', sequence.oid, 'UPDATE') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE')) + ) + """))).scalar_one()) except Exception as exc: print(json.dumps({ "role_fingerprint": "", "connection_limit": None, + "representative_preflight_passed": False, + "table_privilege_gap_count": None, + "sequence_privilege_gap_count": None, "error_type": type(exc).__name__, })) return @@ -2742,6 +2880,11 @@ jobs: str(row["role_name"]).encode("utf-8") ).hexdigest(), "connection_limit": int(row["connection_limit"]), + "representative_preflight_passed": ( + table_gap_count == 0 and sequence_gap_count == 0 + ), + "table_privilege_gap_count": table_gap_count, + "sequence_privilege_gap_count": sequence_gap_count, "error_type": None, })) @@ -2795,9 +2938,33 @@ jobs: | json_field connection_limit 2>/dev/null || true) BROKER_DB_CONNECTION_LIMIT=$(printf '%s' "$BROKER_DB_IDENTITY" \ | json_field connection_limit 2>/dev/null || true) + API_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$API_DB_IDENTITY" \ + | json_field representative_preflight_passed 2>/dev/null || true)") + WORKER_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field representative_preflight_passed 2>/dev/null || true)") + BROKER_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field representative_preflight_passed 2>/dev/null || true)") + API_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field table_privilege_gap_count 2>/dev/null || true) + WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field table_privilege_gap_count 2>/dev/null || true) + BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field table_privilege_gap_count 2>/dev/null || true) + API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field sequence_privilege_gap_count 2>/dev/null || true) + WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field sequence_privilege_gap_count 2>/dev/null || true) + BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field sequence_privilege_gap_count 2>/dev/null || true) API_DB_CONNECTION_LIMIT=${API_DB_CONNECTION_LIMIT:-0} WORKER_DB_CONNECTION_LIMIT=${WORKER_DB_CONNECTION_LIMIT:-0} BROKER_DB_CONNECTION_LIMIT=${BROKER_DB_CONNECTION_LIMIT:-0} + API_DB_TABLE_PRIVILEGE_GAP_COUNT=${API_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} + WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} + BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} + API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1} + WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1} + BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT=${BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT:-1} API_DB_PROJECTION=$(workload_secret_projection awoooi-api api) WORKER_DB_PROJECTION=$(workload_secret_projection awoooi-worker worker) @@ -2854,14 +3021,27 @@ jobs: && [ "$DISTINCT_DB_ROLE_FINGERPRINTS" = "true" ]; then DB_IDENTITY_ISOLATED=true fi + REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED=false + if [ "$API_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ + && [ "$WORKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ + && [ "$BROKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ + && [ "$API_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ + && [ "$WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ + && [ "$BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ + && [ "$API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ + && [ "$WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ + && [ "$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" -eq 0 ]; then + REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED=true + fi CONNECTION_BUDGET_VERIFIED=false if [ "$DB_IDENTITY_ISOLATED" = "true" ] \ + && [ "$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" = "true" ] \ && [ "$API_DB_NULL_POOL" = "true" ] \ && [ "$WORKER_DB_NULL_POOL" = "true" ] \ && [ "$BROKER_DB_NULL_POOL" = "true" ] \ - && [ "$API_DB_CONNECTION_LIMIT" -ge 1 ] \ - && [ "$WORKER_DB_CONNECTION_LIMIT" -ge 1 ] \ - && [ "$BROKER_DB_CONNECTION_LIMIT" -ge 1 ]; then + && [ "$API_DB_CONNECTION_LIMIT" -ge 4 ] \ + && [ "$WORKER_DB_CONNECTION_LIMIT" -ge 2 ] \ + && [ "$BROKER_DB_CONNECTION_LIMIT" -ge 2 ]; then CONNECTION_BUDGET_VERIFIED=true fi @@ -2887,7 +3067,7 @@ jobs: --from-literal=worker_ssh_egress_denied=true \ --from-literal=broker_ssh_egress_allowlisted=true \ --from-literal=legacy_namespace_egress_allow_absent=true \ - --from-literal=db_identity_verifier=runtime_role_fingerprint_and_secret_projection_v1 \ + --from-literal=db_identity_verifier=runtime_role_fingerprint_secret_projection_and_representative_preflight_v2 \ --from-literal=api_database_secret_ref="$API_DATABASE_SECRET_REF" \ --from-literal=worker_database_secret_ref="$WORKER_DATABASE_SECRET_REF" \ --from-literal=broker_database_secret_ref="$BROKER_DATABASE_SECRET_REF" \ @@ -2897,9 +3077,19 @@ jobs: --from-literal=api_db_connection_limit="$API_DB_CONNECTION_LIMIT" \ --from-literal=worker_db_connection_limit="$WORKER_DB_CONNECTION_LIMIT" \ --from-literal=broker_db_connection_limit="$BROKER_DB_CONNECTION_LIMIT" \ - --from-literal=api_required_connection_budget=1 \ - --from-literal=worker_required_connection_budget=1 \ - --from-literal=broker_required_connection_budget=1 \ + --from-literal=api_required_connection_budget=4 \ + --from-literal=worker_required_connection_budget=2 \ + --from-literal=broker_required_connection_budget=2 \ + --from-literal=api_representative_db_preflight_passed="$API_DB_REPRESENTATIVE_PREFLIGHT" \ + --from-literal=worker_representative_db_preflight_passed="$WORKER_DB_REPRESENTATIVE_PREFLIGHT" \ + --from-literal=broker_representative_db_preflight_passed="$BROKER_DB_REPRESENTATIVE_PREFLIGHT" \ + --from-literal=api_table_privilege_gap_count="$API_DB_TABLE_PRIVILEGE_GAP_COUNT" \ + --from-literal=worker_table_privilege_gap_count="$WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT" \ + --from-literal=broker_table_privilege_gap_count="$BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT" \ + --from-literal=api_sequence_privilege_gap_count="$API_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \ + --from-literal=worker_sequence_privilege_gap_count="$WORKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \ + --from-literal=broker_sequence_privilege_gap_count="$BROKER_DB_SEQUENCE_PRIVILEGE_GAP_COUNT" \ + --from-literal=representative_db_preflights_verified="$REPRESENTATIVE_DB_PREFLIGHTS_VERIFIED" \ --from-literal=api_database_null_pool="$API_DB_NULL_POOL" \ --from-literal=worker_database_null_pool="$WORKER_DB_NULL_POOL" \ --from-literal=broker_database_null_pool="$BROKER_DB_NULL_POOL" \ @@ -2972,9 +3162,15 @@ jobs: last_status = f"fetch_failed:{type(exc).__name__}" else: boundary = payload.get("executor_trust_boundary") or {} + database_boundary = ( + boundary.get("database_identity_boundary") or {} + ) last_status = str(boundary.get("status") or "missing") if ( boundary.get("production_boundary_verified") is True + and boundary.get("full_workload_boundary_verified") is True + and database_boundary.get("db_identity_isolated") is True + and database_boundary.get("connection_budget_verified") is True and str(boundary.get("verified_source_sha") or "").lower() == expected ): @@ -2991,12 +3187,23 @@ jobs: ROLLOUT_EXIT=${PIPESTATUS[0]} set -e + WORKLOAD_DB_VERIFY_EXIT=0 + cat scripts/ops/awoooi-workload-db-identity-bootstrap.sh | \ + ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \ + "AWOOOI_KUBECTL_SUDO=true AWOOOI_KUBECONFIG=/etc/rancher/k3s/k3s.yaml AWOOOI_KUBERNETES_SERVER='${{ env.K8S_API_SERVER }}' AWOOOI_RUN_ID='${GITEA_RUN_NUMBER:-cd}' bash -s -- verify" \ + || WORKLOAD_DB_VERIFY_EXIT=$? + if [ "$WORKLOAD_DB_VERIFY_EXIT" -ne 0 ]; then + restore_deploy_controls || true + echo "❌ workload database post-deploy verifier failed" + exit "$WORKLOAD_DB_VERIFY_EXIT" + fi + SECURITY_BOUNDARY_VERIFIED="0" if grep -q '^AWOOOI_SECURITY_BOUNDARY_VERIFIED=1$' "$ROLLOUT_LOG"; then SECURITY_BOUNDARY_VERIFIED="1" fi if [ "$SECURITY_BOUNDARY_VERIFIED" != "1" ]; then - restore_network_policy + restore_deploy_controls || true echo "❌ executor security boundary did not reach its terminal verifier" if [ "$ROLLOUT_EXIT" -eq 0 ]; then exit 1 @@ -3099,6 +3306,7 @@ jobs: PY if [ "$DEPLOY_READBACK_EXIT" -ne 0 ]; then + restore_deploy_controls || true exit "$DEPLOY_READBACK_EXIT" fi diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 4894d880d..daa4d4e18 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -551,7 +551,7 @@ _AI_AUTOMATION_PROGRAM_WORK_ITEMS: list[dict[str, Any]] = [ "order": 2, "status": "in_progress", "title": "收斂 API、Worker 與 executor 的最小權限信任邊界", - "problem": "API/Worker 的 RBAC 與 SSH mount 已拆分並由 production receipt 證明;剩餘缺口是 SSH egress policy、每 workload DB role/secret 隔離,以及首條成功 capability lifecycle 尚未同 run 關閉。", + "problem": "NetworkPolicy、RBAC、SSH mount 與 capability lifecycle 已由 production receipt 證明;剩餘缺口只有每 workload DB identity、Secret projection 與 connection budget 隔離。", "professional_practice": "zero trust workload identity、least privilege RBAC、namespace-scoped broker、capability allowlist、short-lived credentials", "asset_scope_ids": ["hosts", "services", "products", "tools", "observability_and_security"], "acceptance": [ @@ -562,7 +562,7 @@ _AI_AUTOMATION_PROGRAM_WORK_ITEMS: list[dict[str, Any]] = [ "API/Worker 到 host:22 的 socket verifier 必須 denied,只有 broker 可連 allowlisted SSH endpoints", "API、Worker 與 broker 使用分離的 DB identity/secret projection 與可驗證 connection budget", ], - "next_action": "部署 broker-only SSH NetworkPolicy 與 v2 public verifier;再修復 allowlisted canary 的 returncode=2,完成同 run capability issue/check/apply/verify/revoke,並拆分 workload DB role/secret。", + "next_action": "建立 API、Worker、broker 專用 LOGIN role 與獨立 Secret projection,執行 representative DB preflight、受控 rollout、live snapshot rollback 與 public-safe budget verifier。", }, { "id": "AIA-P0-002", diff --git a/apps/api/src/services/executor_trust_boundary_readback.py b/apps/api/src/services/executor_trust_boundary_readback.py index 5104099a9..5742e32a6 100644 --- a/apps/api/src/services/executor_trust_boundary_readback.py +++ b/apps/api/src/services/executor_trust_boundary_readback.py @@ -82,6 +82,17 @@ def _build_database_identity_boundary( f"{workload_id}_monolithic_secret_env_from_absent" ) ), + "representative_preflight_passed": _truthy( + receipt.get( + f"{workload_id}_representative_db_preflight_passed" + ) + ), + "table_privilege_gap_count": _int_or_none( + receipt.get(f"{workload_id}_table_privilege_gap_count") + ), + "sequence_privilege_gap_count": _int_or_none( + receipt.get(f"{workload_id}_sequence_privilege_gap_count") + ), } secret_refs = [ @@ -119,6 +130,15 @@ def _build_database_identity_boundary( and workloads[workload_id]["required_connection_budget"] > 0 for workload_id in _DB_WORKLOAD_IDS ) + representative_preflights_verified = bool( + all( + workloads[workload_id]["representative_preflight_passed"] is True + and workloads[workload_id]["table_privilege_gap_count"] == 0 + and workloads[workload_id]["sequence_privilege_gap_count"] == 0 + for workload_id in _DB_WORKLOAD_IDS + ) + and _truthy(receipt.get("representative_db_preflights_verified")) + ) db_identity_isolated = bool( distinct_secret_refs and distinct_role_fingerprints @@ -129,6 +149,7 @@ def _build_database_identity_boundary( db_identity_isolated and null_pool_verified and workload_budget_verified + and representative_preflights_verified and _truthy(receipt.get("connection_budget_verified")) ) @@ -147,6 +168,14 @@ def _build_database_identity_boundary( blockers.append(f"{workload_id}_database_role_fingerprint_unavailable") if workload["database_null_pool"] is not True: blockers.append(f"{workload_id}_database_null_pool_not_verified") + if ( + workload["representative_preflight_passed"] is not True + or workload["table_privilege_gap_count"] != 0 + or workload["sequence_privilege_gap_count"] != 0 + ): + blockers.append( + f"{workload_id}_representative_db_preflight_not_verified" + ) connection_limit = workload["connection_limit"] required_budget = workload["required_connection_budget"] if ( @@ -170,7 +199,7 @@ def _build_database_identity_boundary( for workload_id in _DB_WORKLOAD_IDS ] return { - "schema_version": "awoooi_workload_db_identity_budget_readback_v1", + "schema_version": "awoooi_workload_db_identity_budget_readback_v2", "status": "verified_ready" if not blockers else "in_progress", "receipt_ref": receipt_ref, "verifier": receipt.get("db_identity_verifier") or None, @@ -188,6 +217,9 @@ def _build_database_identity_boundary( "distinct_db_role_fingerprints": distinct_role_fingerprints, "database_null_pool_verified": null_pool_verified, "workload_connection_budgets_verified": workload_budget_verified, + "representative_db_preflights_verified": ( + representative_preflights_verified + ), }, "connection_budget": { "observed_total": ( diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 0f42dd4b7..c06452eee 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -2019,6 +2019,13 @@ def test_ai_automation_trust_boundary_requires_db_identity_and_budget_receipt() assert trust_boundary["runtime_progress"]["controls"][ "workload_db_connection_budget_verified" ] is False + assert "SSH egress policy" not in trust_boundary["problem"] + assert "capability lifecycle 尚未" not in trust_boundary["problem"] + assert "DB identity、Secret projection 與 connection budget" in ( + trust_boundary["problem"] + ) + assert "representative DB preflight" in trust_boundary["next_action"] + assert "live snapshot rollback" in trust_boundary["next_action"] assert program["summary"]["next_work_item_id"] == "AIA-P0-011" assert program["summary"]["done_count"] == 1 assert program["summary"]["completion_percent"] == 5 diff --git a/apps/api/tests/test_executor_trust_boundary_readback.py b/apps/api/tests/test_executor_trust_boundary_readback.py index 3a5fd8bc1..43e3ca6a7 100644 --- a/apps/api/tests/test_executor_trust_boundary_readback.py +++ b/apps/api/tests/test_executor_trust_boundary_readback.py @@ -25,19 +25,32 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]: "worker_ssh_egress_denied": "true", "broker_ssh_egress_allowlisted": "true", "legacy_namespace_egress_allow_absent": "true", - "db_identity_verifier": "runtime_role_fingerprint_and_secret_projection_v1", + "db_identity_verifier": ( + "runtime_role_fingerprint_secret_projection_and_" + "representative_preflight_v2" + ), "api_database_secret_ref": "awoooi-api-db", "worker_database_secret_ref": "awoooi-worker-db", "broker_database_secret_ref": "awoooi-broker-db", "api_db_role_fingerprint": "a" * 64, "worker_db_role_fingerprint": "b" * 64, "broker_db_role_fingerprint": "c" * 64, - "api_db_connection_limit": "2", + "api_db_connection_limit": "4", "worker_db_connection_limit": "2", "broker_db_connection_limit": "2", - "api_required_connection_budget": "1", - "worker_required_connection_budget": "1", - "broker_required_connection_budget": "1", + "api_required_connection_budget": "4", + "worker_required_connection_budget": "2", + "broker_required_connection_budget": "2", + "api_representative_db_preflight_passed": "true", + "worker_representative_db_preflight_passed": "true", + "broker_representative_db_preflight_passed": "true", + "api_table_privilege_gap_count": "0", + "worker_table_privilege_gap_count": "0", + "broker_table_privilege_gap_count": "0", + "api_sequence_privilege_gap_count": "0", + "worker_sequence_privilege_gap_count": "0", + "broker_sequence_privilege_gap_count": "0", + "representative_db_preflights_verified": "true", "api_database_null_pool": "true", "worker_database_null_pool": "true", "broker_database_null_pool": "true", @@ -67,9 +80,12 @@ def test_executor_trust_boundary_requires_live_controls_and_matching_source() -> assert database_boundary["db_identity_isolated"] is True assert database_boundary["connection_budget_verified"] is True assert database_boundary["connection_budget"] == { - "observed_total": 6, - "required_total": 3, + "observed_total": 8, + "required_total": 8, } + assert database_boundary["controls"][ + "representative_db_preflights_verified" + ] is True assert database_boundary["secret_values_read"] is False assert database_boundary["role_names_exposed"] is False @@ -126,3 +142,22 @@ def test_executor_trust_boundary_keeps_shared_database_identity_in_progress() -> assert "database_role_fingerprints_not_distinct" in database_boundary[ "active_blockers" ] + + +def test_executor_trust_boundary_requires_representative_database_preflight() -> None: + receipt = _verified_receipt() + receipt["worker_representative_db_preflight_passed"] = "false" + receipt["worker_table_privilege_gap_count"] = "1" + receipt["representative_db_preflights_verified"] = "false" + + readback = build_executor_trust_boundary_readback( + receipt, + deployed_source_sha="abc123", + ) + + database_boundary = readback["database_identity_boundary"] + assert database_boundary["db_identity_isolated"] is True + assert database_boundary["connection_budget_verified"] is False + assert "worker_representative_db_preflight_not_verified" in ( + database_boundary["active_blockers"] + ) diff --git a/apps/api/tests/test_signal_worker_ansible_executor_binding.py b/apps/api/tests/test_signal_worker_ansible_executor_binding.py index 1474d753e..e5db47418 100644 --- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py +++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py @@ -157,8 +157,14 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "awoooi-executor-boundary-verification" in workflow assert "awoooi_executor_boundary_verification_v3" in workflow assert "executor_boundary_public_readback=verified_ready" in workflow + assert 'boundary.get("full_workload_boundary_verified") is True' in workflow + assert 'database_boundary.get("db_identity_isolated") is True' in workflow + assert 'database_boundary.get("connection_budget_verified") is True' in workflow assert "verified_source_sha" in workflow - assert "runtime_role_fingerprint_and_secret_projection_v1" in workflow + assert ( + "runtime_role_fingerprint_secret_projection_and_" + "representative_preflight_v2" + ) in workflow assert "db_identity_isolated" in workflow assert "connection_budget_verified" in workflow assert "api_monolithic_secret_env_from_absent" in workflow @@ -166,6 +172,17 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "distinct_db_role_fingerprints" in workflow assert "current_user AS role_name" in workflow assert "hashlib.sha256" in workflow + assert "representative_db_preflights_verified" in workflow + assert "api_required_connection_budget=4" in workflow + assert "worker_required_connection_budget=2" in workflow + assert "broker_required_connection_budget=2" in workflow + assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow + assert "restore_workload_database_projection" in workflow + assert "awoooi-workload-db-identity-bootstrap.sh" in workflow + assert "secret object pre-existence" in workflow.lower() + assert "get secret '${secret_name}'" in workflow + assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow + assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow assert "argocd.argoproj.io/sync-options=Prune=false" in workflow assert "argocd.argoproj.io/compare-options=IgnoreExtraneous" in workflow assert "github.com/kubernetes-sigs/kustomize" not in workflow diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py new file mode 100644 index 000000000..702b03cc9 --- /dev/null +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -0,0 +1,133 @@ +from pathlib import Path + +import yaml + +REPO_ROOT = Path(__file__).resolve().parents[3] + + +def _deployment_container(path: str, container_name: str) -> dict: + documents = yaml.safe_load_all((REPO_ROOT / path).read_text()) + deployment = next( + document + for document in documents + if document and document.get("kind") == "Deployment" + ) + return next( + container + for container in deployment["spec"]["template"]["spec"]["containers"] + if container["name"] == container_name + ) + + +def _env_by_name(container: dict) -> dict[str, dict]: + return {entry["name"]: entry for entry in container.get("env", [])} + + +def test_runtime_workloads_use_distinct_database_secret_projections() -> None: + workload_specs = { + "api": ( + "k8s/awoooi-prod/06-deployment-api.yaml", + "api", + "awoooi-api-db", + ), + "worker": ( + "k8s/awoooi-prod/08-deployment-worker.yaml", + "worker", + "awoooi-worker-db", + ), + "broker": ( + "k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml", + "broker", + "awoooi-broker-db", + ), + } + observed_database_secret_refs: set[str] = set() + + for workload_id, (path, container_name, expected_secret) in workload_specs.items(): + container = _deployment_container(path, container_name) + secret_env_from = [ + source + for source in container.get("envFrom", []) + if source.get("secretRef") + ] + assert secret_env_from == [], workload_id + + env_names = [entry["name"] for entry in container.get("env", [])] + assert len(env_names) == len(set(env_names)), workload_id + env = _env_by_name(container) + database_ref = env["DATABASE_URL"]["valueFrom"]["secretKeyRef"] + assert database_ref == { + "name": expected_secret, + "key": "DATABASE_URL", + } + observed_database_secret_refs.add(database_ref["name"]) + assert env["DATABASE_NULL_POOL"]["value"] == "true" + + assert observed_database_secret_refs == { + "awoooi-api-db", + "awoooi-worker-db", + "awoooi-broker-db", + } + + +def test_api_and_worker_project_shared_secrets_explicitly() -> None: + for path, container_name in ( + ("k8s/awoooi-prod/06-deployment-api.yaml", "api"), + ("k8s/awoooi-prod/08-deployment-worker.yaml", "worker"), + ): + env = _env_by_name(_deployment_container(path, container_name)) + for env_name in ( + "REDIS_URL", + "GEMINI_API_KEY", + "OPENCLAW_TG_BOT_TOKEN", + "SRE_GROUP_CHAT_ID", + "WEBHOOK_HMAC_SECRET", + "AWOOOP_OPERATOR_API_KEY", + ): + secret_ref = env[env_name]["valueFrom"]["secretKeyRef"] + assert secret_ref["name"] == "awoooi-secrets" + assert secret_ref["key"] == env_name + assert secret_ref["optional"] is True + + +def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None: + script = ( + REPO_ROOT / "scripts/ops/awoooi-workload-db-identity-bootstrap.sh" + ).read_text() + + assert "set +x" in script + assert "openssl rand -hex 32" in script + assert "--from-file=DATABASE_URL=/dev/stdin" in script + assert "--from-literal=DATABASE_URL" not in script + assert "docker exec -i -u postgres" in script + assert "table_privilege_gap_count" in script + assert "sequence_privilege_gap_count" in script + assert '"incidents"' in script + assert '"knowledge_entries"' in script + assert '"awooop_run_state"' in script + assert "ALTER ROLE %I NOLOGIN" in script + assert "DROP ROLE" not in script.upper() + assert "base64 -d" not in script + assert "GITHUB_RUN_ID" not in script + assert 'check|apply|verify|disable' in script + assert '"secret_value_exposed": False' in script + assert '"secret_value_logged": False' in script + assert "secret_value_read" not in script + + +def test_cd_uses_live_spec_rollback_and_non_mutating_post_verifier() -> None: + workflow = (REPO_ROOT / ".gitea/workflows/cd.yaml").read_text() + + assert "WORKLOAD_DB_ROLLBACK_FILE" in workflow + assert "WORKLOAD_DB_SECRET_PREEXISTENCE_FILE" in workflow + assert "restore_workload_database_projection" in workflow + assert "bash -s -- apply" in workflow + assert "bash -s -- verify" in workflow + assert workflow.index("bash -s -- apply") < workflow.index( + "bash -s -- verify" + ) + assert '"spec": item["spec"]' in workflow + assert '"data": item' not in workflow + assert "get secret '${secret_name}' -n awoooi-prod -o jsonpath" not in workflow + assert "get secret awoooi-api-db -n awoooi-prod -o jsonpath" not in workflow + assert "HEAD^" not in workflow diff --git a/k8s/awoooi-prod/06-deployment-api.yaml b/k8s/awoooi-prod/06-deployment-api.yaml index c6522e565..1ee71f34f 100644 --- a/k8s/awoooi-prod/06-deployment-api.yaml +++ b/k8s/awoooi-prod/06-deployment-api.yaml @@ -71,11 +71,86 @@ spec: envFrom: - configMapRef: name: awoooi-config - - secretRef: - name: awoooi-secrets # 2026-04-12 ogt: env 優先於 envFrom — 覆蓋 configmap 特定值 # 說明: Kubernetes env: 優先於 envFrom:,用於 live-patch 後需同步回 Git env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: awoooi-api-db + key: DATABASE_URL + - name: REDIS_URL + valueFrom: + secretKeyRef: + name: awoooi-secrets + key: REDIS_URL + optional: true + - name: GEMINI_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GEMINI_API_KEY, optional: true} + - name: CLAUDE_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: CLAUDE_API_KEY, optional: true} + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: ANTHROPIC_API_KEY, optional: true} + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: NVIDIA_API_KEY, optional: true} + - name: OPENCLAW_TG_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_BOT_TOKEN, optional: true} + - name: OPENCLAW_TG_CHAT_ID + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_CHAT_ID, optional: true} + - name: OPENCLAW_TG_USER_WHITELIST + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_USER_WHITELIST, optional: true} + - name: SRE_GROUP_CHAT_ID + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SRE_GROUP_CHAT_ID, optional: true} + - name: WEBHOOK_HMAC_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: WEBHOOK_HMAC_SECRET, optional: true} + - name: AWOOOP_OPERATOR_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: AWOOOP_OPERATOR_API_KEY, optional: true} + - name: JWT_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: JWT_SECRET, optional: true} + - name: JWT_ALGORITHM + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: JWT_ALGORITHM, optional: true} + - name: LANGFUSE_PUBLIC_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: LANGFUSE_PUBLIC_KEY, optional: true} + - name: LANGFUSE_SECRET_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: LANGFUSE_SECRET_KEY, optional: true} + - name: SENTRY_DSN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SENTRY_DSN, optional: true} + - name: SENTRY_AUTH_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SENTRY_AUTH_TOKEN, optional: true} + - name: ARGOCD_API_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: ARGOCD_API_TOKEN, optional: true} + - name: GITEA_API_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GITEA_API_TOKEN, optional: true} + - name: GITEA_WEBHOOK_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GITEA_WEBHOOK_SECRET, optional: true} + - name: NEMOTRON_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: NEMOTRON_BOT_TOKEN, optional: true} + - name: OPENCLAW_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_BOT_TOKEN, optional: true} + - name: SMTP_HOST + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SMTP_HOST, optional: true} - name: AWOOOI_BUILD_COMMIT_SHA # 2026-06-29 Codex: CD rewrites this to the deployed image tag so # production deploy readback does not rely on a stale static snapshot. diff --git a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml index 7fa68f8ea..f4a6c3833 100644 --- a/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml +++ b/k8s/awoooi-prod/08-deployment-ansible-executor-broker.yaml @@ -43,7 +43,7 @@ spec: - name: DATABASE_URL valueFrom: secretKeyRef: - name: awoooi-secrets + name: awoooi-broker-db key: DATABASE_URL - name: OPENCLAW_TG_BOT_TOKEN valueFrom: diff --git a/k8s/awoooi-prod/08-deployment-worker.yaml b/k8s/awoooi-prod/08-deployment-worker.yaml index bc3910f3d..8a46970dc 100644 --- a/k8s/awoooi-prod/08-deployment-worker.yaml +++ b/k8s/awoooi-prod/08-deployment-worker.yaml @@ -76,9 +76,81 @@ spec: envFrom: - configMapRef: name: awoooi-config - - secretRef: - name: awoooi-secrets env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: awoooi-worker-db + key: DATABASE_URL + - name: REDIS_URL + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: REDIS_URL, optional: true} + - name: GEMINI_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GEMINI_API_KEY, optional: true} + - name: CLAUDE_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: CLAUDE_API_KEY, optional: true} + - name: ANTHROPIC_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: ANTHROPIC_API_KEY, optional: true} + - name: NVIDIA_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: NVIDIA_API_KEY, optional: true} + - name: OPENCLAW_TG_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_BOT_TOKEN, optional: true} + - name: OPENCLAW_TG_CHAT_ID + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_CHAT_ID, optional: true} + - name: OPENCLAW_TG_USER_WHITELIST + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_TG_USER_WHITELIST, optional: true} + - name: SRE_GROUP_CHAT_ID + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SRE_GROUP_CHAT_ID, optional: true} + - name: WEBHOOK_HMAC_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: WEBHOOK_HMAC_SECRET, optional: true} + - name: AWOOOP_OPERATOR_API_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: AWOOOP_OPERATOR_API_KEY, optional: true} + - name: JWT_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: JWT_SECRET, optional: true} + - name: JWT_ALGORITHM + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: JWT_ALGORITHM, optional: true} + - name: LANGFUSE_PUBLIC_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: LANGFUSE_PUBLIC_KEY, optional: true} + - name: LANGFUSE_SECRET_KEY + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: LANGFUSE_SECRET_KEY, optional: true} + - name: SENTRY_DSN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SENTRY_DSN, optional: true} + - name: SENTRY_AUTH_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SENTRY_AUTH_TOKEN, optional: true} + - name: ARGOCD_API_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: ARGOCD_API_TOKEN, optional: true} + - name: GITEA_API_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GITEA_API_TOKEN, optional: true} + - name: GITEA_WEBHOOK_SECRET + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: GITEA_WEBHOOK_SECRET, optional: true} + - name: NEMOTRON_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: NEMOTRON_BOT_TOKEN, optional: true} + - name: OPENCLAW_BOT_TOKEN + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: OPENCLAW_BOT_TOKEN, optional: true} + - name: SMTP_HOST + valueFrom: + secretKeyRef: {name: awoooi-secrets, key: SMTP_HOST, optional: true} - name: WORKER_MODE value: "true" - name: CONSUMER_GROUP diff --git a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh new file mode 100755 index 000000000..b2920efa3 --- /dev/null +++ b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash +set -euo pipefail +set +x + +MODE="${1:-check}" +shift || true + +NAMESPACE="${AWOOOI_NAMESPACE:-awoooi-prod}" +BROKER_DEPLOYMENT="${AWOOOI_DB_BOOTSTRAP_BROKER_DEPLOYMENT:-awoooi-ansible-executor-broker}" +BROKER_CONTAINER="${AWOOOI_DB_BOOTSTRAP_BROKER_CONTAINER:-broker}" +POSTGRES_SSH_TARGET="${AWOOOI_DB_BOOTSTRAP_POSTGRES_SSH_TARGET:-ollama@192.168.0.188}" +POSTGRES_CONTAINER="${AWOOOI_DB_BOOTSTRAP_POSTGRES_CONTAINER:-k3s-postgres-recovery}" +POSTGRES_DATABASE="${AWOOOI_DB_BOOTSTRAP_DATABASE:-awoooi_prod}" +POSTGRES_HOST="${AWOOOI_DB_BOOTSTRAP_DATABASE_HOST:-192.168.0.188}" +POSTGRES_PORT="${AWOOOI_DB_BOOTSTRAP_DATABASE_PORT:-5432}" +SHARED_APP_ROLE="${AWOOOI_DB_BOOTSTRAP_SHARED_ROLE:-awoooi}" +RLS_APP_ROLE="${AWOOOI_DB_BOOTSTRAP_RLS_ROLE:-awooop_app}" +SSH_KEY_PATH="${AWOOOI_DB_BOOTSTRAP_SSH_KEY_PATH:-/run/secrets/ssh_mcp_key}" +KNOWN_HOSTS_PATH="${AWOOOI_DB_BOOTSTRAP_KNOWN_HOSTS_PATH:-/etc/ssh-mcp/known_hosts}" + +case "$MODE" in + check|apply|verify|disable) ;; + *) + echo "usage: $0 check|apply|verify|disable [api|worker|broker ...]" >&2 + exit 2 + ;; +esac + +KUBECTL=(kubectl) +if [ "${AWOOOI_KUBECTL_SUDO:-false}" = "true" ]; then + KUBECTL=(sudo kubectl) +fi +if [ -n "${AWOOOI_KUBECONFIG:-}" ]; then + KUBECTL+=(--kubeconfig="${AWOOOI_KUBECONFIG}") +fi +if [ -n "${AWOOOI_KUBERNETES_SERVER:-}" ]; then + KUBECTL+=(--server="${AWOOOI_KUBERNETES_SERVER}") +fi + +kube() { + "${KUBECTL[@]}" "$@" +} + +workload_role() { + case "$1" in + api) printf 'awoooi_api_runtime' ;; + worker) printf 'awoooi_worker_runtime' ;; + broker) printf 'awoooi_broker_runtime' ;; + *) return 1 ;; + esac +} + +workload_secret() { + case "$1" in + api) printf 'awoooi-api-db' ;; + worker) printf 'awoooi-worker-db' ;; + broker) printf 'awoooi-broker-db' ;; + *) return 1 ;; + esac +} + +workload_connection_limit() { + case "$1" in + api) printf '4' ;; + worker|broker) printf '2' ;; + *) return 1 ;; + esac +} + +workload_deployment() { + case "$1" in + api) printf 'awoooi-api|api' ;; + worker) printf 'awoooi-worker|worker' ;; + broker) printf 'awoooi-ansible-executor-broker|broker' ;; + *) return 1 ;; + esac +} + +selected_workloads=("$@") +if [ "${#selected_workloads[@]}" -eq 0 ]; then + selected_workloads=(api worker broker) +fi +for workload in "${selected_workloads[@]}"; do + workload_role "$workload" >/dev/null || { + echo "invalid workload id: $workload" >&2 + exit 2 + } +done + +admin_psql() { + kube exec -i "deployment/${BROKER_DEPLOYMENT}" -n "$NAMESPACE" \ + -c "$BROKER_CONTAINER" -- \ + ssh -i "$SSH_KEY_PATH" \ + -o BatchMode=yes \ + -o ConnectTimeout=10 \ + -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile="$KNOWN_HOSTS_PATH" \ + "$POSTGRES_SSH_TARGET" \ + "docker exec -i -u postgres ${POSTGRES_CONTAINER} psql -h /tmp -U postgres -d ${POSTGRES_DATABASE} -v ON_ERROR_STOP=1 -Atq" +} + +require_control_path() { + kube get "deployment/${BROKER_DEPLOYMENT}" -n "$NAMESPACE" >/dev/null + kube exec "deployment/${BROKER_DEPLOYMENT}" -n "$NAMESPACE" \ + -c "$BROKER_CONTAINER" -- test -s "$SSH_KEY_PATH" + kube exec "deployment/${BROKER_DEPLOYMENT}" -n "$NAMESPACE" \ + -c "$BROKER_CONTAINER" -- test -s "$KNOWN_HOSTS_PATH" + test "$(printf 'SELECT 1;\n' | admin_psql | tail -n 1)" = "1" + role_count="$(printf \ + "SELECT count(*) FROM pg_roles WHERE rolname IN ('%s','%s');\n" \ + "$SHARED_APP_ROLE" "$RLS_APP_ROLE" | admin_psql | tail -n 1)" + test "$role_count" = "2" || { + echo "required shared database roles are unavailable" >&2 + exit 1 + } +} + +role_state() { + role="$1" + printf "SELECT concat(rolcanlogin, '|', rolconnlimit, '|', pg_has_role('%s', '%s', 'member'), '|', pg_has_role('%s', '%s', 'member')) FROM pg_roles WHERE rolname = '%s';\n" \ + "$role" "$SHARED_APP_ROLE" "$role" "$RLS_APP_ROLE" "$role" \ + | admin_psql | tail -n 1 +} + +disable_role() { + role="$1" + cat </dev/null +DO \$disable\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN + EXECUTE format('ALTER ROLE %I NOLOGIN', '${role}'); + END IF; +END +\$disable\$; +SQL +} + +ensure_existing_role_contract() { + role="$1" + connection_limit="$2" + cat </dev/null +DO \$contract\$ +BEGIN + IF NOT EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN + RAISE EXCEPTION 'workload database role missing'; + END IF; + EXECUTE format( + 'ALTER ROLE %I LOGIN INHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT %s', + '${role}', ${connection_limit} + ); +END +\$contract\$; +GRANT ${SHARED_APP_ROLE} TO ${role}; +GRANT ${RLS_APP_ROLE} TO ${role}; +SQL +} + +create_role_and_secret() { + workload="$1" + role="$2" + secret="$3" + connection_limit="$4" + password="$(openssl rand -hex 32)" + test "${#password}" -eq 64 + + cat </dev/null +DO \$bootstrap\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = '${role}') THEN + EXECUTE format( + 'ALTER ROLE %I LOGIN INHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT %s PASSWORD %L', + '${role}', ${connection_limit}, '${password}' + ); + ELSE + EXECUTE format( + 'CREATE ROLE %I LOGIN INHERIT NOSUPERUSER NOCREATEDB NOCREATEROLE NOREPLICATION CONNECTION LIMIT %s PASSWORD %L', + '${role}', ${connection_limit}, '${password}' + ); + END IF; +END +\$bootstrap\$; +GRANT ${SHARED_APP_ROLE} TO ${role}; +GRANT ${RLS_APP_ROLE} TO ${role}; +SQL + + database_url="postgresql+asyncpg://${role}:${password}@${POSTGRES_HOST}:${POSTGRES_PORT}/${POSTGRES_DATABASE}?ssl=disable" + if ! printf '%s' "$database_url" \ + | kube create secret generic "$secret" -n "$NAMESPACE" \ + --from-file=DATABASE_URL=/dev/stdin \ + --dry-run=client -o yaml \ + | kube apply -f - >/dev/null; then + disable_role "$role" + unset database_url password + echo "workload database Secret apply failed; role was disabled" >&2 + return 1 + fi + if ! kube label secret "$secret" -n "$NAMESPACE" \ + app.kubernetes.io/managed-by=awoooi-workload-db-identity-bootstrap \ + awoooi.wooo.work/database-workload="$workload" --overwrite >/dev/null; then + kube delete secret "$secret" -n "$NAMESPACE" \ + --ignore-not-found=true >/dev/null 2>&1 || true + disable_role "$role" + unset database_url password + echo "workload database Secret label failed; new Secret removed and role disabled" >&2 + return 1 + fi + unset database_url password +} + +preflight_workload() { + workload="$1" + secret="$2" + connection_limit="$3" + deployment_spec="$(workload_deployment "$workload")" + deployment="${deployment_spec%%|*}" + container="${deployment_spec#*|}" + image="$(kube get "deployment/${deployment}" -n "$NAMESPACE" \ + -o "jsonpath={.spec.template.spec.containers[?(@.name=='${container}')].image}")" + test -n "$image" + suffix="${AWOOOI_RUN_ID:-manual}-${RANDOM}" + pod_name="awoooi-db-preflight-${workload}-${suffix}" + pod_name="${pod_name:0:63}" + + cleanup_preflight() { + kube delete pod "$pod_name" -n "$NAMESPACE" \ + --ignore-not-found=true --wait=false >/dev/null 2>&1 || true + } + cleanup_preflight + cat </dev/null +apiVersion: v1 +kind: Pod +metadata: + name: ${pod_name} + namespace: ${NAMESPACE} + labels: + system: awoooi + component: workload-db-preflight + awoooi.wooo.work/database-workload: ${workload} +spec: + restartPolicy: Never + automountServiceAccountToken: false + containers: + - name: preflight + image: ${image} + imagePullPolicy: IfNotPresent + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: ${secret} + key: DATABASE_URL + - name: EXPECTED_CONNECTION_LIMIT + value: "${connection_limit}" + - name: WORKLOAD_ID + value: "${workload}" + command: ["python", "-c"] + args: + - | + import asyncio + import hashlib + import json + import os + + from sqlalchemy import text + from sqlalchemy.ext.asyncio import create_async_engine + from sqlalchemy.pool import NullPool + + + async def main() -> None: + engine = create_async_engine( + os.environ["DATABASE_URL"], poolclass=NullPool + ) + try: + async with engine.connect() as conn: + identity = ( + await conn.execute(text(""" + SELECT current_user AS role_name, rolconnlimit + FROM pg_roles WHERE rolname = current_user + """)) + ).mappings().one() + for table_name in ( + "incidents", + "knowledge_entries", + "awooop_run_state", + ): + await conn.execute( + text(f'SELECT count(*) FROM "{table_name}"') + ) + table_gap_count = int((await conn.execute(text(""" + SELECT count(*) + FROM pg_class relation + JOIN pg_namespace namespace + ON namespace.oid = relation.relnamespace + WHERE namespace.nspname = 'public' + AND relation.relkind IN ('r', 'p', 'v', 'm') + AND ( + (has_table_privilege('awoooi', relation.oid, 'SELECT') + AND NOT has_table_privilege(current_user, relation.oid, 'SELECT')) + OR (has_table_privilege('awoooi', relation.oid, 'INSERT') + AND NOT has_table_privilege(current_user, relation.oid, 'INSERT')) + OR (has_table_privilege('awoooi', relation.oid, 'UPDATE') + AND NOT has_table_privilege(current_user, relation.oid, 'UPDATE')) + OR (has_table_privilege('awoooi', relation.oid, 'DELETE') + AND NOT has_table_privilege(current_user, relation.oid, 'DELETE')) + ) + """))).scalar_one()) + sequence_gap_count = int((await conn.execute(text(""" + SELECT count(*) + FROM pg_class sequence + JOIN pg_namespace namespace + ON namespace.oid = sequence.relnamespace + WHERE namespace.nspname = 'public' + AND sequence.relkind = 'S' + AND ( + (has_sequence_privilege('awoooi', sequence.oid, 'USAGE') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'USAGE')) + OR (has_sequence_privilege('awoooi', sequence.oid, 'SELECT') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'SELECT')) + OR (has_sequence_privilege('awoooi', sequence.oid, 'UPDATE') + AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE')) + ) + """))).scalar_one()) + finally: + await engine.dispose() + + expected_limit = int(os.environ["EXPECTED_CONNECTION_LIMIT"]) + observed_limit = int(identity["rolconnlimit"]) + if observed_limit != expected_limit: + raise RuntimeError("connection_limit_mismatch") + if table_gap_count or sequence_gap_count: + raise RuntimeError("database_privilege_gap") + print(json.dumps({ + "status": "verified", + "workload": os.environ["WORKLOAD_ID"], + "role_fingerprint": hashlib.sha256( + str(identity["role_name"]).encode("utf-8") + ).hexdigest(), + "connection_limit": observed_limit, + "representative_read_count": 3, + "table_privilege_gap_count": table_gap_count, + "sequence_privilege_gap_count": sequence_gap_count, + "secret_value_exposed": False, + "secret_value_logged": False, + }, separators=(",", ":"))) + + + try: + asyncio.run(main()) + except Exception as exc: + print(json.dumps({ + "status": "failed", + "workload": os.environ.get("WORKLOAD_ID", "unknown"), + "error_type": type(exc).__name__, + "secret_value_exposed": False, + "secret_value_logged": False, + }, separators=(",", ":"))) + raise SystemExit(1) +YAML + + phase="" + for _ in $(seq 1 60); do + phase="$(kube get pod "$pod_name" -n "$NAMESPACE" \ + -o jsonpath='{.status.phase}' 2>/dev/null || true)" + case "$phase" in + Succeeded) break ;; + Failed) + kube logs "$pod_name" -n "$NAMESPACE" -c preflight || true + cleanup_preflight + return 1 + ;; + esac + sleep 2 + done + if [ "$phase" != "Succeeded" ]; then + echo "workload database preflight timed out: $workload" >&2 + cleanup_preflight + return 1 + fi + kube logs "$pod_name" -n "$NAMESPACE" -c preflight + cleanup_preflight +} + +require_control_path + +if [ "$MODE" = "disable" ]; then + for workload in "${selected_workloads[@]}"; do + disable_role "$(workload_role "$workload")" + echo "workload=$workload role_login_enabled=false" + done + exit 0 +fi + +new_workloads=() +rollback_new_workloads() { + for workload in "${new_workloads[@]}"; do + secret="$(workload_secret "$workload")" + role="$(workload_role "$workload")" + kube delete secret "$secret" -n "$NAMESPACE" \ + --ignore-not-found=true >/dev/null 2>&1 || true + disable_role "$role" >/dev/null 2>&1 || true + echo "workload=$workload bootstrap_rollback=secret_removed_role_disabled" + done +} +if [ "$MODE" = "apply" ]; then + trap 'status=$?; if [ "$status" -ne 0 ]; then rollback_new_workloads; fi; exit "$status"' EXIT +fi + +ready=true +for workload in "${selected_workloads[@]}"; do + role="$(workload_role "$workload")" + secret="$(workload_secret "$workload")" + connection_limit="$(workload_connection_limit "$workload")" + secret_present=false + if kube get secret "$secret" -n "$NAMESPACE" >/dev/null 2>&1; then + secret_present=true + fi + state="$(role_state "$role")" + role_present=false + if [ -n "$state" ]; then + role_present=true + fi + + if [ "$MODE" = "apply" ]; then + if [ "$secret_present" = "true" ]; then + test "$role_present" = "true" || { + echo "workload=$workload secret_present=true role_present=false" >&2 + exit 1 + } + ensure_existing_role_contract "$role" "$connection_limit" + else + create_role_and_secret "$workload" "$role" "$secret" "$connection_limit" + new_workloads+=("$workload") + secret_present=true + role_present=true + fi + state="$(role_state "$role")" + fi + + expected_state="t|${connection_limit}|t|t" + state_ready=false + if [ "$secret_present" = "true" ] && [ "$state" = "$expected_state" ]; then + state_ready=true + else + ready=false + fi + echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit" + + if [ "$MODE" = "apply" ] || [ "$MODE" = "verify" ]; then + preflight_workload "$workload" "$secret" "$connection_limit" + fi +done + +if [ "$ready" != "true" ]; then + echo "WORKLOAD_DB_IDENTITY_BOOTSTRAP_READY=0" + exit 1 +fi +trap - EXIT +echo "WORKLOAD_DB_IDENTITY_BOOTSTRAP_READY=1"