From fd0f2d544a20f6ca92b10bffc533a021d13171d5 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 19:55:03 +0800 Subject: [PATCH] fix(cd): align rollout verifier with startup budget --- .gitea/workflows/cd.yaml | 63 ++++++++++++++++--- .../test_executor_network_policy_boundary.py | 50 ++++++++++++--- 2 files changed, 98 insertions(+), 15 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 5009bbb42..5e610631c 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -2948,6 +2948,11 @@ jobs: RISK_FILE="$(mktemp)" UNKNOWN_STATUS_COUNT=0 HEALTH_FAILURE_COUNT=0 + DEPLOY_STAGE="argocd_sync" + DEPLOY_TERMINAL_RECORDED=0 + # The API probes permit 305 seconds for cold-start DB bootstrap plus + # 20 seconds for readiness. Keep rollout above that bounded budget. + ROLLOUT_TIMEOUT_SECONDS=360 record_rollout_risk() { local message="$1" @@ -2955,7 +2960,24 @@ jobs: echo "⚠️ Rollout risk observed: $message" >&2 } + emit_deploy_terminal_receipt() { + local status="$1" + local stage="$2" + local workload="$3" + if [ "$DEPLOY_TERMINAL_RECORDED" = "1" ]; then + return 0 + fi + printf '%s\n' \ + "AWOOOI_DEPLOY_TERMINAL_RECEIPT=status=${status};stage=${stage};workload=${workload};source_revision=${SOURCE_REVISION};workflow_run_id=${WORKFLOW_RUN_ID};verifier_no_write=true;rollback_required=true" + DEPLOY_TERMINAL_RECORDED=1 + } + emit_rollout_evidence() { + local exit_code=$? + if [ "$exit_code" -ne 0 ]; then + emit_deploy_terminal_receipt \ + "blocked_with_safe_next_action" "$DEPLOY_STAGE" "unknown" + fi if [ -s "$RISK_FILE" ]; then local summary local kubectl_count @@ -2967,6 +2989,7 @@ jobs: echo "AWOOOI_ROLLOUT_RISK=0" fi rm -f "$RISK_FILE" + return "$exit_code" } trap emit_rollout_evidence EXIT @@ -3097,13 +3120,26 @@ jobs: sleep 5 done - # Two-replica rollouts can legitimately need more than two minutes - # before the second replacement is available. Keep this bounded, but - # do not abort before the independent boundary verifier can run. - $KUBECTL rollout status deployment/awoooi-api -n awoooi-prod --timeout=300s - $KUBECTL rollout status deployment/awoooi-web -n awoooi-prod --timeout=300s - $KUBECTL rollout status deployment/awoooi-worker -n awoooi-prod --timeout=300s - $KUBECTL rollout status deployment/awoooi-ansible-executor-broker -n awoooi-prod --timeout=300s + wait_for_rollout() { + local workload="$1" + DEPLOY_STAGE="rollout_${workload}" + if ! $KUBECTL rollout status "deployment/${workload}" \ + -n awoooi-prod --timeout="${ROLLOUT_TIMEOUT_SECONDS}s"; then + record_rollout_risk \ + "deployment_rollout_timeout workload=${workload} timeout_seconds=${ROLLOUT_TIMEOUT_SECONDS}" + emit_deploy_terminal_receipt \ + "blocked_with_safe_next_action" "$DEPLOY_STAGE" "$workload" + return 1 + fi + } + + # Two-replica rollouts can legitimately need more than two minutes. + # Keep convergence bounded by the declared probe budget, and leave a + # terminal receipt before the independent boundary verifier can run. + wait_for_rollout awoooi-api || exit 1 + wait_for_rollout awoooi-web || exit 1 + wait_for_rollout awoooi-worker || exit 1 + wait_for_rollout awoooi-ansible-executor-broker || exit 1 echo "✅ 部署完成" # AIA-P0-011: production must prove that the public API cannot mutate @@ -3132,6 +3168,7 @@ jobs: ;; esac } + DEPLOY_STAGE="executor_boundary_collect_live_identity_and_policy" echo "executor_boundary_stage=collect_live_identity_and_policy" API_SA=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \ -o jsonpath='{.spec.template.spec.serviceAccountName}') @@ -3287,6 +3324,7 @@ jobs: test "$LEGACY_EGRESS_RULE_COUNT" = "0" || { echo "❌ legacy namespace policy still creates an allow rule"; exit 1; } + DEPLOY_STAGE="executor_boundary_static_identity_and_policy_verified" echo "executor_boundary_stage=static_identity_and_policy_verified" verify_ssh_denied() { @@ -3368,6 +3406,7 @@ jobs: verify_broker_ssh_allowed || { echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1; } + DEPLOY_STAGE="executor_boundary_live_socket_boundary_verified" echo "executor_boundary_stage=live_socket_boundary_verified" read_workload_db_identity_once() { @@ -4206,6 +4245,7 @@ jobs: ) PY echo "AWOOOI_SECURITY_BOUNDARY_VERIFIED=1" + DEPLOY_TERMINAL_RECORDED=1 ARGOCD_WAIT ROLLOUT_EXIT=${PIPESTATUS[0]} set -e @@ -4222,12 +4262,19 @@ jobs: fi SECURITY_BOUNDARY_VERIFIED="0" + DEPLOY_TERMINAL_RECEIPT=$(grep '^AWOOOI_DEPLOY_TERMINAL_RECEIPT=' \ + "$ROLLOUT_LOG" | tail -1 | \ + sed 's/^AWOOOI_DEPLOY_TERMINAL_RECEIPT=//' || true) if grep -q '^AWOOOI_SECURITY_BOUNDARY_VERIFIED=1$' "$ROLLOUT_LOG"; then SECURITY_BOUNDARY_VERIFIED="1" fi if [ "$SECURITY_BOUNDARY_VERIFIED" != "1" ]; then restore_deploy_controls || true - echo "❌ executor security boundary did not reach its terminal verifier" + if [ -n "$DEPLOY_TERMINAL_RECEIPT" ]; then + echo "❌ deploy verifier terminal receipt: ${DEPLOY_TERMINAL_RECEIPT}" + else + echo "❌ executor security boundary did not reach its terminal verifier" + fi if [ "$ROLLOUT_EXIT" -eq 0 ]; then exit 1 fi diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py index c33c0ac68..c783c9606 100644 --- a/apps/api/tests/test_executor_network_policy_boundary.py +++ b/apps/api/tests/test_executor_network_policy_boundary.py @@ -224,16 +224,52 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None: assert "workload DB identity/budget remains in progress" in workflow -def test_cd_allows_bounded_rollout_convergence_before_boundary_verifier() -> None: +def test_cd_bounded_rollout_budget_and_failure_terminal_receipt() -> None: repo_root = Path(__file__).resolve().parents[3] workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text() - rollout_gate = workflow.split("# Two-replica rollouts", 1)[1].split( - "# AIA-P0-011:", 1 - )[0] + api_deployment = next( + document + for document in yaml.safe_load_all( + (repo_root / "k8s/awoooi-prod/06-deployment-api.yaml").read_text() + ) + if document + and document.get("kind") == "Deployment" + and document.get("metadata", {}).get("name") == "awoooi-api" + ) + api_container = next( + container + for container in api_deployment["spec"]["template"]["spec"]["containers"] + if container["name"] == "api" + ) + startup_probe = api_container["startupProbe"] + readiness_probe = api_container["readinessProbe"] + startup_budget_seconds = ( + startup_probe["initialDelaySeconds"] + + startup_probe["periodSeconds"] * startup_probe["failureThreshold"] + ) + readiness_budget_seconds = ( + readiness_probe["initialDelaySeconds"] + + readiness_probe["periodSeconds"] * readiness_probe["failureThreshold"] + ) + timeout_match = re.search(r"^\s*ROLLOUT_TIMEOUT_SECONDS=(\d+)$", workflow, re.M) - assert rollout_gate.count("--timeout=300s") == 4 - assert "--timeout=120s" not in rollout_gate - assert "independent boundary verifier" in rollout_gate + assert timeout_match is not None + assert int(timeout_match.group(1)) >= ( + startup_budget_seconds + readiness_budget_seconds + 30 + ) + pre_boundary_rollout = workflow.split( + 'echo "executor_boundary_stage=collect_live_identity_and_policy"', 1 + )[0] + assert "rollout status deployment/awoooi-api" not in pre_boundary_rollout + assert pre_boundary_rollout.count("wait_for_rollout awoooi-") == 4 + assert "independent boundary verifier" in pre_boundary_rollout + assert "AWOOOI_DEPLOY_TERMINAL_RECEIPT=" in workflow + assert "status=${status};stage=${stage};workload=${workload}" in workflow + assert "source_revision=${SOURCE_REVISION}" in workflow + assert "workflow_run_id=${WORKFLOW_RUN_ID}" in workflow + assert "verifier_no_write=true" in workflow + assert "rollback_required=true" in workflow + assert "deploy verifier terminal receipt:" in workflow def test_broker_probe_accepts_partial_live_route_evidence(tmp_path: Path) -> None: