fix(cd): align rollout verifier with startup budget
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
E2E Health Check / e2e-health (push) Failing after 47s

This commit is contained in:
ogt
2026-07-14 19:55:03 +08:00
parent e0af904759
commit fd0f2d544a
2 changed files with 98 additions and 15 deletions

View File

@@ -2948,6 +2948,11 @@ jobs:
RISK_FILE="$(mktemp)"
UNKNOWN_STATUS_COUNT=0
HEALTH_FAILURE_COUNT=0
DEPLOY_STAGE="argocd_sync"
DEPLOY_TERMINAL_RECORDED=0
# The API probes permit 305 seconds for cold-start DB bootstrap plus
# 20 seconds for readiness. Keep rollout above that bounded budget.
ROLLOUT_TIMEOUT_SECONDS=360
record_rollout_risk() {
local message="$1"
@@ -2955,7 +2960,24 @@ jobs:
echo "⚠️ Rollout risk observed: $message" >&2
}
emit_deploy_terminal_receipt() {
local status="$1"
local stage="$2"
local workload="$3"
if [ "$DEPLOY_TERMINAL_RECORDED" = "1" ]; then
return 0
fi
printf '%s\n' \
"AWOOOI_DEPLOY_TERMINAL_RECEIPT=status=${status};stage=${stage};workload=${workload};source_revision=${SOURCE_REVISION};workflow_run_id=${WORKFLOW_RUN_ID};verifier_no_write=true;rollback_required=true"
DEPLOY_TERMINAL_RECORDED=1
}
emit_rollout_evidence() {
local exit_code=$?
if [ "$exit_code" -ne 0 ]; then
emit_deploy_terminal_receipt \
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "unknown"
fi
if [ -s "$RISK_FILE" ]; then
local summary
local kubectl_count
@@ -2967,6 +2989,7 @@ jobs:
echo "AWOOOI_ROLLOUT_RISK=0"
fi
rm -f "$RISK_FILE"
return "$exit_code"
}
trap emit_rollout_evidence EXIT
@@ -3097,13 +3120,26 @@ jobs:
sleep 5
done
# Two-replica rollouts can legitimately need more than two minutes
# before the second replacement is available. Keep this bounded, but
# do not abort before the independent boundary verifier can run.
$KUBECTL rollout status deployment/awoooi-api -n awoooi-prod --timeout=300s
$KUBECTL rollout status deployment/awoooi-web -n awoooi-prod --timeout=300s
$KUBECTL rollout status deployment/awoooi-worker -n awoooi-prod --timeout=300s
$KUBECTL rollout status deployment/awoooi-ansible-executor-broker -n awoooi-prod --timeout=300s
wait_for_rollout() {
local workload="$1"
DEPLOY_STAGE="rollout_${workload}"
if ! $KUBECTL rollout status "deployment/${workload}" \
-n awoooi-prod --timeout="${ROLLOUT_TIMEOUT_SECONDS}s"; then
record_rollout_risk \
"deployment_rollout_timeout workload=${workload} timeout_seconds=${ROLLOUT_TIMEOUT_SECONDS}"
emit_deploy_terminal_receipt \
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "$workload"
return 1
fi
}
# Two-replica rollouts can legitimately need more than two minutes.
# Keep convergence bounded by the declared probe budget, and leave a
# terminal receipt before the independent boundary verifier can run.
wait_for_rollout awoooi-api || exit 1
wait_for_rollout awoooi-web || exit 1
wait_for_rollout awoooi-worker || exit 1
wait_for_rollout awoooi-ansible-executor-broker || exit 1
echo "✅ 部署完成"
# AIA-P0-011: production must prove that the public API cannot mutate
@@ -3132,6 +3168,7 @@ jobs:
;;
esac
}
DEPLOY_STAGE="executor_boundary_collect_live_identity_and_policy"
echo "executor_boundary_stage=collect_live_identity_and_policy"
API_SA=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
-o jsonpath='{.spec.template.spec.serviceAccountName}')
@@ -3287,6 +3324,7 @@ jobs:
test "$LEGACY_EGRESS_RULE_COUNT" = "0" || {
echo "❌ legacy namespace policy still creates an allow rule"; exit 1;
}
DEPLOY_STAGE="executor_boundary_static_identity_and_policy_verified"
echo "executor_boundary_stage=static_identity_and_policy_verified"
verify_ssh_denied() {
@@ -3368,6 +3406,7 @@ jobs:
verify_broker_ssh_allowed || {
echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1;
}
DEPLOY_STAGE="executor_boundary_live_socket_boundary_verified"
echo "executor_boundary_stage=live_socket_boundary_verified"
read_workload_db_identity_once() {
@@ -4206,6 +4245,7 @@ jobs:
)
PY
echo "AWOOOI_SECURITY_BOUNDARY_VERIFIED=1"
DEPLOY_TERMINAL_RECORDED=1
ARGOCD_WAIT
ROLLOUT_EXIT=${PIPESTATUS[0]}
set -e
@@ -4222,12 +4262,19 @@ jobs:
fi
SECURITY_BOUNDARY_VERIFIED="0"
DEPLOY_TERMINAL_RECEIPT=$(grep '^AWOOOI_DEPLOY_TERMINAL_RECEIPT=' \
"$ROLLOUT_LOG" | tail -1 | \
sed 's/^AWOOOI_DEPLOY_TERMINAL_RECEIPT=//' || true)
if grep -q '^AWOOOI_SECURITY_BOUNDARY_VERIFIED=1$' "$ROLLOUT_LOG"; then
SECURITY_BOUNDARY_VERIFIED="1"
fi
if [ "$SECURITY_BOUNDARY_VERIFIED" != "1" ]; then
restore_deploy_controls || true
echo "❌ executor security boundary did not reach its terminal verifier"
if [ -n "$DEPLOY_TERMINAL_RECEIPT" ]; then
echo "❌ deploy verifier terminal receipt: ${DEPLOY_TERMINAL_RECEIPT}"
else
echo "❌ executor security boundary did not reach its terminal verifier"
fi
if [ "$ROLLOUT_EXIT" -eq 0 ]; then
exit 1
fi

View File

@@ -224,16 +224,52 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None:
assert "workload DB identity/budget remains in progress" in workflow
def test_cd_allows_bounded_rollout_convergence_before_boundary_verifier() -> None:
def test_cd_bounded_rollout_budget_and_failure_terminal_receipt() -> None:
repo_root = Path(__file__).resolve().parents[3]
workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text()
rollout_gate = workflow.split("# Two-replica rollouts", 1)[1].split(
"# AIA-P0-011:", 1
)[0]
api_deployment = next(
document
for document in yaml.safe_load_all(
(repo_root / "k8s/awoooi-prod/06-deployment-api.yaml").read_text()
)
if document
and document.get("kind") == "Deployment"
and document.get("metadata", {}).get("name") == "awoooi-api"
)
api_container = next(
container
for container in api_deployment["spec"]["template"]["spec"]["containers"]
if container["name"] == "api"
)
startup_probe = api_container["startupProbe"]
readiness_probe = api_container["readinessProbe"]
startup_budget_seconds = (
startup_probe["initialDelaySeconds"]
+ startup_probe["periodSeconds"] * startup_probe["failureThreshold"]
)
readiness_budget_seconds = (
readiness_probe["initialDelaySeconds"]
+ readiness_probe["periodSeconds"] * readiness_probe["failureThreshold"]
)
timeout_match = re.search(r"^\s*ROLLOUT_TIMEOUT_SECONDS=(\d+)$", workflow, re.M)
assert rollout_gate.count("--timeout=300s") == 4
assert "--timeout=120s" not in rollout_gate
assert "independent boundary verifier" in rollout_gate
assert timeout_match is not None
assert int(timeout_match.group(1)) >= (
startup_budget_seconds + readiness_budget_seconds + 30
)
pre_boundary_rollout = workflow.split(
'echo "executor_boundary_stage=collect_live_identity_and_policy"', 1
)[0]
assert "rollout status deployment/awoooi-api" not in pre_boundary_rollout
assert pre_boundary_rollout.count("wait_for_rollout awoooi-") == 4
assert "independent boundary verifier" in pre_boundary_rollout
assert "AWOOOI_DEPLOY_TERMINAL_RECEIPT=" in workflow
assert "status=${status};stage=${stage};workload=${workload}" in workflow
assert "source_revision=${SOURCE_REVISION}" in workflow
assert "workflow_run_id=${WORKFLOW_RUN_ID}" in workflow
assert "verifier_no_write=true" in workflow
assert "rollback_required=true" in workflow
assert "deploy verifier terminal receipt:" in workflow
def test_broker_probe_accepts_partial_live_route_evidence(tmp_path: Path) -> None: