fix(cd): align rollout verifier with startup budget
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
E2E Health Check / e2e-health (push) Failing after 47s
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m46s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
E2E Health Check / e2e-health (push) Failing after 47s
This commit is contained in:
@@ -2948,6 +2948,11 @@ jobs:
|
||||
RISK_FILE="$(mktemp)"
|
||||
UNKNOWN_STATUS_COUNT=0
|
||||
HEALTH_FAILURE_COUNT=0
|
||||
DEPLOY_STAGE="argocd_sync"
|
||||
DEPLOY_TERMINAL_RECORDED=0
|
||||
# The API probes permit 305 seconds for cold-start DB bootstrap plus
|
||||
# 20 seconds for readiness. Keep rollout above that bounded budget.
|
||||
ROLLOUT_TIMEOUT_SECONDS=360
|
||||
|
||||
record_rollout_risk() {
|
||||
local message="$1"
|
||||
@@ -2955,7 +2960,24 @@ jobs:
|
||||
echo "⚠️ Rollout risk observed: $message" >&2
|
||||
}
|
||||
|
||||
emit_deploy_terminal_receipt() {
|
||||
local status="$1"
|
||||
local stage="$2"
|
||||
local workload="$3"
|
||||
if [ "$DEPLOY_TERMINAL_RECORDED" = "1" ]; then
|
||||
return 0
|
||||
fi
|
||||
printf '%s\n' \
|
||||
"AWOOOI_DEPLOY_TERMINAL_RECEIPT=status=${status};stage=${stage};workload=${workload};source_revision=${SOURCE_REVISION};workflow_run_id=${WORKFLOW_RUN_ID};verifier_no_write=true;rollback_required=true"
|
||||
DEPLOY_TERMINAL_RECORDED=1
|
||||
}
|
||||
|
||||
emit_rollout_evidence() {
|
||||
local exit_code=$?
|
||||
if [ "$exit_code" -ne 0 ]; then
|
||||
emit_deploy_terminal_receipt \
|
||||
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "unknown"
|
||||
fi
|
||||
if [ -s "$RISK_FILE" ]; then
|
||||
local summary
|
||||
local kubectl_count
|
||||
@@ -2967,6 +2989,7 @@ jobs:
|
||||
echo "AWOOOI_ROLLOUT_RISK=0"
|
||||
fi
|
||||
rm -f "$RISK_FILE"
|
||||
return "$exit_code"
|
||||
}
|
||||
trap emit_rollout_evidence EXIT
|
||||
|
||||
@@ -3097,13 +3120,26 @@ jobs:
|
||||
sleep 5
|
||||
done
|
||||
|
||||
# Two-replica rollouts can legitimately need more than two minutes
|
||||
# before the second replacement is available. Keep this bounded, but
|
||||
# do not abort before the independent boundary verifier can run.
|
||||
$KUBECTL rollout status deployment/awoooi-api -n awoooi-prod --timeout=300s
|
||||
$KUBECTL rollout status deployment/awoooi-web -n awoooi-prod --timeout=300s
|
||||
$KUBECTL rollout status deployment/awoooi-worker -n awoooi-prod --timeout=300s
|
||||
$KUBECTL rollout status deployment/awoooi-ansible-executor-broker -n awoooi-prod --timeout=300s
|
||||
wait_for_rollout() {
|
||||
local workload="$1"
|
||||
DEPLOY_STAGE="rollout_${workload}"
|
||||
if ! $KUBECTL rollout status "deployment/${workload}" \
|
||||
-n awoooi-prod --timeout="${ROLLOUT_TIMEOUT_SECONDS}s"; then
|
||||
record_rollout_risk \
|
||||
"deployment_rollout_timeout workload=${workload} timeout_seconds=${ROLLOUT_TIMEOUT_SECONDS}"
|
||||
emit_deploy_terminal_receipt \
|
||||
"blocked_with_safe_next_action" "$DEPLOY_STAGE" "$workload"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Two-replica rollouts can legitimately need more than two minutes.
|
||||
# Keep convergence bounded by the declared probe budget, and leave a
|
||||
# terminal receipt before the independent boundary verifier can run.
|
||||
wait_for_rollout awoooi-api || exit 1
|
||||
wait_for_rollout awoooi-web || exit 1
|
||||
wait_for_rollout awoooi-worker || exit 1
|
||||
wait_for_rollout awoooi-ansible-executor-broker || exit 1
|
||||
echo "✅ 部署完成"
|
||||
|
||||
# AIA-P0-011: production must prove that the public API cannot mutate
|
||||
@@ -3132,6 +3168,7 @@ jobs:
|
||||
;;
|
||||
esac
|
||||
}
|
||||
DEPLOY_STAGE="executor_boundary_collect_live_identity_and_policy"
|
||||
echo "executor_boundary_stage=collect_live_identity_and_policy"
|
||||
API_SA=$($KUBECTL get deployment/awoooi-api -n awoooi-prod \
|
||||
-o jsonpath='{.spec.template.spec.serviceAccountName}')
|
||||
@@ -3287,6 +3324,7 @@ jobs:
|
||||
test "$LEGACY_EGRESS_RULE_COUNT" = "0" || {
|
||||
echo "❌ legacy namespace policy still creates an allow rule"; exit 1;
|
||||
}
|
||||
DEPLOY_STAGE="executor_boundary_static_identity_and_policy_verified"
|
||||
echo "executor_boundary_stage=static_identity_and_policy_verified"
|
||||
|
||||
verify_ssh_denied() {
|
||||
@@ -3368,6 +3406,7 @@ jobs:
|
||||
verify_broker_ssh_allowed || {
|
||||
echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1;
|
||||
}
|
||||
DEPLOY_STAGE="executor_boundary_live_socket_boundary_verified"
|
||||
echo "executor_boundary_stage=live_socket_boundary_verified"
|
||||
|
||||
read_workload_db_identity_once() {
|
||||
@@ -4206,6 +4245,7 @@ jobs:
|
||||
)
|
||||
PY
|
||||
echo "AWOOOI_SECURITY_BOUNDARY_VERIFIED=1"
|
||||
DEPLOY_TERMINAL_RECORDED=1
|
||||
ARGOCD_WAIT
|
||||
ROLLOUT_EXIT=${PIPESTATUS[0]}
|
||||
set -e
|
||||
@@ -4222,12 +4262,19 @@ jobs:
|
||||
fi
|
||||
|
||||
SECURITY_BOUNDARY_VERIFIED="0"
|
||||
DEPLOY_TERMINAL_RECEIPT=$(grep '^AWOOOI_DEPLOY_TERMINAL_RECEIPT=' \
|
||||
"$ROLLOUT_LOG" | tail -1 | \
|
||||
sed 's/^AWOOOI_DEPLOY_TERMINAL_RECEIPT=//' || true)
|
||||
if grep -q '^AWOOOI_SECURITY_BOUNDARY_VERIFIED=1$' "$ROLLOUT_LOG"; then
|
||||
SECURITY_BOUNDARY_VERIFIED="1"
|
||||
fi
|
||||
if [ "$SECURITY_BOUNDARY_VERIFIED" != "1" ]; then
|
||||
restore_deploy_controls || true
|
||||
echo "❌ executor security boundary did not reach its terminal verifier"
|
||||
if [ -n "$DEPLOY_TERMINAL_RECEIPT" ]; then
|
||||
echo "❌ deploy verifier terminal receipt: ${DEPLOY_TERMINAL_RECEIPT}"
|
||||
else
|
||||
echo "❌ executor security boundary did not reach its terminal verifier"
|
||||
fi
|
||||
if [ "$ROLLOUT_EXIT" -eq 0 ]; then
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -224,16 +224,52 @@ def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None:
|
||||
assert "workload DB identity/budget remains in progress" in workflow
|
||||
|
||||
|
||||
def test_cd_allows_bounded_rollout_convergence_before_boundary_verifier() -> None:
|
||||
def test_cd_bounded_rollout_budget_and_failure_terminal_receipt() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text()
|
||||
rollout_gate = workflow.split("# Two-replica rollouts", 1)[1].split(
|
||||
"# AIA-P0-011:", 1
|
||||
)[0]
|
||||
api_deployment = next(
|
||||
document
|
||||
for document in yaml.safe_load_all(
|
||||
(repo_root / "k8s/awoooi-prod/06-deployment-api.yaml").read_text()
|
||||
)
|
||||
if document
|
||||
and document.get("kind") == "Deployment"
|
||||
and document.get("metadata", {}).get("name") == "awoooi-api"
|
||||
)
|
||||
api_container = next(
|
||||
container
|
||||
for container in api_deployment["spec"]["template"]["spec"]["containers"]
|
||||
if container["name"] == "api"
|
||||
)
|
||||
startup_probe = api_container["startupProbe"]
|
||||
readiness_probe = api_container["readinessProbe"]
|
||||
startup_budget_seconds = (
|
||||
startup_probe["initialDelaySeconds"]
|
||||
+ startup_probe["periodSeconds"] * startup_probe["failureThreshold"]
|
||||
)
|
||||
readiness_budget_seconds = (
|
||||
readiness_probe["initialDelaySeconds"]
|
||||
+ readiness_probe["periodSeconds"] * readiness_probe["failureThreshold"]
|
||||
)
|
||||
timeout_match = re.search(r"^\s*ROLLOUT_TIMEOUT_SECONDS=(\d+)$", workflow, re.M)
|
||||
|
||||
assert rollout_gate.count("--timeout=300s") == 4
|
||||
assert "--timeout=120s" not in rollout_gate
|
||||
assert "independent boundary verifier" in rollout_gate
|
||||
assert timeout_match is not None
|
||||
assert int(timeout_match.group(1)) >= (
|
||||
startup_budget_seconds + readiness_budget_seconds + 30
|
||||
)
|
||||
pre_boundary_rollout = workflow.split(
|
||||
'echo "executor_boundary_stage=collect_live_identity_and_policy"', 1
|
||||
)[0]
|
||||
assert "rollout status deployment/awoooi-api" not in pre_boundary_rollout
|
||||
assert pre_boundary_rollout.count("wait_for_rollout awoooi-") == 4
|
||||
assert "independent boundary verifier" in pre_boundary_rollout
|
||||
assert "AWOOOI_DEPLOY_TERMINAL_RECEIPT=" in workflow
|
||||
assert "status=${status};stage=${stage};workload=${workload}" in workflow
|
||||
assert "source_revision=${SOURCE_REVISION}" in workflow
|
||||
assert "workflow_run_id=${WORKFLOW_RUN_ID}" in workflow
|
||||
assert "verifier_no_write=true" in workflow
|
||||
assert "rollback_required=true" in workflow
|
||||
assert "deploy verifier terminal receipt:" in workflow
|
||||
|
||||
|
||||
def test_broker_probe_accepts_partial_live_route_evidence(tmp_path: Path) -> None:
|
||||
|
||||
Reference in New Issue
Block a user