feat(automation): harden D037 runtime closure receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s

This commit is contained in:
ogt
2026-07-14 13:43:38 +08:00
parent 3047b8d045
commit a6307e785d
31 changed files with 2616 additions and 369 deletions

View File

@@ -77,6 +77,14 @@ workload_required_connection_budget() {
esac
}
workload_representative_probe_count() {
# Capacity is proven by the PostgreSQL role limit and production HTTP
# concurrency. The isolated pod opens one representative connection for
# role, privilege, and query verification without consuming the live
# workload's entire role budget.
printf '1'
}
workload_deployment() {
case "$1" in
api) printf 'awoooi-api|api' ;;
@@ -222,6 +230,7 @@ preflight_workload_once() {
secret="$2"
connection_limit="$3"
required_connection_budget="$4"
representative_probe_count="$5"
deployment_spec="$(workload_deployment "$workload")"
deployment="${deployment_spec%%|*}"
container="${deployment_spec#*|}"
@@ -262,7 +271,9 @@ spec:
key: DATABASE_URL
- name: EXPECTED_CONNECTION_LIMIT
value: "${connection_limit}"
- name: REQUIRED_CONCURRENT_CONNECTIONS
- name: REPRESENTATIVE_CONNECTION_PROBE_COUNT
value: "${representative_probe_count}"
- name: REQUIRED_CONNECTION_BUDGET
value: "${required_connection_budget}"
- name: WORKLOAD_ID
value: "${workload}"
@@ -287,7 +298,15 @@ spec:
async with engine.connect() as conn:
identity = (
await conn.execute(text("""
SELECT current_user AS role_name, rolconnlimit
SELECT
current_user AS role_name,
rolconnlimit,
(
SELECT count(*)
FROM pg_stat_activity
WHERE usename = current_user
AND pid <> pg_backend_pid()
) AS active_role_connection_count
FROM pg_roles WHERE rolname = current_user
"""))
).mappings().one()
@@ -333,39 +352,38 @@ spec:
AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE'))
)
"""))).scalar_one())
required_concurrency = int(
os.environ["REQUIRED_CONCURRENT_CONNECTIONS"]
)
existing_role_connection_count = int((
await conn.execute(text("""
SELECT count(*)
FROM pg_stat_activity
WHERE usename = current_user
"""))
).scalar_one())
additional_probe_count = max(
0,
required_concurrency
- existing_role_connection_count,
)
representative_probe_count = int(
os.environ["REPRESENTATIVE_CONNECTION_PROBE_COUNT"]
)
if representative_probe_count < 1:
raise RuntimeError("representative_probe_count_invalid")
async def probe_connection() -> None:
async with engine.connect() as probe:
await probe.execute(
text("SELECT pg_sleep(0.25)")
)
async def probe_connection() -> None:
async with engine.connect() as probe:
await probe.execute(text("SELECT pg_sleep(0.25)"))
await asyncio.gather(*(
probe_connection()
for _ in range(additional_probe_count)
))
await asyncio.gather(*(
probe_connection()
for _ in range(representative_probe_count)
))
finally:
await engine.dispose()
expected_limit = int(os.environ["EXPECTED_CONNECTION_LIMIT"])
required_budget = int(os.environ["REQUIRED_CONNECTION_BUDGET"])
observed_limit = int(identity["rolconnlimit"])
active_role_connection_count = int(
identity["active_role_connection_count"]
)
available_connection_headroom = (
observed_limit - active_role_connection_count
)
if observed_limit != expected_limit:
raise RuntimeError("connection_limit_mismatch")
if observed_limit < required_budget:
raise RuntimeError("connection_budget_below_required")
if available_connection_headroom < required_budget:
raise RuntimeError("connection_headroom_below_required")
if table_gap_count or sequence_gap_count:
raise RuntimeError("database_privilege_gap")
print(json.dumps({
@@ -375,12 +393,11 @@ spec:
str(identity["role_name"]).encode("utf-8")
).hexdigest(),
"connection_limit": observed_limit,
"concurrent_connection_probe_count": required_concurrency,
"existing_role_connection_count": (
existing_role_connection_count
),
"additional_connection_probe_count": (
additional_probe_count
"required_connection_budget": required_budget,
"active_role_connection_count": active_role_connection_count,
"available_connection_headroom": available_connection_headroom,
"representative_connection_probe_count": (
representative_probe_count
),
"representative_read_count": 3,
"table_privilege_gap_count": table_gap_count,
@@ -474,6 +491,7 @@ for workload in "${selected_workloads[@]}"; do
secret="$(workload_secret "$workload")"
connection_limit="$(workload_connection_limit "$workload")"
required_connection_budget="$(workload_required_connection_budget "$workload")"
representative_probe_count="$(workload_representative_probe_count "$workload")"
secret_present=false
if kube get secret "$secret" -n "$NAMESPACE" >/dev/null 2>&1; then
secret_present=true
@@ -507,11 +525,11 @@ for workload in "${selected_workloads[@]}"; do
else
ready=false
fi
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget"
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget representative_probe_count=$representative_probe_count"
if [ "$MODE" = "apply" ] || [ "$MODE" = "verify" ]; then
preflight_workload "$workload" "$secret" "$connection_limit" \
"$required_connection_budget"
"$required_connection_budget" "$representative_probe_count"
fi
done