fix(cd): stabilize ready-pod SSH boundary probe
This commit is contained in:
@@ -3695,9 +3695,34 @@ jobs:
|
||||
verify_ssh_denied() {
|
||||
workload="$1"
|
||||
container="$2"
|
||||
$KUBECTL exec -i "deployment/$workload" -n awoooi-prod \
|
||||
-c "$container" -- python - \
|
||||
192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 <<'PY'
|
||||
last_failure="workload_not_ready"
|
||||
for attempt in 1 2 3; do
|
||||
ready_pod_ref=$(
|
||||
$KUBECTL get pods -n awoooi-prod -l "app=$workload" \
|
||||
--field-selector=status.phase=Running \
|
||||
--sort-by=.metadata.creationTimestamp \
|
||||
-o name 2>/dev/null | tail -n 1
|
||||
) || ready_pod_ref=""
|
||||
if [ -z "$ready_pod_ref" ]; then
|
||||
last_failure="running_pod_unavailable"
|
||||
elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \
|
||||
-n awoooi-prod --timeout=10s >/dev/null 2>&1; then
|
||||
last_failure="ready_pod_unavailable"
|
||||
else
|
||||
container_ready=$(
|
||||
$KUBECTL get "$ready_pod_ref" -n awoooi-prod \
|
||||
-o jsonpath="{.status.containerStatuses[?(@.name=='$container')].ready}" \
|
||||
2>/dev/null
|
||||
) || container_ready=""
|
||||
if [ "$container_ready" != "true" ]; then
|
||||
last_failure="container_not_ready"
|
||||
else
|
||||
set +e
|
||||
probe_output=$(
|
||||
$KUBECTL exec -i "$ready_pod_ref" -n awoooi-prod \
|
||||
-c "$container" -- python - \
|
||||
192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \
|
||||
2>&1 <<'PY'
|
||||
import socket
|
||||
import sys
|
||||
|
||||
@@ -3713,6 +3738,29 @@ jobs:
|
||||
print("unexpected_ssh_reachable=" + ",".join(reachable))
|
||||
raise SystemExit(1 if reachable else 0)
|
||||
PY
|
||||
)
|
||||
probe_status=$?
|
||||
set -e
|
||||
if [ "$probe_status" = "0" ] && \
|
||||
printf '%s\n' "$probe_output" | grep -qx 'unexpected_ssh_reachable='; then
|
||||
printf '%s\n' "$probe_output"
|
||||
return 0
|
||||
fi
|
||||
if printf '%s\n' "$probe_output" | \
|
||||
grep -Eq '^unexpected_ssh_reachable=.+$'; then
|
||||
printf '%s\n' "$probe_output"
|
||||
return 1
|
||||
fi
|
||||
last_failure="kubectl_exec_transport_failed"
|
||||
fi
|
||||
fi
|
||||
echo "ssh_denial_probe_retry workload=$workload attempt=$attempt reason=$last_failure" >&2
|
||||
if [ "$attempt" != "3" ]; then
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
echo "ssh_denial_probe_transport_unavailable workload=$workload reason=$last_failure" >&2
|
||||
return 2
|
||||
}
|
||||
verify_broker_ssh_allowed() {
|
||||
$KUBECTL exec -i deployment/awoooi-ansible-executor-broker \
|
||||
@@ -3763,10 +3811,10 @@ jobs:
|
||||
PY
|
||||
}
|
||||
verify_ssh_denied awoooi-api api || {
|
||||
echo "❌ API can still establish host SSH connections"; exit 1;
|
||||
echo "❌ API SSH-denial verifier did not prove the boundary"; exit 1;
|
||||
}
|
||||
verify_ssh_denied awoooi-worker worker || {
|
||||
echo "❌ signal worker can still establish host SSH connections"; exit 1;
|
||||
echo "❌ signal worker SSH-denial verifier did not prove the boundary"; exit 1;
|
||||
}
|
||||
verify_broker_ssh_allowed || {
|
||||
echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1;
|
||||
|
||||
@@ -198,6 +198,28 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
|
||||
assert "ssh-mcp-key known_hosts 更新失敗,停止部署" in workflow
|
||||
|
||||
|
||||
def test_cd_ssh_denial_probe_targets_ready_pod_and_classifies_transport() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text()
|
||||
probe = workflow.split("verify_ssh_denied() {", 1)[1].split(
|
||||
"verify_broker_ssh_allowed() {", 1
|
||||
)[0]
|
||||
|
||||
assert "for attempt in 1 2 3; do" in probe
|
||||
assert "--field-selector=status.phase=Running" in probe
|
||||
assert "--sort-by=.metadata.creationTimestamp" in probe
|
||||
assert '--for=condition=Ready "$ready_pod_ref"' in probe
|
||||
assert "status.containerStatuses" in probe
|
||||
assert '$KUBECTL exec -i "$ready_pod_ref"' in probe
|
||||
assert '$KUBECTL exec -i "deployment/$workload"' not in probe
|
||||
assert "probe_status=$?" in probe
|
||||
assert "grep -Eq '^unexpected_ssh_reachable=.+$'" in probe
|
||||
assert 'last_failure="kubectl_exec_transport_failed"' in probe
|
||||
assert "ssh_denial_probe_transport_unavailable" in probe
|
||||
assert "API can still establish host SSH connections" not in workflow
|
||||
assert "signal worker can still establish host SSH connections" not in workflow
|
||||
|
||||
|
||||
def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None:
|
||||
repo_root = Path(__file__).resolve().parents[3]
|
||||
workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text()
|
||||
|
||||
Reference in New Issue
Block a user