From d4600fbcf836e34552f6fbbf2c5632bf227a5428 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 17 Jul 2026 09:37:05 +0800 Subject: [PATCH] fix(cd): stabilize ready-pod SSH boundary probe --- .gitea/workflows/cd.yaml | 58 +++++++++++++++++-- .../test_executor_network_policy_boundary.py | 22 +++++++ 2 files changed, 75 insertions(+), 5 deletions(-) diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 2c6cc9286..06ef78af7 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3695,9 +3695,34 @@ jobs: verify_ssh_denied() { workload="$1" container="$2" - $KUBECTL exec -i "deployment/$workload" -n awoooi-prod \ - -c "$container" -- python - \ - 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 <<'PY' + last_failure="workload_not_ready" + for attempt in 1 2 3; do + ready_pod_ref=$( + $KUBECTL get pods -n awoooi-prod -l "app=$workload" \ + --field-selector=status.phase=Running \ + --sort-by=.metadata.creationTimestamp \ + -o name 2>/dev/null | tail -n 1 + ) || ready_pod_ref="" + if [ -z "$ready_pod_ref" ]; then + last_failure="running_pod_unavailable" + elif ! $KUBECTL wait --for=condition=Ready "$ready_pod_ref" \ + -n awoooi-prod --timeout=10s >/dev/null 2>&1; then + last_failure="ready_pod_unavailable" + else + container_ready=$( + $KUBECTL get "$ready_pod_ref" -n awoooi-prod \ + -o jsonpath="{.status.containerStatuses[?(@.name=='$container')].ready}" \ + 2>/dev/null + ) || container_ready="" + if [ "$container_ready" != "true" ]; then + last_failure="container_not_ready" + else + set +e + probe_output=$( + $KUBECTL exec -i "$ready_pod_ref" -n awoooi-prod \ + -c "$container" -- python - \ + 192.168.0.110 192.168.0.112 192.168.0.120 192.168.0.121 192.168.0.188 \ + 2>&1 <<'PY' import socket import sys @@ -3713,6 +3738,29 @@ jobs: print("unexpected_ssh_reachable=" + ",".join(reachable)) raise SystemExit(1 if reachable else 0) PY + ) + probe_status=$? + set -e + if [ "$probe_status" = "0" ] && \ + printf '%s\n' "$probe_output" | grep -qx 'unexpected_ssh_reachable='; then + printf '%s\n' "$probe_output" + return 0 + fi + if printf '%s\n' "$probe_output" | \ + grep -Eq '^unexpected_ssh_reachable=.+$'; then + printf '%s\n' "$probe_output" + return 1 + fi + last_failure="kubectl_exec_transport_failed" + fi + fi + echo "ssh_denial_probe_retry workload=$workload attempt=$attempt reason=$last_failure" >&2 + if [ "$attempt" != "3" ]; then + sleep 2 + fi + done + echo "ssh_denial_probe_transport_unavailable workload=$workload reason=$last_failure" >&2 + return 2 } verify_broker_ssh_allowed() { $KUBECTL exec -i deployment/awoooi-ansible-executor-broker \ @@ -3763,10 +3811,10 @@ jobs: PY } verify_ssh_denied awoooi-api api || { - echo "❌ API can still establish host SSH connections"; exit 1; + echo "❌ API SSH-denial verifier did not prove the boundary"; exit 1; } verify_ssh_denied awoooi-worker worker || { - echo "❌ signal worker can still establish host SSH connections"; exit 1; + echo "❌ signal worker SSH-denial verifier did not prove the boundary"; exit 1; } verify_broker_ssh_allowed || { echo "❌ execution broker has no live route evidence for allowlisted SSH endpoints"; exit 1; diff --git a/apps/api/tests/test_executor_network_policy_boundary.py b/apps/api/tests/test_executor_network_policy_boundary.py index 027581780..1a643eba3 100644 --- a/apps/api/tests/test_executor_network_policy_boundary.py +++ b/apps/api/tests/test_executor_network_policy_boundary.py @@ -198,6 +198,28 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None: assert "ssh-mcp-key known_hosts 更新失敗，停止部署" in workflow +def test_cd_ssh_denial_probe_targets_ready_pod_and_classifies_transport() -> None: + repo_root = Path(__file__).resolve().parents[3] + workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text() + probe = workflow.split("verify_ssh_denied() {", 1)[1].split( + "verify_broker_ssh_allowed() {", 1 + )[0] + + assert "for attempt in 1 2 3; do" in probe + assert "--field-selector=status.phase=Running" in probe + assert "--sort-by=.metadata.creationTimestamp" in probe + assert '--for=condition=Ready "$ready_pod_ref"' in probe + assert "status.containerStatuses" in probe + assert '$KUBECTL exec -i "$ready_pod_ref"' in probe + assert '$KUBECTL exec -i "deployment/$workload"' not in probe + assert "probe_status=$?" in probe + assert "grep -Eq '^unexpected_ssh_reachable=.+$'" in probe + assert 'last_failure="kubectl_exec_transport_failed"' in probe + assert "ssh_denial_probe_transport_unavailable" in probe + assert "API can still establish host SSH connections" not in workflow + assert "signal worker can still establish host SSH connections" not in workflow + + def test_cd_concurrency_probe_degrades_to_terminal_false_receipt() -> None: repo_root = Path(__file__).resolve().parents[3] workflow = (repo_root / ".gitea/workflows/cd.yaml").read_text()