fix(cd): snapshot live network policy for rollback

This commit is contained in:
ogt
2026-07-11 01:56:38 +08:00
parent 3eb5d9188c
commit 565edcdfb3
2 changed files with 84 additions and 6 deletions

View File

@@ -2188,17 +2188,90 @@ jobs:
# ─── Step 1: Apply NetworkPolicy + ConfigMap + ServiceRegistry ───
# NetworkPolicy is intentionally outside kustomize because commonLabels
# mutates nested selectors. Keep a source rollback artifact until the
# mutates nested selectors. Keep a live rollback artifact until the
# live socket and public receipt verifiers have both passed.
NETWORK_POLICY_ROLLBACK_FILE="/tmp/awoooi-network-policy-rollback.yaml"
git show HEAD^:k8s/awoooi-prod/02-network-policy.yaml \
> "$NETWORK_POLICY_ROLLBACK_FILE"
# The runner checkout is intentionally single-commit, so HEAD^ is not
# available here. Snapshot live policy truth before any mutation.
NETWORK_POLICY_ROLLBACK_FILE="/tmp/awoooi-network-policy-rollback.json"
NETWORK_POLICY_LIVE_RAW_FILE="/tmp/awoooi-network-policy-live.json"
NETWORK_POLICY_CHANGE_ACTIVE="0"
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get networkpolicy -n awoooi-prod -o json" \
> "$NETWORK_POLICY_LIVE_RAW_FILE"
python3 - "$NETWORK_POLICY_LIVE_RAW_FILE" \
"$NETWORK_POLICY_ROLLBACK_FILE" <<'PY'
import json
import sys
from pathlib import Path
source_path = Path(sys.argv[1])
rollback_path = Path(sys.argv[2])
payload = json.loads(source_path.read_text(encoding="utf-8"))
items = payload.get("items") or []
if not items:
raise SystemExit("live NetworkPolicy snapshot is empty")
sanitized = []
for item in items:
metadata = item.get("metadata") or {}
clean_metadata = {
"name": metadata["name"],
"namespace": metadata.get("namespace", "awoooi-prod"),
}
if metadata.get("labels"):
clean_metadata["labels"] = metadata["labels"]
annotations = dict(metadata.get("annotations") or {})
annotations.pop(
"kubectl.kubernetes.io/last-applied-configuration",
None,
)
if annotations:
clean_metadata["annotations"] = annotations
sanitized.append(
{
"apiVersion": item["apiVersion"],
"kind": item["kind"],
"metadata": clean_metadata,
"spec": item["spec"],
}
)
rollback_path.write_text(
json.dumps(
{"apiVersion": "v1", "kind": "List", "items": sanitized},
separators=(",", ":"),
),
encoding="utf-8",
)
PY
rm -f "$NETWORK_POLICY_LIVE_RAW_FILE"
restore_network_policy() {
echo "⏪ restoring previous AWOOOI NetworkPolicy contract"
cat "$NETWORK_POLICY_ROLLBACK_FILE" | \
if cat "$NETWORK_POLICY_ROLLBACK_FILE" | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"; then
NETWORK_POLICY_CHANGE_ACTIVE="0"
return 0
fi
return 1
}
rollback_network_policy_on_exit() {
exit_code=$?
trap - EXIT
if [ "$exit_code" -ne 0 ] \
&& [ "$NETWORK_POLICY_CHANGE_ACTIVE" = "1" ]; then
echo "⏪ deploy exited before the terminal boundary verifier"
set +e
restore_network_policy
restore_exit=$?
set -e
if [ "$restore_exit" -ne 0 ]; then
echo "❌ live NetworkPolicy rollback failed"
fi
fi
exit "$exit_code"
}
trap rollback_network_policy_on_exit EXIT
cat k8s/awoooi-prod/02-network-policy.yaml | \
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply --dry-run=server -f - >/dev/null"
@@ -2208,6 +2281,7 @@ jobs:
restore_network_policy
exit 1
fi
NETWORK_POLICY_CHANGE_ACTIVE="1"
echo "✅ NetworkPolicy 已受控更新"
cat k8s/awoooi-prod/04-configmap.yaml | \
@@ -2775,6 +2849,7 @@ jobs:
fi
exit "$ROLLOUT_EXIT"
fi
NETWORK_POLICY_CHANGE_ACTIVE="0"
ROLLOUT_RISK="0"
ROLLOUT_SUMMARY=""

View File

@@ -70,6 +70,9 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
assert "02-network-policy.yaml" in workflow
assert "NETWORK_POLICY_ROLLBACK_FILE" in workflow
assert "get networkpolicy -n awoooi-prod -o json" in workflow
assert "rollback_network_policy_on_exit" in workflow
assert "git show HEAD^:k8s/awoooi-prod/02-network-policy.yaml" not in workflow
assert "allow-ansible-executor-broker-ssh-egress" in workflow
assert "api_ssh_egress_denied=true" in workflow
assert "worker_ssh_egress_denied=true" in workflow