fix(cd): snapshot live network policy for rollback
This commit is contained in:
@@ -2188,17 +2188,90 @@ jobs:
|
||||
|
||||
# ─── Step 1: Apply NetworkPolicy + ConfigMap + ServiceRegistry ───
|
||||
# NetworkPolicy is intentionally outside kustomize because commonLabels
|
||||
# mutates nested selectors. Keep a source rollback artifact until the
|
||||
# mutates nested selectors. Keep a live rollback artifact until the
|
||||
# live socket and public receipt verifiers have both passed.
|
||||
NETWORK_POLICY_ROLLBACK_FILE="/tmp/awoooi-network-policy-rollback.yaml"
|
||||
git show HEAD^:k8s/awoooi-prod/02-network-policy.yaml \
|
||||
> "$NETWORK_POLICY_ROLLBACK_FILE"
|
||||
# The runner checkout is intentionally single-commit, so HEAD^ is not
|
||||
# available here. Snapshot live policy truth before any mutation.
|
||||
NETWORK_POLICY_ROLLBACK_FILE="/tmp/awoooi-network-policy-rollback.json"
|
||||
NETWORK_POLICY_LIVE_RAW_FILE="/tmp/awoooi-network-policy-live.json"
|
||||
NETWORK_POLICY_CHANGE_ACTIVE="0"
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL get networkpolicy -n awoooi-prod -o json" \
|
||||
> "$NETWORK_POLICY_LIVE_RAW_FILE"
|
||||
python3 - "$NETWORK_POLICY_LIVE_RAW_FILE" \
|
||||
"$NETWORK_POLICY_ROLLBACK_FILE" <<'PY'
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
source_path = Path(sys.argv[1])
|
||||
rollback_path = Path(sys.argv[2])
|
||||
payload = json.loads(source_path.read_text(encoding="utf-8"))
|
||||
items = payload.get("items") or []
|
||||
if not items:
|
||||
raise SystemExit("live NetworkPolicy snapshot is empty")
|
||||
|
||||
sanitized = []
|
||||
for item in items:
|
||||
metadata = item.get("metadata") or {}
|
||||
clean_metadata = {
|
||||
"name": metadata["name"],
|
||||
"namespace": metadata.get("namespace", "awoooi-prod"),
|
||||
}
|
||||
if metadata.get("labels"):
|
||||
clean_metadata["labels"] = metadata["labels"]
|
||||
annotations = dict(metadata.get("annotations") or {})
|
||||
annotations.pop(
|
||||
"kubectl.kubernetes.io/last-applied-configuration",
|
||||
None,
|
||||
)
|
||||
if annotations:
|
||||
clean_metadata["annotations"] = annotations
|
||||
sanitized.append(
|
||||
{
|
||||
"apiVersion": item["apiVersion"],
|
||||
"kind": item["kind"],
|
||||
"metadata": clean_metadata,
|
||||
"spec": item["spec"],
|
||||
}
|
||||
)
|
||||
|
||||
rollback_path.write_text(
|
||||
json.dumps(
|
||||
{"apiVersion": "v1", "kind": "List", "items": sanitized},
|
||||
separators=(",", ":"),
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
PY
|
||||
rm -f "$NETWORK_POLICY_LIVE_RAW_FILE"
|
||||
restore_network_policy() {
|
||||
echo "⏪ restoring previous AWOOOI NetworkPolicy contract"
|
||||
cat "$NETWORK_POLICY_ROLLBACK_FILE" | \
|
||||
if cat "$NETWORK_POLICY_ROLLBACK_FILE" | \
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply -f -"; then
|
||||
NETWORK_POLICY_CHANGE_ACTIVE="0"
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
rollback_network_policy_on_exit() {
|
||||
exit_code=$?
|
||||
trap - EXIT
|
||||
if [ "$exit_code" -ne 0 ] \
|
||||
&& [ "$NETWORK_POLICY_CHANGE_ACTIVE" = "1" ]; then
|
||||
echo "⏪ deploy exited before the terminal boundary verifier"
|
||||
set +e
|
||||
restore_network_policy
|
||||
restore_exit=$?
|
||||
set -e
|
||||
if [ "$restore_exit" -ne 0 ]; then
|
||||
echo "❌ live NetworkPolicy rollback failed"
|
||||
fi
|
||||
fi
|
||||
exit "$exit_code"
|
||||
}
|
||||
trap rollback_network_policy_on_exit EXIT
|
||||
cat k8s/awoooi-prod/02-network-policy.yaml | \
|
||||
ssh $SSH_OPTS "wooo@${{ env.K8S_SSH_HOST }}" \
|
||||
"KUBECTL='sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml --server=${{ env.K8S_API_SERVER }}'; \$KUBECTL apply --dry-run=server -f - >/dev/null"
|
||||
@@ -2208,6 +2281,7 @@ jobs:
|
||||
restore_network_policy
|
||||
exit 1
|
||||
fi
|
||||
NETWORK_POLICY_CHANGE_ACTIVE="1"
|
||||
echo "✅ NetworkPolicy 已受控更新"
|
||||
|
||||
cat k8s/awoooi-prod/04-configmap.yaml | \
|
||||
@@ -2775,6 +2849,7 @@ jobs:
|
||||
fi
|
||||
exit "$ROLLOUT_EXIT"
|
||||
fi
|
||||
NETWORK_POLICY_CHANGE_ACTIVE="0"
|
||||
|
||||
ROLLOUT_RISK="0"
|
||||
ROLLOUT_SUMMARY=""
|
||||
|
||||
@@ -70,6 +70,9 @@ def test_cd_applies_rolls_back_and_verifies_network_boundary() -> None:
|
||||
|
||||
assert "02-network-policy.yaml" in workflow
|
||||
assert "NETWORK_POLICY_ROLLBACK_FILE" in workflow
|
||||
assert "get networkpolicy -n awoooi-prod -o json" in workflow
|
||||
assert "rollback_network_policy_on_exit" in workflow
|
||||
assert "git show HEAD^:k8s/awoooi-prod/02-network-policy.yaml" not in workflow
|
||||
assert "allow-ansible-executor-broker-ssh-egress" in workflow
|
||||
assert "api_ssh_egress_denied=true" in workflow
|
||||
assert "worker_ssh_egress_denied=true" in workflow
|
||||
|
||||
Reference in New Issue
Block a user