b10a7d150a
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m42s
CD Pipeline / build-and-deploy (push) Failing after 11m22s
CD Pipeline / post-deploy-checks (push) Has been skipped
384 lines
14 KiB
Python
384 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Fail-closed verifier for the bounded AWOOOI API two-replica stabilization."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import subprocess
|
|
import time
|
|
import urllib.error
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
MANIFEST = ROOT / "k8s" / "awoooi-prod" / "06-deployment-api.yaml"
|
|
NAMESPACE = "awoooi-prod"
|
|
DEPLOYMENT = "awoooi-api"
|
|
BOUNDARY_RECEIPT = "awoooi-executor-boundary-verification"
|
|
PUBLIC_LIVENESS_URL = "https://awoooi.wooo.work/api/v1/health/live"
|
|
EDGE_VIP_LIVENESS_URL = "http://192.168.0.125:32334/api/v1/health/live"
|
|
EDGE_LIVENESS_URLS = {
|
|
"vip125": EDGE_VIP_LIVENESS_URL,
|
|
"node120": "http://192.168.0.120:32334/api/v1/health/live",
|
|
"node121": "http://192.168.0.121:32334/api/v1/health/live",
|
|
}
|
|
EDGE_VANTAGE_HOSTS = ("192.168.0.120", "192.168.0.110")
|
|
ROLLBACK_COMMAND = (
|
|
"kubectl -n awoooi-prod scale deployment/awoooi-api --replicas=1"
|
|
)
|
|
|
|
|
|
def _source_documents() -> list[dict[str, Any]]:
|
|
"""Render the manifest with kubectl's client parser; never contact the API."""
|
|
completed = subprocess.run(
|
|
["kubectl", "create", "--dry-run=client", "-f", str(MANIFEST), "-o", "json"],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
)
|
|
decoder = json.JSONDecoder()
|
|
remaining = completed.stdout
|
|
documents: list[dict[str, Any]] = []
|
|
while remaining.strip():
|
|
document, offset = decoder.raw_decode(remaining.lstrip())
|
|
remaining = remaining.lstrip()[offset:]
|
|
documents.append(document)
|
|
return documents
|
|
|
|
|
|
def _source_deployment() -> dict[str, Any]:
|
|
return next(
|
|
document
|
|
for document in _source_documents()
|
|
if document.get("kind") == "Deployment"
|
|
)
|
|
|
|
|
|
def _source_service() -> dict[str, Any]:
|
|
return next(
|
|
document
|
|
for document in _source_documents()
|
|
if document.get("kind") == "Service"
|
|
)
|
|
|
|
|
|
def _env(container: dict[str, Any]) -> dict[str, str | None]:
|
|
return {entry["name"]: entry.get("value") for entry in container.get("env", [])}
|
|
|
|
|
|
def evaluate_source_contract(deployment: dict[str, Any]) -> list[str]:
|
|
failures: list[str] = []
|
|
spec = deployment.get("spec") or {}
|
|
strategy = (spec.get("strategy") or {}).get("rollingUpdate") or {}
|
|
container = spec["template"]["spec"]["containers"][0]
|
|
env = _env(container)
|
|
|
|
expectations = {
|
|
"replicas": (spec.get("replicas"), 2),
|
|
"max_surge": (strategy.get("maxSurge"), 0),
|
|
"max_unavailable": (strategy.get("maxUnavailable"), 1),
|
|
"cpu_limit": ((container.get("resources") or {}).get("limits", {}).get("cpu"), "1"),
|
|
"memory_limit": ((container.get("resources") or {}).get("limits", {}).get("memory"), "1Gi"),
|
|
"database_pool_size": (env.get("DATABASE_POOL_SIZE"), "1"),
|
|
"database_max_overflow": (env.get("DATABASE_MAX_OVERFLOW"), "0"),
|
|
"database_null_pool": (env.get("DATABASE_NULL_POOL"), "true"),
|
|
"database_bootstrap_ddl_enabled": (
|
|
env.get("DATABASE_BOOTSTRAP_DDL_ENABLED"),
|
|
"false",
|
|
),
|
|
"api_background_tasks": (env.get("AWOOOI_API_BACKGROUND_TASKS_ENABLED"), "false"),
|
|
}
|
|
for name, (actual, expected) in expectations.items():
|
|
if actual != expected:
|
|
failures.append(f"source_{name}_mismatch:{actual!r}!={expected!r}")
|
|
|
|
probes = {
|
|
name: container.get(name) or {}
|
|
for name in ("startupProbe", "livenessProbe", "readinessProbe")
|
|
}
|
|
for name, probe in probes.items():
|
|
expected_path = (
|
|
"/api/v1/health/ready"
|
|
if name == "readinessProbe"
|
|
else "/api/v1/health/live"
|
|
)
|
|
if (probe.get("httpGet") or {}).get("path") != expected_path:
|
|
failures.append(f"source_{name}_not_lightweight")
|
|
return failures
|
|
|
|
|
|
def evaluate_routing_contract(service: dict[str, Any], edge_config: str) -> list[str]:
|
|
failures: list[str] = []
|
|
spec = service.get("spec") or {}
|
|
ports = spec.get("ports") or []
|
|
api_port = next((item for item in ports if item.get("name") == "http"), {})
|
|
if service.get("metadata", {}).get("name") != "awoooi-api-svc":
|
|
failures.append("source_api_service_name_mismatch")
|
|
if spec.get("type") != "NodePort":
|
|
failures.append("source_api_service_not_nodeport")
|
|
if api_port.get("port") != 8000 or api_port.get("targetPort") != 8000:
|
|
failures.append("source_api_service_port_mismatch")
|
|
if api_port.get("nodePort") != 32334:
|
|
failures.append("source_api_nodeport_mismatch")
|
|
if "server 192.168.0.125:32334;" not in edge_config:
|
|
failures.append("source_edge_vip_upstream_mismatch")
|
|
return failures
|
|
|
|
|
|
def evaluate_live_state(
|
|
deployment: dict[str, Any],
|
|
receipt: dict[str, str],
|
|
pods: list[dict[str, Any]],
|
|
restart_before: dict[str, int],
|
|
restart_after: dict[str, int],
|
|
cpu_millicores: dict[str, int],
|
|
max_cpu_millicores: int,
|
|
) -> list[str]:
|
|
failures = evaluate_source_contract(deployment)
|
|
status = deployment.get("status") or {}
|
|
for name, actual in {
|
|
"desired_replicas": status.get("replicas", 0),
|
|
"updated_replicas": status.get("updatedReplicas", 0),
|
|
"available_replicas": status.get("availableReplicas", 0),
|
|
"ready_replicas": status.get("readyReplicas", 0),
|
|
}.items():
|
|
if actual != 2:
|
|
failures.append(f"live_{name}_not_two:{actual}")
|
|
if status.get("unavailableReplicas", 0) not in (0, None):
|
|
failures.append(f"live_unavailable_replicas:{status['unavailableReplicas']}")
|
|
|
|
expected_receipt = {
|
|
"connection_budget_verified": "true",
|
|
"api_database_null_pool": "true",
|
|
"api_representative_db_preflight_passed": "true",
|
|
"api_http_concurrency_probe_passed": "true",
|
|
}
|
|
for key, expected in expected_receipt.items():
|
|
if receipt.get(key) != expected:
|
|
failures.append(f"receipt_{key}_not_verified")
|
|
for key, minimum in {
|
|
"api_db_connection_limit": 12,
|
|
"api_required_connection_budget": 8,
|
|
"api_available_connection_headroom": 8,
|
|
}.items():
|
|
try:
|
|
actual = int(receipt.get(key, ""))
|
|
except ValueError:
|
|
actual = -1
|
|
if actual < minimum:
|
|
failures.append(f"receipt_{key}_below_{minimum}:{actual}")
|
|
|
|
ready_pods = 0
|
|
for pod in pods:
|
|
statuses = (pod.get("status") or {}).get("containerStatuses") or []
|
|
api = next((item for item in statuses if item.get("name") == "api"), None)
|
|
if api and api.get("ready") is True:
|
|
ready_pods += 1
|
|
if ready_pods != 2:
|
|
failures.append(f"live_ready_pod_count_not_two:{ready_pods}")
|
|
|
|
all_names = set(restart_before) | set(restart_after)
|
|
restart_delta = sum(
|
|
max(0, restart_after.get(name, 0) - restart_before.get(name, 0))
|
|
for name in all_names
|
|
)
|
|
if restart_delta:
|
|
failures.append(f"live_restart_delta:{restart_delta}")
|
|
if set(cpu_millicores) != {pod["metadata"]["name"] for pod in pods}:
|
|
failures.append("live_cpu_sample_incomplete")
|
|
for pod_name, cpu in cpu_millicores.items():
|
|
if cpu >= max_cpu_millicores:
|
|
failures.append(f"live_cpu_saturated:{pod_name}:{cpu}m")
|
|
return failures
|
|
|
|
|
|
def _kubectl_json(*args: str) -> dict[str, Any]:
|
|
completed = subprocess.run(
|
|
["kubectl", *args],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
)
|
|
return json.loads(completed.stdout)
|
|
|
|
|
|
def _restart_counts(pods: list[dict[str, Any]]) -> dict[str, int]:
|
|
counts: dict[str, int] = {}
|
|
for pod in pods:
|
|
statuses = (pod.get("status") or {}).get("containerStatuses") or []
|
|
api = next((item for item in statuses if item.get("name") == "api"), None)
|
|
counts[pod["metadata"]["name"]] = int((api or {}).get("restartCount", 0))
|
|
return counts
|
|
|
|
|
|
def _cpu_samples() -> dict[str, int]:
|
|
completed = subprocess.run(
|
|
[
|
|
"kubectl", "-n", NAMESPACE, "top", "pod", "-l", f"app={DEPLOYMENT}",
|
|
"--containers", "--no-headers",
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=20,
|
|
)
|
|
samples: dict[str, int] = {}
|
|
for line in completed.stdout.splitlines():
|
|
fields = line.split()
|
|
if len(fields) >= 3 and fields[1] == "api" and fields[2].endswith("m"):
|
|
samples[fields[0]] = int(fields[2][:-1])
|
|
return samples
|
|
|
|
|
|
def _http_status(url: str) -> int | None:
|
|
try:
|
|
with urllib.request.urlopen(url, timeout=8) as response:
|
|
return int(response.status)
|
|
except urllib.error.HTTPError as exc:
|
|
return int(exc.code)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def evaluate_edge_receipt(
|
|
vantage_host: str | None,
|
|
statuses: dict[str, int | None],
|
|
) -> list[str]:
|
|
if not vantage_host:
|
|
return ["edge_production_vantage_unavailable"]
|
|
failures: list[str] = []
|
|
for endpoint in EDGE_LIVENESS_URLS:
|
|
if statuses.get(endpoint) != 200:
|
|
failures.append(
|
|
f"edge_{endpoint}_liveness_unavailable:{statuses.get(endpoint)}"
|
|
)
|
|
return failures
|
|
|
|
|
|
def _edge_status_from_production_vantage() -> tuple[str | None, dict[str, int | None]]:
|
|
for host in EDGE_VANTAGE_HOSTS:
|
|
statuses: dict[str, int | None] = {}
|
|
vantage_reachable = True
|
|
for endpoint, url in EDGE_LIVENESS_URLS.items():
|
|
completed = subprocess.run(
|
|
[
|
|
"ssh",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "ConnectTimeout=5",
|
|
"-o", "ConnectionAttempts=1",
|
|
"-o", "ServerAliveInterval=3",
|
|
"-o", "ServerAliveCountMax=1",
|
|
f"wooo@{host}",
|
|
(
|
|
"curl -sS -o /dev/null -w '%{http_code}' "
|
|
f"--connect-timeout 3 --max-time 6 {url}"
|
|
),
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=12,
|
|
)
|
|
if completed.returncode == 255:
|
|
vantage_reachable = False
|
|
break
|
|
try:
|
|
statuses[endpoint] = int(completed.stdout.strip())
|
|
except ValueError:
|
|
statuses[endpoint] = None
|
|
if vantage_reachable:
|
|
return host, statuses
|
|
return None, {}
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--live", action="store_true")
|
|
parser.add_argument("--sample-seconds", type=int, default=35)
|
|
parser.add_argument("--max-pod-cpu-millicores", type=int, default=900)
|
|
args = parser.parse_args()
|
|
|
|
source = _source_deployment()
|
|
failures = evaluate_source_contract(source)
|
|
edge_config = (ROOT / "ops" / "nginx" / "awoooi.wooo.work.conf").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
failures.extend(evaluate_routing_contract(_source_service(), edge_config))
|
|
result: dict[str, Any] = {
|
|
"schema_version": "awoooi_api_liveness_stabilization_verifier_v1",
|
|
"mode": "live" if args.live else "source",
|
|
"rollback": {"target_replicas": 1, "command": ROLLBACK_COMMAND},
|
|
"routing_truth": {
|
|
"kubernetes_service": "awoooi-api-svc:8000",
|
|
"node_port": 32334,
|
|
"edge_vip": "192.168.0.125:32334",
|
|
"public_liveness": PUBLIC_LIVENESS_URL,
|
|
"edge_probe_vantage": "production_host_120_then_110",
|
|
"local_edge_probe_classification": "vantage_unqualified",
|
|
"edge_reachability_is_separate_from_replica_rollback": True,
|
|
},
|
|
}
|
|
try:
|
|
if args.live:
|
|
deployment_before = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "deployment", DEPLOYMENT, "-o", "json"
|
|
)
|
|
receipt = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "configmap", BOUNDARY_RECEIPT, "-o", "json"
|
|
).get("data") or {}
|
|
pods_before = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "pods", "-l", f"app={DEPLOYMENT}", "-o", "json"
|
|
).get("items") or []
|
|
restart_before = _restart_counts(pods_before)
|
|
time.sleep(max(0, args.sample_seconds))
|
|
deployment_after = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "deployment", DEPLOYMENT, "-o", "json"
|
|
)
|
|
pods_after = _kubectl_json(
|
|
"-n", NAMESPACE, "get", "pods", "-l", f"app={DEPLOYMENT}", "-o", "json"
|
|
).get("items") or []
|
|
failures = evaluate_live_state(
|
|
deployment_after,
|
|
receipt,
|
|
pods_after,
|
|
restart_before,
|
|
_restart_counts(pods_after),
|
|
_cpu_samples(),
|
|
args.max_pod_cpu_millicores,
|
|
)
|
|
result["sample_seconds"] = max(0, args.sample_seconds)
|
|
desired_before = (deployment_before.get("spec") or {}).get("replicas")
|
|
desired_after = (deployment_after.get("spec") or {}).get("replicas")
|
|
result["desired_replicas_before"] = desired_before
|
|
result["desired_replicas_after"] = desired_after
|
|
if desired_before != desired_after:
|
|
failures.append(
|
|
f"live_desired_replica_drift:{desired_before}->{desired_after}"
|
|
)
|
|
public_status = _http_status(PUBLIC_LIVENESS_URL)
|
|
local_edge_vip_status = _http_status(EDGE_VIP_LIVENESS_URL)
|
|
edge_vantage, edge_statuses = _edge_status_from_production_vantage()
|
|
result["routing_truth"]["public_status"] = public_status
|
|
result["routing_truth"]["local_edge_vip_status"] = local_edge_vip_status
|
|
result["routing_truth"]["qualified_vantage_host"] = edge_vantage
|
|
result["routing_truth"]["qualified_edge_statuses"] = edge_statuses
|
|
if public_status != 200:
|
|
failures.append(f"public_liveness_unavailable:{public_status}")
|
|
failures.extend(evaluate_edge_receipt(edge_vantage, edge_statuses))
|
|
except Exception as exc: # fail closed without leaking command output
|
|
failures.append(f"verifier_error:{type(exc).__name__}")
|
|
|
|
result["status"] = "verified" if not failures else "blocked_with_safe_rollback"
|
|
result["failures"] = failures
|
|
print(json.dumps(result, ensure_ascii=False, sort_keys=True))
|
|
return 0 if not failures else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|