b10a7d150a
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m42s
CD Pipeline / build-and-deploy (push) Failing after 11m22s
CD Pipeline / post-deploy-checks (push) Has been skipped
101 lines
3.3 KiB
Python
101 lines
3.3 KiB
Python
from __future__ import annotations
|
|
|
|
import importlib.util
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[2]
|
|
SCRIPT = ROOT / "ops" / "runner" / "verify_awoooi_api_liveness_stabilization.py"
|
|
SPEC = importlib.util.spec_from_file_location("api_liveness_verifier", SCRIPT)
|
|
assert SPEC and SPEC.loader
|
|
verifier = importlib.util.module_from_spec(SPEC)
|
|
SPEC.loader.exec_module(verifier)
|
|
|
|
|
|
def _live_deployment() -> dict:
|
|
deployment = verifier._source_deployment()
|
|
deployment["status"] = {
|
|
"replicas": 2,
|
|
"updatedReplicas": 2,
|
|
"availableReplicas": 2,
|
|
"readyReplicas": 2,
|
|
}
|
|
return deployment
|
|
|
|
|
|
def _receipt() -> dict[str, str]:
|
|
return {
|
|
"connection_budget_verified": "true",
|
|
"api_database_null_pool": "true",
|
|
"api_representative_db_preflight_passed": "true",
|
|
"api_http_concurrency_probe_passed": "true",
|
|
"api_db_connection_limit": "12",
|
|
"api_required_connection_budget": "8",
|
|
"api_available_connection_headroom": "11",
|
|
}
|
|
|
|
|
|
def _pods() -> list[dict]:
|
|
return [
|
|
{
|
|
"metadata": {"name": name},
|
|
"status": {"containerStatuses": [{"name": "api", "ready": True}]},
|
|
}
|
|
for name in ("awoooi-api-a", "awoooi-api-b")
|
|
]
|
|
|
|
|
|
def test_source_contract_is_bounded_and_rollback_ready() -> None:
|
|
deployment = verifier._source_deployment()
|
|
assert verifier.evaluate_source_contract(deployment) == []
|
|
edge_config = (verifier.ROOT / "ops/nginx/awoooi.wooo.work.conf").read_text()
|
|
assert verifier.evaluate_routing_contract(
|
|
verifier._source_service(), edge_config
|
|
) == []
|
|
assert verifier.ROLLBACK_COMMAND.endswith("--replicas=1")
|
|
|
|
|
|
def test_live_contract_accepts_two_ready_unsaturated_stable_pods() -> None:
|
|
pods = _pods()
|
|
restarts = {pod["metadata"]["name"]: 0 for pod in pods}
|
|
cpu = {pod["metadata"]["name"]: 410 for pod in pods}
|
|
assert verifier.evaluate_live_state(
|
|
_live_deployment(), _receipt(), pods, restarts, restarts, cpu, 900
|
|
) == []
|
|
|
|
|
|
def test_live_contract_fails_closed_on_budget_restart_or_cpu_regression() -> None:
|
|
pods = _pods()
|
|
receipt = _receipt()
|
|
receipt["connection_budget_verified"] = "false"
|
|
failures = verifier.evaluate_live_state(
|
|
_live_deployment(),
|
|
receipt,
|
|
pods,
|
|
{"awoooi-api-a": 0, "awoooi-api-b": 0},
|
|
{"awoooi-api-a": 1, "awoooi-api-b": 0},
|
|
{"awoooi-api-a": 1000, "awoooi-api-b": 300},
|
|
900,
|
|
)
|
|
assert "receipt_connection_budget_verified_not_verified" in failures
|
|
assert "live_restart_delta:1" in failures
|
|
assert "live_cpu_saturated:awoooi-api-a:1000m" in failures
|
|
|
|
|
|
def test_edge_receipt_requires_a_qualified_production_vantage() -> None:
|
|
assert verifier.evaluate_edge_receipt(
|
|
"192.168.0.120",
|
|
{"vip125": 200, "node120": 200, "node121": 200},
|
|
) == []
|
|
assert verifier.evaluate_edge_receipt(
|
|
None,
|
|
{"vip125": None, "node120": None, "node121": None},
|
|
) == ["edge_production_vantage_unavailable"]
|
|
|
|
|
|
def test_local_vip_failure_is_not_used_as_production_edge_receipt() -> None:
|
|
qualified = {"vip125": 200, "node120": 200, "node121": 200}
|
|
local_unqualified_status = None
|
|
assert local_unqualified_status is None
|
|
assert verifier.evaluate_edge_receipt("192.168.0.110", qualified) == []
|