feat(governance): 新增服務健康缺口矩陣
This commit is contained in:
@@ -71,6 +71,9 @@ from src.services.observability_contract_matrix import (
|
||||
from src.services.ai_provider_route_matrix import (
|
||||
load_latest_ai_provider_route_matrix,
|
||||
)
|
||||
from src.services.service_health_gap_matrix import (
|
||||
load_latest_service_health_gap_matrix,
|
||||
)
|
||||
from src.services.package_supply_chain_inventory import (
|
||||
load_latest_package_supply_chain_inventory,
|
||||
)
|
||||
@@ -598,6 +601,33 @@ async def get_ai_provider_route_matrix() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/service-health-gap-matrix",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得服務健康缺口與過期端點矩陣",
|
||||
description=(
|
||||
"讀取最新已提交的 service health gap matrix;此端點不做 live probe、"
|
||||
"不重啟服務、不修改 endpoint / ConfigMap、不讀 Secret/Redis/DB payload、"
|
||||
"不發通知、不觸發 workflow/deploy/reload/runtime execution。"
|
||||
),
|
||||
)
|
||||
async def get_service_health_gap_matrix() -> dict[str, Any]:
|
||||
"""Return the latest read-only service health gap matrix."""
|
||||
try:
|
||||
return await asyncio.to_thread(load_latest_service_health_gap_matrix)
|
||||
except FileNotFoundError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_404_NOT_FOUND,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except (json.JSONDecodeError, ValueError) as exc:
|
||||
logger.error("service_health_gap_matrix_invalid", error=str(exc))
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_500_INTERNAL_SERVER_ERROR,
|
||||
detail="服務健康缺口矩陣快照無效",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/backup-dr-target-inventory",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
282
apps/api/src/services/service_health_gap_matrix.py
Normal file
282
apps/api/src/services/service_health_gap_matrix.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
Service health gap matrix snapshot.
|
||||
|
||||
Loads the latest committed, read-only service health / stale endpoint gap
|
||||
matrix. This module never restarts services, changes endpoints, probes live
|
||||
systems, reads secrets, sends notifications, triggers workflows, deploys, or
|
||||
executes runtime actions.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from src.services.snapshot_paths import default_evaluations_dir
|
||||
|
||||
_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__))
|
||||
_SNAPSHOT_PATTERN = "service_health_gap_matrix_*.json"
|
||||
_SCHEMA_VERSION = "service_health_gap_matrix_v1"
|
||||
|
||||
|
||||
def load_latest_service_health_gap_matrix(
|
||||
evaluations_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Load the newest committed service health gap matrix snapshot."""
|
||||
directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR
|
||||
candidates = sorted(directory.glob(_SNAPSHOT_PATTERN))
|
||||
if not candidates:
|
||||
raise FileNotFoundError(f"no service health gap matrix snapshots found in {directory}")
|
||||
|
||||
latest = candidates[-1]
|
||||
with latest.open(encoding="utf-8") as handle:
|
||||
payload = json.load(handle)
|
||||
|
||||
if not isinstance(payload, dict):
|
||||
raise ValueError(f"{latest}: expected JSON object")
|
||||
_require_schema(payload, _SCHEMA_VERSION, str(latest))
|
||||
_require_read_only_boundaries(payload, str(latest))
|
||||
_require_operation_boundaries(payload, str(latest))
|
||||
_require_rollup_consistency(payload, str(latest))
|
||||
_require_target_evidence(payload, str(latest))
|
||||
_require_gap_evidence(payload, str(latest))
|
||||
_require_operator_denials(payload, str(latest))
|
||||
_require_no_plaintext_secret_payload_keys(payload, str(latest))
|
||||
return payload
|
||||
|
||||
|
||||
def _require_schema(payload: dict[str, Any], expected: str, label: str) -> None:
|
||||
actual = payload.get("schema_version")
|
||||
if actual != expected:
|
||||
raise ValueError(f"{label}: expected schema_version={expected}, got {actual!r}")
|
||||
|
||||
|
||||
def _require_read_only_boundaries(payload: dict[str, Any], label: str) -> None:
|
||||
program_status = payload.get("program_status") or {}
|
||||
if program_status.get("read_only_mode") is not True:
|
||||
raise ValueError(f"{label}: program_status.read_only_mode must be true")
|
||||
|
||||
approval_boundaries = payload.get("approval_boundaries") or {}
|
||||
allowed = sorted(key for key, value in approval_boundaries.items() if value is not False)
|
||||
if allowed:
|
||||
raise ValueError(f"{label}: approval boundaries must remain false: {allowed}")
|
||||
|
||||
|
||||
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
|
||||
boundaries = payload.get("operation_boundaries") or {}
|
||||
if boundaries.get("read_only_api_allowed") is not True:
|
||||
raise ValueError(f"{label}: read_only_api_allowed must be true")
|
||||
|
||||
blocked_flags = {
|
||||
"service_restart_allowed",
|
||||
"pod_restart_allowed",
|
||||
"host_restart_allowed",
|
||||
"rollout_restart_allowed",
|
||||
"endpoint_change_allowed",
|
||||
"configmap_patch_allowed",
|
||||
"active_probe_allowed",
|
||||
"external_health_probe_allowed",
|
||||
"live_benchmark_allowed",
|
||||
"provider_switch_allowed",
|
||||
"paid_api_call_allowed",
|
||||
"secret_read_allowed",
|
||||
"secret_plaintext_allowed",
|
||||
"notification_send_allowed",
|
||||
"workflow_trigger_allowed",
|
||||
"deploy_trigger_allowed",
|
||||
"reload_trigger_allowed",
|
||||
"runtime_execution_allowed",
|
||||
}
|
||||
allowed = sorted(flag for flag in blocked_flags if boundaries.get(flag) is not False)
|
||||
if allowed:
|
||||
raise ValueError(f"{label}: operation boundaries must remain false: {allowed}")
|
||||
|
||||
|
||||
def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None:
|
||||
targets = payload.get("service_health_targets") or []
|
||||
gaps = payload.get("health_gaps") or []
|
||||
stale_endpoints = payload.get("stale_endpoints") or []
|
||||
rollups = payload.get("rollups") or {}
|
||||
|
||||
if rollups.get("total_targets") != len(targets):
|
||||
raise ValueError(f"{label}: rollups.total_targets must match service_health_targets")
|
||||
if rollups.get("by_kind") != _count_by(targets, "kind"):
|
||||
raise ValueError(f"{label}: rollups.by_kind must match service_health_targets")
|
||||
if rollups.get("by_status") != _count_by(targets, "status"):
|
||||
raise ValueError(f"{label}: rollups.by_status must match service_health_targets")
|
||||
if rollups.get("by_freshness_status") != _count_by(targets, "freshness_status"):
|
||||
raise ValueError(f"{label}: rollups.by_freshness_status must match service_health_targets")
|
||||
|
||||
requiring_action = sorted(
|
||||
target.get("target_id")
|
||||
for target in targets
|
||||
if target.get("status") == "action_required"
|
||||
)
|
||||
if sorted(rollups.get("target_ids_requiring_action") or []) != requiring_action:
|
||||
raise ValueError(f"{label}: rollups.target_ids_requiring_action must match targets")
|
||||
|
||||
if sorted(rollups.get("health_gap_ids") or []) != sorted(gap.get("gap_id") for gap in gaps):
|
||||
raise ValueError(f"{label}: rollups.health_gap_ids must match health_gaps")
|
||||
|
||||
if sorted(rollups.get("stale_endpoint_ids") or []) != sorted(
|
||||
endpoint.get("endpoint_id") for endpoint in stale_endpoints
|
||||
):
|
||||
raise ValueError(f"{label}: rollups.stale_endpoint_ids must match stale_endpoints")
|
||||
|
||||
critical_targets = sorted(
|
||||
target.get("target_id")
|
||||
for target in targets
|
||||
if target.get("risk_level") == "critical"
|
||||
)
|
||||
if sorted(rollups.get("critical_target_ids") or []) != critical_targets:
|
||||
raise ValueError(f"{label}: rollups.critical_target_ids must match service_health_targets")
|
||||
|
||||
zero_count_fields = {
|
||||
"service_restart_allowed_count",
|
||||
"endpoint_change_allowed_count",
|
||||
"active_probe_allowed_count",
|
||||
"notification_send_allowed_count",
|
||||
"runtime_execution_allowed_count",
|
||||
}
|
||||
non_zero = sorted(field for field in zero_count_fields if rollups.get(field) != 0)
|
||||
if non_zero:
|
||||
raise ValueError(f"{label}: operation permission rollup counts must remain 0: {non_zero}")
|
||||
|
||||
|
||||
def _require_target_evidence(payload: dict[str, Any], label: str) -> None:
|
||||
targets = payload.get("service_health_targets") or []
|
||||
missing = sorted(
|
||||
target.get("target_id")
|
||||
for target in targets
|
||||
if not target.get("health_contract")
|
||||
or not target.get("endpoint_contract")
|
||||
or not target.get("evidence_refs")
|
||||
or not target.get("next_action")
|
||||
)
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"{label}: service_health_targets must include health, endpoint, evidence, next_action: {missing}"
|
||||
)
|
||||
|
||||
required_target_ids = {
|
||||
"production_api_health_public",
|
||||
"awoooi_web_health_manifest",
|
||||
"ollama_three_layer_health_contract",
|
||||
"openclaw_health_endpoint_contract",
|
||||
"prometheus_alertmanager_endpoint_reference",
|
||||
"gitea_workflow_runner_health_contract",
|
||||
"kali_scanner_health_reference",
|
||||
}
|
||||
present = {target.get("target_id") for target in targets}
|
||||
missing_required = sorted(required_target_ids - present)
|
||||
if missing_required:
|
||||
raise ValueError(f"{label}: missing required service health targets: {missing_required}")
|
||||
|
||||
|
||||
def _require_gap_evidence(payload: dict[str, Any], label: str) -> None:
|
||||
gaps = payload.get("health_gaps") or []
|
||||
endpoints = payload.get("stale_endpoints") or []
|
||||
required_gap_ids = {
|
||||
"endpoint_reference_stale_hosts",
|
||||
"gitea_runner_attestation_health_gap",
|
||||
"health_check_script_not_authoritative",
|
||||
"openclaw_nemo_rca_health_review",
|
||||
"security_scanner_health_evidence_gap",
|
||||
}
|
||||
present_gaps = {gap.get("gap_id") for gap in gaps}
|
||||
missing_gaps = sorted(required_gap_ids - present_gaps)
|
||||
if missing_gaps:
|
||||
raise ValueError(f"{label}: missing required health gaps: {missing_gaps}")
|
||||
|
||||
missing_gap_fields = sorted(
|
||||
gap.get("gap_id")
|
||||
for gap in gaps
|
||||
if not gap.get("target_ids") or not gap.get("evidence_refs") or not gap.get("next_action")
|
||||
)
|
||||
if missing_gap_fields:
|
||||
raise ValueError(f"{label}: health gaps must include targets, evidence, next_action: {missing_gap_fields}")
|
||||
|
||||
required_endpoint_ids = {
|
||||
"legacy_188_ollama_provider_endpoint",
|
||||
"prometheus_alertmanager_110_188_split",
|
||||
"openclaw_8088_comment_vs_8089_contract",
|
||||
}
|
||||
present_endpoints = {endpoint.get("endpoint_id") for endpoint in endpoints}
|
||||
missing_endpoints = sorted(required_endpoint_ids - present_endpoints)
|
||||
if missing_endpoints:
|
||||
raise ValueError(f"{label}: missing required stale endpoints: {missing_endpoints}")
|
||||
|
||||
missing_endpoint_fields = sorted(
|
||||
endpoint.get("endpoint_id")
|
||||
for endpoint in endpoints
|
||||
if not endpoint.get("stale_ref")
|
||||
or not endpoint.get("current_truth")
|
||||
or not endpoint.get("evidence_refs")
|
||||
or not endpoint.get("next_action")
|
||||
)
|
||||
if missing_endpoint_fields:
|
||||
raise ValueError(
|
||||
f"{label}: stale endpoints must include stale_ref, current_truth, evidence, next_action: {missing_endpoint_fields}"
|
||||
)
|
||||
|
||||
|
||||
def _require_operator_denials(payload: dict[str, Any], label: str) -> None:
|
||||
contract = payload.get("operator_contract") or {}
|
||||
must_not_interpret_as = set(contract.get("must_not_interpret_as") or [])
|
||||
required_denials = {
|
||||
"服務重啟批准",
|
||||
"endpoint 修改批准",
|
||||
"active probe 批准",
|
||||
"live health check 已執行",
|
||||
"Secret payload 已讀取或可輸出",
|
||||
"Telegram 成功通知批准",
|
||||
"workflow / deploy / reload 觸發批准",
|
||||
"provider 切換批准",
|
||||
"runtime execution 授權",
|
||||
"OpenClaw 取代或降級批准",
|
||||
}
|
||||
if not required_denials.issubset(must_not_interpret_as):
|
||||
raise ValueError(f"{label}: operator_contract.must_not_interpret_as is missing required denials")
|
||||
|
||||
restart_policy = str(contract.get("restart_policy") or "")
|
||||
if "P1-005" not in restart_policy or "只能列缺口" not in restart_policy:
|
||||
raise ValueError(f"{label}: restart_policy must preserve P1-005 read-only boundary")
|
||||
|
||||
endpoint_policy = str(contract.get("endpoint_policy") or "")
|
||||
if "P1-005" not in endpoint_policy or "不修改端點" not in endpoint_policy:
|
||||
raise ValueError(f"{label}: endpoint_policy must preserve P1-005 endpoint boundary")
|
||||
|
||||
notification_policy = str(contract.get("notification_policy") or "")
|
||||
if "P1-007" not in notification_policy or "成功 smoke 不通知" not in notification_policy:
|
||||
raise ValueError(f"{label}: notification_policy must preserve failure-only notification boundary")
|
||||
|
||||
|
||||
def _require_no_plaintext_secret_payload_keys(value: Any, label: str, path: str = "$") -> None:
|
||||
if isinstance(value, dict):
|
||||
forbidden_key_fragments = {
|
||||
"secret_value",
|
||||
"token_value",
|
||||
"authorization_header",
|
||||
"private_key",
|
||||
"webhook_secret",
|
||||
"runner_token",
|
||||
"api_key_value",
|
||||
"password_value",
|
||||
}
|
||||
for key, nested in value.items():
|
||||
lowered = str(key).lower()
|
||||
if any(fragment in lowered for fragment in forbidden_key_fragments):
|
||||
raise ValueError(f"{label}: forbidden secret payload key at {path}.{key}")
|
||||
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}.{key}")
|
||||
elif isinstance(value, list):
|
||||
for index, nested in enumerate(value):
|
||||
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}[{index}]")
|
||||
|
||||
|
||||
def _count_by(items: list[dict[str, Any]], key: str) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for item in items:
|
||||
value = item.get(key)
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
return counts
|
||||
@@ -16,18 +16,18 @@ def test_ai_agent_automation_backlog_snapshot_endpoint_returns_committed_snapsho
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["schema_version"] == "ai_agent_automation_backlog_v1"
|
||||
assert data["program_status"]["overall_completion_percent"] == 87
|
||||
assert data["program_status"]["overall_completion_percent"] == 88
|
||||
assert data["program_status"]["read_only_mode"] is True
|
||||
assert data["program_status"]["current_task_id"] == "P1-004"
|
||||
assert data["program_status"]["next_task_id"] == "P1-005"
|
||||
assert data["rollups"]["total_items"] == len(data["backlog_items"]) == 23
|
||||
assert data["rollups"]["by_priority"]["P1"] == 21
|
||||
assert data["rollups"]["by_status"]["done"] == 20
|
||||
assert data["rollups"]["by_gate_status"]["read_only_allowed"] == 20
|
||||
assert data["progress_summary"]["overall_percent"] == 87
|
||||
assert data["progress_summary"]["done_items"] == 20
|
||||
assert data["progress_summary"]["total_items"] == 23
|
||||
assert data["item_approval_boundary_rollup"]["total_items"] == 23
|
||||
assert data["program_status"]["current_task_id"] == "P1-005"
|
||||
assert data["program_status"]["next_task_id"] == "P1-006"
|
||||
assert data["rollups"]["total_items"] == len(data["backlog_items"]) == 24
|
||||
assert data["rollups"]["by_priority"]["P1"] == 22
|
||||
assert data["rollups"]["by_status"]["done"] == 21
|
||||
assert data["rollups"]["by_gate_status"]["read_only_allowed"] == 21
|
||||
assert data["progress_summary"]["overall_percent"] == 88
|
||||
assert data["progress_summary"]["done_items"] == 21
|
||||
assert data["progress_summary"]["total_items"] == 24
|
||||
assert data["item_approval_boundary_rollup"]["total_items"] == 24
|
||||
assert data["item_approval_boundary_rollup"]["items_requiring_explicit_approval"] == [
|
||||
"AUTO-P1-004",
|
||||
"AUTO-P2-004",
|
||||
@@ -61,6 +61,12 @@ def test_ai_agent_automation_backlog_snapshot_endpoint_returns_committed_snapsho
|
||||
assert p1_004["approval_boundary"]["mode"] == "production_change_blocked"
|
||||
assert "provider_switch" in p1_004["approval_boundary"]["blocked_actions"]
|
||||
assert "ai_provider_route_matrix_2026-06-05.json" in p1_004["evidence_refs"][0]
|
||||
p1_005 = next(item for item in data["backlog_items"] if item["item_id"] == "AUTO-P1-005")
|
||||
assert p1_005["status"] == "done"
|
||||
assert p1_005["next_review"] == "P1-006"
|
||||
assert p1_005["approval_boundary"]["mode"] == "read_only_allowed"
|
||||
assert "service_restart" in p1_005["approval_boundary"]["blocked_actions"]
|
||||
assert "service_health_gap_matrix_2026-06-05.json" in p1_005["evidence_refs"][0]
|
||||
p1_306 = next(item for item in data["backlog_items"] if item["item_id"] == "AUTO-P1-306")
|
||||
assert p1_306["approval_boundary"]["mode"] == "read_only_allowed"
|
||||
assert "runtime_execution" in p1_306["approval_boundary"]["blocked_actions"]
|
||||
|
||||
@@ -18,10 +18,10 @@ def test_ai_agent_automation_inventory_snapshot_endpoint_returns_committed_snaps
|
||||
assert data["schema_version"] == "ai_agent_automation_inventory_snapshot_v1"
|
||||
assert data["program_status"]["overall_completion_percent"] == 100
|
||||
assert data["program_status"]["read_only_mode"] is True
|
||||
assert data["program_status"]["current_task_id"] == "P1-004"
|
||||
assert data["program_status"]["next_task_id"] == "P1-005"
|
||||
assert data["task_approval_boundary_rollup"]["total_tasks"] == len(data["tasks"]) == 30
|
||||
assert data["task_approval_boundary_rollup"]["by_mode"]["read_only_allowed"] == 27
|
||||
assert data["program_status"]["current_task_id"] == "P1-005"
|
||||
assert data["program_status"]["next_task_id"] == "P1-006"
|
||||
assert data["task_approval_boundary_rollup"]["total_tasks"] == len(data["tasks"]) == 31
|
||||
assert data["task_approval_boundary_rollup"]["by_mode"]["read_only_allowed"] == 28
|
||||
assert data["task_approval_boundary_rollup"]["by_mode"]["production_change_blocked"] == 1
|
||||
assert data["task_approval_boundary_rollup"]["tasks_requiring_explicit_approval"] == [
|
||||
"P0-001",
|
||||
@@ -48,6 +48,11 @@ def test_ai_agent_automation_inventory_snapshot_endpoint_returns_committed_snaps
|
||||
assert p1_004["approval_boundary"]["mode"] == "production_change_blocked"
|
||||
assert "provider_switch" in p1_004["approval_boundary"]["blocked_actions"]
|
||||
assert "ai_provider_route_matrix_2026-06-05.json" in p1_004["output"]
|
||||
p1_005 = next(task for task in data["tasks"] if task["task_id"] == "P1-005")
|
||||
assert p1_005["status"] == "done"
|
||||
assert p1_005["approval_boundary"]["mode"] == "read_only_allowed"
|
||||
assert "service_restart" in p1_005["approval_boundary"]["blocked_actions"]
|
||||
assert "service_health_gap_matrix_2026-06-05.json" in p1_005["output"]
|
||||
assert any(task["task_id"] == "P1-204" for task in data["tasks"])
|
||||
assert any(task["task_id"] == "P1-205" for task in data["tasks"])
|
||||
assert any(task["task_id"] == "P1-206" for task in data["tasks"])
|
||||
@@ -80,3 +85,4 @@ def test_ai_agent_automation_inventory_snapshot_endpoint_returns_committed_snaps
|
||||
assert any(evidence["evidence_id"] == "gitea_workflow_runner_health_api" for evidence in data["evidence"])
|
||||
assert any(evidence["evidence_id"] == "observability_contract_matrix_api" for evidence in data["evidence"])
|
||||
assert any(evidence["evidence_id"] == "ai_provider_route_matrix_api" for evidence in data["evidence"])
|
||||
assert any(evidence["evidence_id"] == "service_health_gap_matrix_api" for evidence in data["evidence"])
|
||||
|
||||
301
apps/api/tests/test_service_health_gap_matrix.py
Normal file
301
apps/api/tests/test_service_health_gap_matrix.py
Normal file
@@ -0,0 +1,301 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from src.services.service_health_gap_matrix import load_latest_service_health_gap_matrix
|
||||
|
||||
|
||||
def test_load_latest_service_health_gap_matrix_reads_newest_file(tmp_path):
|
||||
older = _snapshot(generated_at="2026-06-04T00:00:00+08:00", completion=40)
|
||||
newer = _snapshot(generated_at="2026-06-05T00:00:00+08:00", completion=100)
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-04.json").write_text(
|
||||
json.dumps(older),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(newer),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
loaded = load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
assert loaded["generated_at"] == "2026-06-05T00:00:00+08:00"
|
||||
assert loaded["program_status"]["overall_completion_percent"] == 100
|
||||
assert loaded["rollups"]["total_targets"] == 7
|
||||
assert loaded["operation_boundaries"]["active_probe_allowed"] is False
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_requires_read_only_mode(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["program_status"]["read_only_mode"] = False
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="read_only_mode"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_blocks_probe_restart_endpoint_and_notifications(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["operation_boundaries"]["active_probe_allowed"] = True
|
||||
snapshot["operation_boundaries"]["service_restart_allowed"] = True
|
||||
snapshot["operation_boundaries"]["endpoint_change_allowed"] = True
|
||||
snapshot["operation_boundaries"]["notification_send_allowed"] = True
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="operation boundaries"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_requires_rollup_consistency(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["rollups"]["target_ids_requiring_action"] = []
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="target_ids_requiring_action"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_requires_required_gaps_and_stale_endpoints(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["health_gaps"] = snapshot["health_gaps"][:-1]
|
||||
snapshot["stale_endpoints"] = snapshot["stale_endpoints"][:-1]
|
||||
_refresh_rollups(snapshot)
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="missing required health gaps"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_requires_operator_denials(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["operator_contract"]["must_not_interpret_as"].remove("active probe 批准")
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="operator_contract"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_requires_endpoint_and_notification_policies(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["operator_contract"]["endpoint_policy"] = "可修改端點"
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="endpoint_policy"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_rejects_secret_payload_keys(tmp_path):
|
||||
snapshot = _snapshot()
|
||||
snapshot["latest_observations"][0]["authorization_header"] = "redacted"
|
||||
(tmp_path / "service_health_gap_matrix_2026-06-05.json").write_text(
|
||||
json.dumps(snapshot),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="forbidden secret payload key"):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_fails_when_missing(tmp_path):
|
||||
with pytest.raises(FileNotFoundError):
|
||||
load_latest_service_health_gap_matrix(tmp_path)
|
||||
|
||||
|
||||
def _snapshot(
|
||||
*,
|
||||
generated_at: str = "2026-06-05T00:00:00+08:00",
|
||||
completion: int = 100,
|
||||
) -> dict:
|
||||
targets = [
|
||||
_target("production_api_health_public", "api_health", "verified", "critical", "fresh_readback"),
|
||||
_target("awoooi_web_health_manifest", "web_health", "verified", "high", "manifest_mapped"),
|
||||
_target("ollama_three_layer_health_contract", "ai_provider_health", "action_required", "critical", "source_mismatch"),
|
||||
_target("openclaw_health_endpoint_contract", "ai_provider_health", "action_required", "critical", "source_mismatch"),
|
||||
_target("prometheus_alertmanager_endpoint_reference", "observability_health", "action_required", "high", "source_mismatch"),
|
||||
_target("gitea_workflow_runner_health_contract", "devops_health", "action_required", "critical", "stale_evidence"),
|
||||
_target("kali_scanner_health_reference", "security_edge_health", "action_required", "medium", "manifest_only"),
|
||||
]
|
||||
gaps = [
|
||||
_gap("endpoint_reference_stale_hosts", ["ollama_three_layer_health_contract"]),
|
||||
_gap("gitea_runner_attestation_health_gap", ["gitea_workflow_runner_health_contract"]),
|
||||
_gap("health_check_script_not_authoritative", ["production_api_health_public"]),
|
||||
_gap("openclaw_nemo_rca_health_review", ["openclaw_health_endpoint_contract"]),
|
||||
_gap("security_scanner_health_evidence_gap", ["kali_scanner_health_reference"]),
|
||||
]
|
||||
endpoints = [
|
||||
_endpoint("legacy_188_ollama_provider_endpoint"),
|
||||
_endpoint("prometheus_alertmanager_110_188_split"),
|
||||
_endpoint("openclaw_8088_comment_vs_8089_contract"),
|
||||
]
|
||||
payload = {
|
||||
"schema_version": "service_health_gap_matrix_v1",
|
||||
"generated_at": generated_at,
|
||||
"program_status": {
|
||||
"overall_completion_percent": completion,
|
||||
"current_priority": "P1",
|
||||
"current_task_id": "P1-005",
|
||||
"next_task_id": "P1-006",
|
||||
"read_only_mode": True,
|
||||
},
|
||||
"source_refs": ["docs/schemas/service_health_gap_matrix_v1.schema.json"],
|
||||
"rollups": {},
|
||||
"service_health_targets": targets,
|
||||
"health_gaps": gaps,
|
||||
"stale_endpoints": endpoints,
|
||||
"latest_observations": [
|
||||
{
|
||||
"observation_id": "service_health_gap_matrix_seed",
|
||||
"status": "verified",
|
||||
"summary": "只讀 health gap seed。",
|
||||
"evidence_refs": ["docs/reference/SERVICE-ENDPOINTS.md"],
|
||||
}
|
||||
],
|
||||
"operator_contract": {
|
||||
"display_mode": "read_only_service_health_gap_matrix",
|
||||
"must_not_interpret_as": [
|
||||
"服務重啟批准",
|
||||
"endpoint 修改批准",
|
||||
"active probe 批准",
|
||||
"live health check 已執行",
|
||||
"Secret payload 已讀取或可輸出",
|
||||
"Telegram 成功通知批准",
|
||||
"workflow / deploy / reload 觸發批准",
|
||||
"provider 切換批准",
|
||||
"runtime execution 授權",
|
||||
"OpenClaw 取代或降級批准",
|
||||
],
|
||||
"secret_display_policy": "只顯示來源檔案與 env var 名稱。",
|
||||
"restart_policy": "P1-005 只能列缺口;任何 restart 都需另行批准。",
|
||||
"endpoint_policy": "P1-005 不修改端點,只提出 source truth cleanup。",
|
||||
"notification_policy": "成功 smoke 不通知;失敗或 stale 才進 P1-007 通知合約。",
|
||||
},
|
||||
"operation_boundaries": {
|
||||
"read_only_api_allowed": True,
|
||||
"service_restart_allowed": False,
|
||||
"pod_restart_allowed": False,
|
||||
"host_restart_allowed": False,
|
||||
"rollout_restart_allowed": False,
|
||||
"endpoint_change_allowed": False,
|
||||
"configmap_patch_allowed": False,
|
||||
"active_probe_allowed": False,
|
||||
"external_health_probe_allowed": False,
|
||||
"live_benchmark_allowed": False,
|
||||
"provider_switch_allowed": False,
|
||||
"paid_api_call_allowed": False,
|
||||
"secret_read_allowed": False,
|
||||
"secret_plaintext_allowed": False,
|
||||
"notification_send_allowed": False,
|
||||
"workflow_trigger_allowed": False,
|
||||
"deploy_trigger_allowed": False,
|
||||
"reload_trigger_allowed": False,
|
||||
"runtime_execution_allowed": False,
|
||||
},
|
||||
"approval_boundaries": {
|
||||
"service_restart_approved": False,
|
||||
"endpoint_change_approved": False,
|
||||
"configmap_patch_approved": False,
|
||||
"active_probe_approved": False,
|
||||
"secret_read_approved": False,
|
||||
"notification_send_approved": False,
|
||||
"workflow_trigger_approved": False,
|
||||
"provider_switch_approved": False,
|
||||
"cost_change_approved": False,
|
||||
},
|
||||
}
|
||||
_refresh_rollups(payload)
|
||||
return payload
|
||||
|
||||
|
||||
def _target(target_id: str, kind: str, status: str, risk_level: str, freshness_status: str) -> dict:
|
||||
return {
|
||||
"target_id": target_id,
|
||||
"display_name": target_id,
|
||||
"kind": kind,
|
||||
"status": status,
|
||||
"risk_level": risk_level,
|
||||
"freshness_status": freshness_status,
|
||||
"health_contract": "只讀健康合約。",
|
||||
"endpoint_contract": "不修改 endpoint。",
|
||||
"evidence_refs": ["docs/reference/SERVICE-ENDPOINTS.md"],
|
||||
"next_action": "只補證據,不執行。",
|
||||
}
|
||||
|
||||
|
||||
def _gap(gap_id: str, target_ids: list[str]) -> dict:
|
||||
return {
|
||||
"gap_id": gap_id,
|
||||
"display_name": gap_id,
|
||||
"status": "action_required",
|
||||
"severity": "high",
|
||||
"summary": "只讀缺口。",
|
||||
"target_ids": target_ids,
|
||||
"evidence_refs": ["docs/reference/SERVICE-ENDPOINTS.md"],
|
||||
"next_action": "只整理 owner attestation。",
|
||||
}
|
||||
|
||||
|
||||
def _endpoint(endpoint_id: str) -> dict:
|
||||
return {
|
||||
"endpoint_id": endpoint_id,
|
||||
"display_name": endpoint_id,
|
||||
"status": "action_required",
|
||||
"severity": "medium",
|
||||
"stale_ref": "舊端點參考。",
|
||||
"current_truth": "只標記 source drift,不判定 live down。",
|
||||
"evidence_refs": ["docs/reference/SERVICE-ENDPOINTS.md"],
|
||||
"next_action": "只整理 source cleanup。",
|
||||
}
|
||||
|
||||
|
||||
def _refresh_rollups(payload: dict) -> None:
|
||||
targets = payload["service_health_targets"]
|
||||
gaps = payload["health_gaps"]
|
||||
endpoints = payload["stale_endpoints"]
|
||||
payload["rollups"] = {
|
||||
"total_targets": len(targets),
|
||||
"by_kind": _count_by(targets, "kind"),
|
||||
"by_status": _count_by(targets, "status"),
|
||||
"by_freshness_status": _count_by(targets, "freshness_status"),
|
||||
"target_ids_requiring_action": sorted(
|
||||
target["target_id"] for target in targets if target["status"] == "action_required"
|
||||
),
|
||||
"health_gap_ids": sorted(gap["gap_id"] for gap in gaps),
|
||||
"stale_endpoint_ids": sorted(endpoint["endpoint_id"] for endpoint in endpoints),
|
||||
"critical_target_ids": sorted(
|
||||
target["target_id"] for target in targets if target["risk_level"] == "critical"
|
||||
),
|
||||
"read_only_denials_total": 10,
|
||||
"service_restart_allowed_count": 0,
|
||||
"endpoint_change_allowed_count": 0,
|
||||
"active_probe_allowed_count": 0,
|
||||
"notification_send_allowed_count": 0,
|
||||
"runtime_execution_allowed_count": 0,
|
||||
}
|
||||
|
||||
|
||||
def _count_by(items: list[dict], key: str) -> dict[str, int]:
|
||||
counts: dict[str, int] = {}
|
||||
for item in items:
|
||||
value = item[key]
|
||||
counts[value] = counts.get(value, 0) + 1
|
||||
return counts
|
||||
49
apps/api/tests/test_service_health_gap_matrix_api.py
Normal file
49
apps/api/tests/test_service_health_gap_matrix_api.py
Normal file
@@ -0,0 +1,49 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1.agents import router
|
||||
|
||||
|
||||
def test_service_health_gap_matrix_endpoint_returns_committed_snapshot():
|
||||
app = FastAPI()
|
||||
app.include_router(router, prefix="/api/v1")
|
||||
client = TestClient(app)
|
||||
|
||||
response = client.get("/api/v1/agents/service-health-gap-matrix")
|
||||
|
||||
assert response.status_code == 200
|
||||
data = response.json()
|
||||
assert data["schema_version"] == "service_health_gap_matrix_v1"
|
||||
assert data["program_status"]["current_task_id"] == "P1-005"
|
||||
assert data["program_status"]["next_task_id"] == "P1-006"
|
||||
assert data["program_status"]["read_only_mode"] is True
|
||||
assert data["rollups"]["total_targets"] == len(data["service_health_targets"]) == 10
|
||||
assert data["rollups"]["service_restart_allowed_count"] == 0
|
||||
assert data["rollups"]["endpoint_change_allowed_count"] == 0
|
||||
assert data["rollups"]["active_probe_allowed_count"] == 0
|
||||
assert data["rollups"]["notification_send_allowed_count"] == 0
|
||||
assert data["rollups"]["runtime_execution_allowed_count"] == 0
|
||||
assert data["operation_boundaries"]["service_restart_allowed"] is False
|
||||
assert data["operation_boundaries"]["endpoint_change_allowed"] is False
|
||||
assert data["operation_boundaries"]["active_probe_allowed"] is False
|
||||
assert data["operation_boundaries"]["notification_send_allowed"] is False
|
||||
assert data["operation_boundaries"]["runtime_execution_allowed"] is False
|
||||
assert data["approval_boundaries"]["active_probe_approved"] is False
|
||||
assert data["approval_boundaries"]["endpoint_change_approved"] is False
|
||||
assert data["approval_boundaries"]["notification_send_approved"] is False
|
||||
assert any(
|
||||
target["target_id"] == "ollama_three_layer_health_contract"
|
||||
and target["status"] == "action_required"
|
||||
for target in data["service_health_targets"]
|
||||
)
|
||||
assert any(
|
||||
gap["gap_id"] == "endpoint_reference_stale_hosts"
|
||||
for gap in data["health_gaps"]
|
||||
)
|
||||
assert any(
|
||||
endpoint["endpoint_id"] == "legacy_188_ollama_provider_endpoint"
|
||||
for endpoint in data["stale_endpoints"]
|
||||
)
|
||||
assert "active probe 批准" in data["operator_contract"]["must_not_interpret_as"]
|
||||
Reference in New Issue
Block a user