283 lines
12 KiB
Python
283 lines
12 KiB
Python
"""
|
|
Service health gap matrix snapshot.
|
|
|
|
Loads the latest committed, read-only service health / stale endpoint gap
|
|
matrix. This module never restarts services, changes endpoints, probes live
|
|
systems, reads secrets, sends notifications, triggers workflows, deploys, or
|
|
executes runtime actions.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from src.services.snapshot_paths import default_evaluations_dir
|
|
|
|
_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__))
|
|
_SNAPSHOT_PATTERN = "service_health_gap_matrix_*.json"
|
|
_SCHEMA_VERSION = "service_health_gap_matrix_v1"
|
|
|
|
|
|
def load_latest_service_health_gap_matrix(
|
|
evaluations_dir: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Load the newest committed service health gap matrix snapshot."""
|
|
directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR
|
|
candidates = sorted(directory.glob(_SNAPSHOT_PATTERN))
|
|
if not candidates:
|
|
raise FileNotFoundError(f"no service health gap matrix snapshots found in {directory}")
|
|
|
|
latest = candidates[-1]
|
|
with latest.open(encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
|
|
if not isinstance(payload, dict):
|
|
raise ValueError(f"{latest}: expected JSON object")
|
|
_require_schema(payload, _SCHEMA_VERSION, str(latest))
|
|
_require_read_only_boundaries(payload, str(latest))
|
|
_require_operation_boundaries(payload, str(latest))
|
|
_require_rollup_consistency(payload, str(latest))
|
|
_require_target_evidence(payload, str(latest))
|
|
_require_gap_evidence(payload, str(latest))
|
|
_require_operator_denials(payload, str(latest))
|
|
_require_no_plaintext_secret_payload_keys(payload, str(latest))
|
|
return payload
|
|
|
|
|
|
def _require_schema(payload: dict[str, Any], expected: str, label: str) -> None:
|
|
actual = payload.get("schema_version")
|
|
if actual != expected:
|
|
raise ValueError(f"{label}: expected schema_version={expected}, got {actual!r}")
|
|
|
|
|
|
def _require_read_only_boundaries(payload: dict[str, Any], label: str) -> None:
|
|
program_status = payload.get("program_status") or {}
|
|
if program_status.get("read_only_mode") is not True:
|
|
raise ValueError(f"{label}: program_status.read_only_mode must be true")
|
|
|
|
approval_boundaries = payload.get("approval_boundaries") or {}
|
|
allowed = sorted(key for key, value in approval_boundaries.items() if value is not False)
|
|
if allowed:
|
|
raise ValueError(f"{label}: approval boundaries must remain false: {allowed}")
|
|
|
|
|
|
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
|
|
boundaries = payload.get("operation_boundaries") or {}
|
|
if boundaries.get("read_only_api_allowed") is not True:
|
|
raise ValueError(f"{label}: read_only_api_allowed must be true")
|
|
|
|
blocked_flags = {
|
|
"service_restart_allowed",
|
|
"pod_restart_allowed",
|
|
"host_restart_allowed",
|
|
"rollout_restart_allowed",
|
|
"endpoint_change_allowed",
|
|
"configmap_patch_allowed",
|
|
"active_probe_allowed",
|
|
"external_health_probe_allowed",
|
|
"live_benchmark_allowed",
|
|
"provider_switch_allowed",
|
|
"paid_api_call_allowed",
|
|
"secret_read_allowed",
|
|
"secret_plaintext_allowed",
|
|
"notification_send_allowed",
|
|
"workflow_trigger_allowed",
|
|
"deploy_trigger_allowed",
|
|
"reload_trigger_allowed",
|
|
"runtime_execution_allowed",
|
|
}
|
|
allowed = sorted(flag for flag in blocked_flags if boundaries.get(flag) is not False)
|
|
if allowed:
|
|
raise ValueError(f"{label}: operation boundaries must remain false: {allowed}")
|
|
|
|
|
|
def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None:
|
|
targets = payload.get("service_health_targets") or []
|
|
gaps = payload.get("health_gaps") or []
|
|
stale_endpoints = payload.get("stale_endpoints") or []
|
|
rollups = payload.get("rollups") or {}
|
|
|
|
if rollups.get("total_targets") != len(targets):
|
|
raise ValueError(f"{label}: rollups.total_targets must match service_health_targets")
|
|
if rollups.get("by_kind") != _count_by(targets, "kind"):
|
|
raise ValueError(f"{label}: rollups.by_kind must match service_health_targets")
|
|
if rollups.get("by_status") != _count_by(targets, "status"):
|
|
raise ValueError(f"{label}: rollups.by_status must match service_health_targets")
|
|
if rollups.get("by_freshness_status") != _count_by(targets, "freshness_status"):
|
|
raise ValueError(f"{label}: rollups.by_freshness_status must match service_health_targets")
|
|
|
|
requiring_action = sorted(
|
|
target.get("target_id")
|
|
for target in targets
|
|
if target.get("status") == "action_required"
|
|
)
|
|
if sorted(rollups.get("target_ids_requiring_action") or []) != requiring_action:
|
|
raise ValueError(f"{label}: rollups.target_ids_requiring_action must match targets")
|
|
|
|
if sorted(rollups.get("health_gap_ids") or []) != sorted(gap.get("gap_id") for gap in gaps):
|
|
raise ValueError(f"{label}: rollups.health_gap_ids must match health_gaps")
|
|
|
|
if sorted(rollups.get("stale_endpoint_ids") or []) != sorted(
|
|
endpoint.get("endpoint_id") for endpoint in stale_endpoints
|
|
):
|
|
raise ValueError(f"{label}: rollups.stale_endpoint_ids must match stale_endpoints")
|
|
|
|
critical_targets = sorted(
|
|
target.get("target_id")
|
|
for target in targets
|
|
if target.get("risk_level") == "critical"
|
|
)
|
|
if sorted(rollups.get("critical_target_ids") or []) != critical_targets:
|
|
raise ValueError(f"{label}: rollups.critical_target_ids must match service_health_targets")
|
|
|
|
zero_count_fields = {
|
|
"service_restart_allowed_count",
|
|
"endpoint_change_allowed_count",
|
|
"active_probe_allowed_count",
|
|
"notification_send_allowed_count",
|
|
"runtime_execution_allowed_count",
|
|
}
|
|
non_zero = sorted(field for field in zero_count_fields if rollups.get(field) != 0)
|
|
if non_zero:
|
|
raise ValueError(f"{label}: operation permission rollup counts must remain 0: {non_zero}")
|
|
|
|
|
|
def _require_target_evidence(payload: dict[str, Any], label: str) -> None:
|
|
targets = payload.get("service_health_targets") or []
|
|
missing = sorted(
|
|
target.get("target_id")
|
|
for target in targets
|
|
if not target.get("health_contract")
|
|
or not target.get("endpoint_contract")
|
|
or not target.get("evidence_refs")
|
|
or not target.get("next_action")
|
|
)
|
|
if missing:
|
|
raise ValueError(
|
|
f"{label}: service_health_targets must include health, endpoint, evidence, next_action: {missing}"
|
|
)
|
|
|
|
required_target_ids = {
|
|
"production_api_health_public",
|
|
"awoooi_web_health_manifest",
|
|
"ollama_three_layer_health_contract",
|
|
"openclaw_health_endpoint_contract",
|
|
"prometheus_alertmanager_endpoint_reference",
|
|
"gitea_workflow_runner_health_contract",
|
|
"kali_scanner_health_reference",
|
|
}
|
|
present = {target.get("target_id") for target in targets}
|
|
missing_required = sorted(required_target_ids - present)
|
|
if missing_required:
|
|
raise ValueError(f"{label}: missing required service health targets: {missing_required}")
|
|
|
|
|
|
def _require_gap_evidence(payload: dict[str, Any], label: str) -> None:
|
|
gaps = payload.get("health_gaps") or []
|
|
endpoints = payload.get("stale_endpoints") or []
|
|
required_gap_ids = {
|
|
"endpoint_reference_stale_hosts",
|
|
"gitea_runner_attestation_health_gap",
|
|
"health_check_script_not_authoritative",
|
|
"openclaw_nemo_rca_health_review",
|
|
"security_scanner_health_evidence_gap",
|
|
}
|
|
present_gaps = {gap.get("gap_id") for gap in gaps}
|
|
missing_gaps = sorted(required_gap_ids - present_gaps)
|
|
if missing_gaps:
|
|
raise ValueError(f"{label}: missing required health gaps: {missing_gaps}")
|
|
|
|
missing_gap_fields = sorted(
|
|
gap.get("gap_id")
|
|
for gap in gaps
|
|
if not gap.get("target_ids") or not gap.get("evidence_refs") or not gap.get("next_action")
|
|
)
|
|
if missing_gap_fields:
|
|
raise ValueError(f"{label}: health gaps must include targets, evidence, next_action: {missing_gap_fields}")
|
|
|
|
required_endpoint_ids = {
|
|
"legacy_188_ollama_provider_endpoint",
|
|
"prometheus_alertmanager_110_188_split",
|
|
"openclaw_8088_comment_vs_8089_contract",
|
|
}
|
|
present_endpoints = {endpoint.get("endpoint_id") for endpoint in endpoints}
|
|
missing_endpoints = sorted(required_endpoint_ids - present_endpoints)
|
|
if missing_endpoints:
|
|
raise ValueError(f"{label}: missing required stale endpoints: {missing_endpoints}")
|
|
|
|
missing_endpoint_fields = sorted(
|
|
endpoint.get("endpoint_id")
|
|
for endpoint in endpoints
|
|
if not endpoint.get("stale_ref")
|
|
or not endpoint.get("current_truth")
|
|
or not endpoint.get("evidence_refs")
|
|
or not endpoint.get("next_action")
|
|
)
|
|
if missing_endpoint_fields:
|
|
raise ValueError(
|
|
f"{label}: stale endpoints must include stale_ref, current_truth, evidence, next_action: {missing_endpoint_fields}"
|
|
)
|
|
|
|
|
|
def _require_operator_denials(payload: dict[str, Any], label: str) -> None:
|
|
contract = payload.get("operator_contract") or {}
|
|
must_not_interpret_as = set(contract.get("must_not_interpret_as") or [])
|
|
required_denials = {
|
|
"服務重啟批准",
|
|
"endpoint 修改批准",
|
|
"active probe 批准",
|
|
"live health check 已執行",
|
|
"Secret payload 已讀取或可輸出",
|
|
"Telegram 成功通知批准",
|
|
"workflow / deploy / reload 觸發批准",
|
|
"provider 切換批准",
|
|
"runtime execution 授權",
|
|
"OpenClaw 取代或降級批准",
|
|
}
|
|
if not required_denials.issubset(must_not_interpret_as):
|
|
raise ValueError(f"{label}: operator_contract.must_not_interpret_as is missing required denials")
|
|
|
|
restart_policy = str(contract.get("restart_policy") or "")
|
|
if "P1-005" not in restart_policy or "只能列缺口" not in restart_policy:
|
|
raise ValueError(f"{label}: restart_policy must preserve P1-005 read-only boundary")
|
|
|
|
endpoint_policy = str(contract.get("endpoint_policy") or "")
|
|
if "P1-005" not in endpoint_policy or "不修改端點" not in endpoint_policy:
|
|
raise ValueError(f"{label}: endpoint_policy must preserve P1-005 endpoint boundary")
|
|
|
|
notification_policy = str(contract.get("notification_policy") or "")
|
|
if "P1-007" not in notification_policy or "成功 smoke 不通知" not in notification_policy:
|
|
raise ValueError(f"{label}: notification_policy must preserve failure-only notification boundary")
|
|
|
|
|
|
def _require_no_plaintext_secret_payload_keys(value: Any, label: str, path: str = "$") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden_key_fragments = {
|
|
"secret_value",
|
|
"token_value",
|
|
"authorization_header",
|
|
"private_key",
|
|
"webhook_secret",
|
|
"runner_token",
|
|
"api_key_value",
|
|
"password_value",
|
|
}
|
|
for key, nested in value.items():
|
|
lowered = str(key).lower()
|
|
if any(fragment in lowered for fragment in forbidden_key_fragments):
|
|
raise ValueError(f"{label}: forbidden secret payload key at {path}.{key}")
|
|
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, nested in enumerate(value):
|
|
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}[{index}]")
|
|
|
|
|
|
def _count_by(items: list[dict[str, Any]], key: str) -> dict[str, int]:
|
|
counts: dict[str, int] = {}
|
|
for item in items:
|
|
value = item.get(key)
|
|
counts[value] = counts.get(value, 0) + 1
|
|
return counts
|