233 lines
10 KiB
Python
233 lines
10 KiB
Python
"""
|
|
Observability contract and noise-reduction matrix snapshot.
|
|
|
|
Loads the latest committed, read-only Prometheus / Alertmanager / SigNoz /
|
|
Grafana observability contract matrix. This module never mutates alert rules,
|
|
routes, receivers, silences, dashboards, webhooks, collectors, secrets,
|
|
notifications, workflows, or runtime state.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
from src.services.snapshot_paths import default_evaluations_dir
|
|
|
|
_DEFAULT_EVALUATIONS_DIR = default_evaluations_dir(Path(__file__))
|
|
_SNAPSHOT_PATTERN = "observability_contract_matrix_*.json"
|
|
_SCHEMA_VERSION = "observability_contract_matrix_v1"
|
|
|
|
|
|
def load_latest_observability_contract_matrix(
|
|
evaluations_dir: Path | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Load the newest committed observability contract matrix snapshot."""
|
|
directory = evaluations_dir or _DEFAULT_EVALUATIONS_DIR
|
|
candidates = sorted(directory.glob(_SNAPSHOT_PATTERN))
|
|
if not candidates:
|
|
raise FileNotFoundError(f"no observability contract matrix snapshots found in {directory}")
|
|
|
|
latest = candidates[-1]
|
|
with latest.open(encoding="utf-8") as handle:
|
|
payload = json.load(handle)
|
|
|
|
if not isinstance(payload, dict):
|
|
raise ValueError(f"{latest}: expected JSON object")
|
|
_require_schema(payload, _SCHEMA_VERSION, str(latest))
|
|
_require_read_only_boundaries(payload, str(latest))
|
|
_require_operation_boundaries(payload, str(latest))
|
|
_require_rollup_consistency(payload, str(latest))
|
|
_require_surface_evidence(payload, str(latest))
|
|
_require_noise_opportunities(payload, str(latest))
|
|
_require_operator_denials(payload, str(latest))
|
|
_require_no_plaintext_secret_payload_keys(payload, str(latest))
|
|
return payload
|
|
|
|
|
|
def _require_schema(payload: dict[str, Any], expected: str, label: str) -> None:
|
|
actual = payload.get("schema_version")
|
|
if actual != expected:
|
|
raise ValueError(f"{label}: expected schema_version={expected}, got {actual!r}")
|
|
|
|
|
|
def _require_read_only_boundaries(payload: dict[str, Any], label: str) -> None:
|
|
program_status = payload.get("program_status") or {}
|
|
if program_status.get("read_only_mode") is not True:
|
|
raise ValueError(f"{label}: program_status.read_only_mode must be true")
|
|
|
|
approval_boundaries = payload.get("approval_boundaries") or {}
|
|
allowed = sorted(key for key, value in approval_boundaries.items() if value is not False)
|
|
if allowed:
|
|
raise ValueError(f"{label}: approval boundaries must remain false: {allowed}")
|
|
|
|
|
|
def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None:
|
|
boundaries = payload.get("operation_boundaries") or {}
|
|
if boundaries.get("read_only_api_allowed") is not True:
|
|
raise ValueError(f"{label}: read_only_api_allowed must be true")
|
|
|
|
blocked_flags = {
|
|
"prometheus_rule_write_allowed",
|
|
"prometheus_reload_allowed",
|
|
"alertmanager_route_write_allowed",
|
|
"alertmanager_receiver_change_allowed",
|
|
"alertmanager_to_openclaw_allowed",
|
|
"silence_create_allowed",
|
|
"grafana_dashboard_write_allowed",
|
|
"grafana_api_write_allowed",
|
|
"signoz_query_mutation_allowed",
|
|
"signoz_webhook_change_allowed",
|
|
"sentry_webhook_change_allowed",
|
|
"otel_collector_deploy_allowed",
|
|
"event_exporter_restart_allowed",
|
|
"secret_read_allowed",
|
|
"secret_plaintext_allowed",
|
|
"notification_send_allowed",
|
|
"external_api_call_allowed",
|
|
"live_prometheus_query_allowed",
|
|
"workflow_trigger_allowed",
|
|
"deploy_trigger_allowed",
|
|
"reload_trigger_allowed",
|
|
"runtime_execution_allowed",
|
|
}
|
|
allowed = sorted(flag for flag in blocked_flags if boundaries.get(flag) is not False)
|
|
if allowed:
|
|
raise ValueError(f"{label}: operation boundaries must remain false: {allowed}")
|
|
|
|
|
|
def _require_rollup_consistency(payload: dict[str, Any], label: str) -> None:
|
|
surfaces = payload.get("observability_surfaces") or []
|
|
opportunities = payload.get("noise_reduction_opportunities") or []
|
|
gaps = payload.get("classification_gaps") or []
|
|
rollups = payload.get("rollups") or {}
|
|
|
|
if rollups.get("total_surfaces") != len(surfaces):
|
|
raise ValueError(f"{label}: rollups.total_surfaces must match observability_surfaces")
|
|
if rollups.get("by_kind") != _count_by(surfaces, "kind"):
|
|
raise ValueError(f"{label}: rollups.by_kind must match observability_surfaces")
|
|
if rollups.get("by_status") != _count_by(surfaces, "status"):
|
|
raise ValueError(f"{label}: rollups.by_status must match observability_surfaces")
|
|
if rollups.get("by_evidence_status") != _count_by(surfaces, "evidence_status"):
|
|
raise ValueError(f"{label}: rollups.by_evidence_status must match observability_surfaces")
|
|
if rollups.get("by_noise_policy_status") != _count_by(surfaces, "noise_policy_status"):
|
|
raise ValueError(f"{label}: rollups.by_noise_policy_status must match observability_surfaces")
|
|
|
|
action_required = sorted(
|
|
surface.get("surface_id")
|
|
for surface in surfaces
|
|
if surface.get("status") == "action_required"
|
|
)
|
|
if sorted(rollups.get("surface_ids_requiring_action") or []) != action_required:
|
|
raise ValueError(f"{label}: rollups.surface_ids_requiring_action must match surfaces")
|
|
|
|
proposal_only_surfaces = sorted(
|
|
surface.get("surface_id")
|
|
for surface in surfaces
|
|
if surface.get("noise_policy_status") == "proposal_only"
|
|
)
|
|
if sorted(rollups.get("surface_ids_with_proposal_only_noise_policy") or []) != proposal_only_surfaces:
|
|
raise ValueError(
|
|
f"{label}: rollups.surface_ids_with_proposal_only_noise_policy must match surfaces"
|
|
)
|
|
|
|
approval_required = sorted(
|
|
opportunity.get("opportunity_id")
|
|
for opportunity in opportunities
|
|
if opportunity.get("status") == "approval_required"
|
|
)
|
|
if rollups.get("noise_reduction_opportunities_total") != len(opportunities):
|
|
raise ValueError(f"{label}: rollups.noise_reduction_opportunities_total must match opportunities")
|
|
if sorted(rollups.get("approval_required_opportunity_ids") or []) != approval_required:
|
|
raise ValueError(f"{label}: rollups.approval_required_opportunity_ids must match opportunities")
|
|
|
|
if sorted(rollups.get("classification_gap_ids") or []) != sorted(gap.get("gap_id") for gap in gaps):
|
|
raise ValueError(f"{label}: rollups.classification_gap_ids must match classification_gaps")
|
|
|
|
|
|
def _require_surface_evidence(payload: dict[str, Any], label: str) -> None:
|
|
surfaces = payload.get("observability_surfaces") or []
|
|
missing = sorted(
|
|
surface.get("surface_id")
|
|
for surface in surfaces
|
|
if not surface.get("coverage_contract")
|
|
or not surface.get("evidence_refs")
|
|
or not surface.get("next_action")
|
|
)
|
|
if missing:
|
|
raise ValueError(f"{label}: observability_surfaces must include contract, evidence, next_action: {missing}")
|
|
|
|
|
|
def _require_noise_opportunities(payload: dict[str, Any], label: str) -> None:
|
|
opportunities = payload.get("noise_reduction_opportunities") or []
|
|
non_proposal = sorted(
|
|
opportunity.get("opportunity_id")
|
|
for opportunity in opportunities
|
|
if opportunity.get("proposal_only") is not True
|
|
)
|
|
if non_proposal:
|
|
raise ValueError(f"{label}: noise opportunities must stay proposal_only: {non_proposal}")
|
|
|
|
required_ids = {
|
|
"prometheus_noise_rule_tuning",
|
|
"alertmanager_grouping_inhibit_tuning",
|
|
"success_notification_quiet_policy",
|
|
}
|
|
present = {opportunity.get("opportunity_id") for opportunity in opportunities}
|
|
if not required_ids.issubset(present):
|
|
raise ValueError(f"{label}: missing required noise-reduction opportunities")
|
|
|
|
|
|
def _require_operator_denials(payload: dict[str, Any], label: str) -> None:
|
|
contract = payload.get("operator_contract") or {}
|
|
must_not_interpret_as = set(contract.get("must_not_interpret_as") or [])
|
|
required_denials = {
|
|
"Prometheus alert rule 修改批准",
|
|
"Alertmanager receiver / route 修改批准",
|
|
"Alertmanager 指向 OpenClaw receiver 批准",
|
|
"Silence 建立或維護窗口批准",
|
|
"Grafana dashboard 寫入批准",
|
|
"SigNoz / Sentry webhook 設定修改批准",
|
|
"Secret 已讀取或可輸出",
|
|
"Telegram 測試通知批准",
|
|
"deploy / reload / workflow 觸發批准",
|
|
"runtime execution 授權",
|
|
}
|
|
if not required_denials.issubset(must_not_interpret_as):
|
|
raise ValueError(f"{label}: operator_contract.must_not_interpret_as is missing required denials")
|
|
|
|
route_policy = str(contract.get("alertmanager_route_policy") or "")
|
|
if "OpenClaw" not in route_policy or "不接收 Alertmanager webhook" not in route_policy:
|
|
raise ValueError(f"{label}: operator_contract.alertmanager_route_policy must block OpenClaw receiver use")
|
|
|
|
|
|
def _require_no_plaintext_secret_payload_keys(value: Any, label: str, path: str = "$") -> None:
|
|
if isinstance(value, dict):
|
|
forbidden_key_fragments = {
|
|
"secret_value",
|
|
"token_value",
|
|
"authorization_header",
|
|
"private_key",
|
|
"webhook_secret",
|
|
"runner_token",
|
|
"signoz_token",
|
|
"sentry_dsn",
|
|
}
|
|
for key, nested in value.items():
|
|
lowered = str(key).lower()
|
|
if any(fragment in lowered for fragment in forbidden_key_fragments):
|
|
raise ValueError(f"{label}: forbidden secret payload key at {path}.{key}")
|
|
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}.{key}")
|
|
elif isinstance(value, list):
|
|
for index, nested in enumerate(value):
|
|
_require_no_plaintext_secret_payload_keys(nested, label, f"{path}[{index}]")
|
|
|
|
|
|
def _count_by(items: list[dict[str, Any]], key: str) -> dict[str, int]:
|
|
counts: dict[str, int] = {}
|
|
for item in items:
|
|
value = item.get(key)
|
|
counts[value] = counts.get(value, 0) + 1
|
|
return counts
|