""" Read-only ADR-100 SLO status snapshot. GovernanceAgent.check_slo_compliance() can emit governance alerts when an SLO is violated. This service is intentionally read-only so dashboards can show the same Prometheus-backed state without producing Telegram/DB side effects. """ from __future__ import annotations import math from dataclasses import dataclass from typing import Any import httpx import structlog from sqlalchemy import text from src.core.config import settings from src.db.base import get_db_context from src.utils.timezone import now_taipei_iso logger = structlog.get_logger(__name__) @dataclass(frozen=True) class Adr100SloDefinition: name: str query: str target: float hard_red_line: float direction: str unit: str window: str denominator_query: str | None = None denominator_window_seconds: int = 0 minimum_events: float = 1.0 ADR100_SLO_DEFINITIONS: tuple[Adr100SloDefinition, ...] = ( Adr100SloDefinition( name="autonomy_rate", query="sli:autonomy_rate:5m", target=0.80, hard_red_line=0.70, direction="above", unit="percent", window="5m", denominator_query="sum(rate(automation_operation_log_total[5m]))", denominator_window_seconds=300, ), Adr100SloDefinition( name="decision_accuracy", query="sli:decision_accuracy:5m", target=0.90, hard_red_line=0.85, direction="above", unit="percent", window="5m", denominator_query='sum(rate(automation_operation_log_total{outcome="auto_executed"}[5m]))', denominator_window_seconds=300, ), Adr100SloDefinition( name="confidence_calibration", query="sli:confidence_calibration:1h", target=0.80, hard_red_line=0.70, direction="above", unit="percent", window="1h", denominator_query="sum(rate(approval_records_high_confidence_total[1h]))", denominator_window_seconds=3600, ), Adr100SloDefinition( name="km_growth_rate", query="max(knowledge_entries_created_24h) or max(sli:km_growth_rate:24h)", target=20.0, hard_red_line=5.0, direction="above", unit="count", window="24h", ), Adr100SloDefinition( name="truth_chain_quality_summary_latency", query='max(awooop_truth_chain_quality_summary_last_duration_seconds{project_id="awoooi",limit="8",success="true"})', target=2.0, hard_red_line=8.0, direction="below", unit="seconds", window="last_observation", minimum_events=0.0, ), ) class Adr100SloStatusService: """Fetch ADR-100 SLO status from Prometheus without writing governance events.""" async def fetch_report(self) -> dict[str, Any]: prom_url = getattr( settings, "PROMETHEUS_URL", "http://prometheus.observability.svc:9090", ).rstrip("/") metrics: list[dict[str, Any]] = [] async with httpx.AsyncClient(timeout=5.0) as client: for definition in ADR100_SLO_DEFINITIONS: metrics.append(await self._fetch_metric(client, prom_url, definition)) evaluable = [metric for metric in metrics if metric.get("evaluable")] ok_count = sum(1 for metric in evaluable if metric.get("status") == "ok") overall_compliance = (ok_count / len(evaluable)) if evaluable else None verification_coverage = await self._fetch_verification_coverage() overall_status = _overall_status(metrics, evaluable, verification_coverage) return { "schema_version": "adr100_slo_status_v1", "source": "prometheus+postgresql", "evaluated_at": now_taipei_iso(), "overall_status": overall_status, "overall_compliance": overall_compliance, "evaluable_count": len(evaluable), "metric_count": len(metrics), "metrics": metrics, "verification_coverage": verification_coverage, } async def _fetch_metric( self, client: httpx.AsyncClient, prom_url: str, definition: Adr100SloDefinition, ) -> dict[str, Any]: denominator_value: float | None = None sample_count: float | None = None if definition.denominator_query: denominator_result = await _query_prometheus_value( client, prom_url, definition.denominator_query, ) if denominator_result["status"] != "ok": return _metric_payload( definition, value=None, status="no_data", reason=denominator_result["reason"], denominator_value=None, sample_count=None, ) denominator_value = float(denominator_result["value"]) sample_count = denominator_value * definition.denominator_window_seconds if sample_count < definition.minimum_events: return _metric_payload( definition, value=None, status="skipped_low_volume", reason="denominator_below_minimum_events", denominator_value=denominator_value, sample_count=sample_count, ) value_result = await _query_prometheus_value(client, prom_url, definition.query) if value_result["status"] != "ok": status = ( "skipped_low_volume" if value_result["reason"] == "prometheus_nan_or_inf" else "no_data" ) return _metric_payload( definition, value=None, status=status, reason=value_result["reason"], denominator_value=denominator_value, sample_count=sample_count, ) value = float(value_result["value"]) status = _classify_status(value, definition) return _metric_payload( definition, value=value, status=status, reason=None, denominator_value=denominator_value, sample_count=sample_count if sample_count is not None else value, ) async def _fetch_verification_coverage(self) -> dict[str, Any]: """Summarize whether recent auto-repair executions have verifier evidence.""" try: async with get_db_context() as db: summary_row = ( await db.execute(text(_VERIFICATION_COVERAGE_SQL)) ).mappings().one() recent_rows = ( await db.execute(text(_VERIFICATION_COVERAGE_RECENT_SQL)) ).mappings().all() recent_non_success_rows = ( await db.execute(text(_VERIFICATION_COVERAGE_NON_SUCCESS_SQL)) ).mappings().all() except Exception as exc: logger.warning("adr100_verification_coverage_query_error", error=str(exc)) return { "schema_version": "adr100_verification_coverage_v1", "source": "postgresql", "window": "24h", "status": "error", "reason": "postgresql_query_error", "evaluable": False, "total_auto": 0, "successful_auto": 0, "verified_auto": 0, "verified_success": 0, "verified_non_success": 0, "unverified_auto": 0, "coverage_rate": None, "verification_success_rate": None, "last_auto_at": None, "last_verified_auto_at": None, "last_verification_evidence_at": None, "latest_auto_age_seconds": None, "last_verified_auto_age_seconds": None, "recent_unverified": [], "recent_non_success": [], "non_success_breakdown": { "by_verification_result": [], "by_failure_class": [], }, "remediation_queue": _remediation_queue_payload([]), } return _build_verification_coverage_payload( summary_row, recent_rows, recent_non_success_rows, ) _VERIFICATION_COVERAGE_SQL = """ WITH recent_auto AS ( SELECT id, incident_id, success, created_at FROM auto_repair_executions WHERE created_at >= NOW() - INTERVAL '24 hours' ), per_auto AS ( SELECT are.id, are.incident_id, are.success, are.created_at, latest.verification_result, latest.collected_at AS verification_collected_at, latest.self_healing_score FROM recent_auto are LEFT JOIN LATERAL ( SELECT ev.verification_result, ev.collected_at, ev.self_healing_score FROM incident_evidence ev WHERE ev.incident_id = are.incident_id AND ev.verification_result IS NOT NULL ORDER BY ev.collected_at DESC LIMIT 1 ) latest ON TRUE ) SELECT count(*)::int AS total_auto, count(*) FILTER (WHERE success)::int AS successful_auto, count(*) FILTER (WHERE verification_result IS NOT NULL)::int AS verified_auto, count(*) FILTER (WHERE verification_result = 'success')::int AS verified_success, count(*) FILTER (WHERE verification_result IN ('degraded','failed','timeout'))::int AS verified_non_success, count(*) FILTER (WHERE verification_result IS NULL)::int AS unverified_auto, max(created_at) AS last_auto_at, max(created_at) FILTER (WHERE verification_result IS NOT NULL) AS last_verified_auto_at, max(verification_collected_at) AS last_verification_evidence_at, EXTRACT(EPOCH FROM (NOW() - max(created_at)))::int AS latest_auto_age_seconds, EXTRACT(EPOCH FROM (NOW() - (max(created_at) FILTER (WHERE verification_result IS NOT NULL))))::int AS last_verified_auto_age_seconds FROM per_auto """ _VERIFICATION_COVERAGE_RECENT_SQL = """ WITH recent_auto AS ( SELECT id, incident_id, success, created_at FROM auto_repair_executions WHERE created_at >= NOW() - INTERVAL '24 hours' ), per_auto AS ( SELECT are.id, are.incident_id, are.success, are.created_at, latest.verification_result FROM recent_auto are LEFT JOIN LATERAL ( SELECT ev.verification_result FROM incident_evidence ev WHERE ev.incident_id = are.incident_id AND ev.verification_result IS NOT NULL ORDER BY ev.collected_at DESC LIMIT 1 ) latest ON TRUE ) SELECT id, incident_id, success, created_at FROM per_auto WHERE verification_result IS NULL ORDER BY created_at DESC LIMIT 5 """ _VERIFICATION_COVERAGE_NON_SUCCESS_SQL = """ WITH recent_auto AS ( SELECT id, incident_id, success, playbook_id, playbook_name, triggered_by, risk_level, error_message, created_at FROM auto_repair_executions WHERE created_at >= NOW() - INTERVAL '24 hours' ), per_auto AS ( SELECT are.id AS auto_repair_id, are.incident_id, are.success AS auto_success, are.playbook_id, are.playbook_name, are.triggered_by, are.risk_level, left(coalesce(are.error_message, ''), 240) AS auto_error, are.created_at AS auto_created_at, latest.verification_result, latest.collected_at AS verification_collected_at, left(coalesce(latest.post_execution_state::text, ''), 700) AS post_state_text, left(coalesce(latest.evidence_summary, ''), 300) AS evidence_summary FROM recent_auto are LEFT JOIN LATERAL ( SELECT ev.verification_result, ev.collected_at, ev.post_execution_state, ev.evidence_summary FROM incident_evidence ev WHERE ev.incident_id = are.incident_id AND ev.verification_result IS NOT NULL ORDER BY ev.collected_at DESC LIMIT 1 ) latest ON TRUE ) SELECT p.*, i.status::text AS incident_status, i.severity::text AS incident_severity, i.alert_category, i.alertname FROM per_auto p LEFT JOIN incidents i ON i.incident_id = p.incident_id WHERE p.verification_result IS NOT NULL AND p.verification_result <> 'success' ORDER BY p.auto_created_at DESC LIMIT 8 """ async def _query_prometheus_value( client: httpx.AsyncClient, prom_url: str, query: str, ) -> dict[str, Any]: try: response = await client.get( f"{prom_url}/api/v1/query", params={"query": query}, ) data = response.json() if data.get("status") != "success": return {"status": "error", "reason": "prometheus_query_failed"} results = data.get("data", {}).get("result", []) if not results: return { "status": "no_data", "reason": "prometheus_empty_result_metric_not_emitted", } raw_value = results[0]["value"][1] value = float(raw_value) if not math.isfinite(value): return { "status": "skipped", "reason": "prometheus_nan_or_inf", "raw_value": raw_value, } return {"status": "ok", "value": value} except Exception as exc: logger.warning("adr100_slo_prometheus_query_error", query=query, error=str(exc)) return {"status": "error", "reason": "prometheus_query_error"} def _metric_payload( definition: Adr100SloDefinition, *, value: float | None, status: str, reason: str | None, denominator_value: float | None, sample_count: float | None, ) -> dict[str, Any]: return { "name": definition.name, "query": definition.query, "value": value, "target": definition.target, "hard_red_line": definition.hard_red_line, "direction": definition.direction, "unit": definition.unit, "window": definition.window, "status": status, "evaluable": status in {"ok", "warning", "violated"}, "reason": reason, "denominator_query": definition.denominator_query, "denominator_value": denominator_value, "sample_count": sample_count, } def _classify_status(value: float, definition: Adr100SloDefinition) -> str: if definition.direction == "above": if value < definition.hard_red_line: return "violated" if value < definition.target: return "warning" return "ok" if value > definition.hard_red_line: return "violated" if value > definition.target: return "warning" return "ok" def _build_verification_coverage_payload( summary_row: Any, recent_unverified_rows: Any, recent_non_success_rows: Any = (), ) -> dict[str, Any]: row = dict(summary_row) total_auto = int(row.get("total_auto") or 0) verified_auto = int(row.get("verified_auto") or 0) verified_success = int(row.get("verified_success") or 0) verified_non_success = int(row.get("verified_non_success") or 0) unverified_auto = int(row.get("unverified_auto") or 0) if total_auto == 0: status = "skipped_low_volume" reason = "no_auto_repair_executions_24h" evaluable = False elif unverified_auto > 0: status = "warning" reason = "verification_backlog_present" evaluable = True elif verified_non_success > 0: status = "warning" reason = "non_success_verification_present" evaluable = True else: status = "ok" reason = None evaluable = True coverage_rate = (verified_auto / total_auto) if total_auto else None verification_success_rate = (verified_success / verified_auto) if verified_auto else None recent_non_success = [ _non_success_finding_payload(dict(raw)) for raw in recent_non_success_rows ] remediation_queue = _remediation_queue_payload(recent_non_success) return { "schema_version": "adr100_verification_coverage_v1", "source": "postgresql", "window": "24h", "status": status, "reason": reason, "evaluable": evaluable, "total_auto": total_auto, "successful_auto": int(row.get("successful_auto") or 0), "verified_auto": verified_auto, "verified_success": verified_success, "verified_non_success": verified_non_success, "unverified_auto": unverified_auto, "coverage_rate": coverage_rate, "verification_success_rate": verification_success_rate, "last_auto_at": _iso(row.get("last_auto_at")), "last_verified_auto_at": _iso(row.get("last_verified_auto_at")), "last_verification_evidence_at": _iso(row.get("last_verification_evidence_at")), "latest_auto_age_seconds": _int_or_none(row.get("latest_auto_age_seconds")), "last_verified_auto_age_seconds": _int_or_none(row.get("last_verified_auto_age_seconds")), "recent_unverified": [ { "id": str(item.get("id")), "incident_id": str(item.get("incident_id")), "success": bool(item.get("success")), "created_at": _iso(item.get("created_at")), } for item in (dict(raw) for raw in recent_unverified_rows) ], "recent_non_success": recent_non_success, "non_success_breakdown": { "by_verification_result": _count_breakdown( item["verification_result"] for item in recent_non_success ), "by_failure_class": _count_breakdown( item["failure_class"] for item in recent_non_success ), "by_remediation_status": _count_breakdown( item["remediation_status"] for item in remediation_queue["items"] ), }, "remediation_queue": remediation_queue, } def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]: failure_class = _classify_non_success_failure(row) remediation = _remediation_for_failure_class(failure_class) return { "auto_repair_id": str(row.get("auto_repair_id")), "incident_id": str(row.get("incident_id")), "incident_status": str(row.get("incident_status") or "unknown"), "incident_severity": str(row.get("incident_severity") or "unknown"), "alert_category": row.get("alert_category"), "alertname": row.get("alertname"), "auto_success": bool(row.get("auto_success")), "playbook_id": row.get("playbook_id"), "playbook_name": row.get("playbook_name"), "triggered_by": row.get("triggered_by"), "risk_level": row.get("risk_level"), "verification_result": str(row.get("verification_result") or "unknown"), "failure_class": failure_class, "next_step": _next_step_for_failure_class(failure_class), "remediation_status": remediation["status"], "remediation_action": remediation["action"], "remediation_owner": remediation["owner"], "remediation_reason": remediation["reason"], "auto_error_excerpt": _short_text(row.get("auto_error"), 180), "evidence_excerpt": _short_text(row.get("evidence_summary"), 180), "auto_created_at": _iso(row.get("auto_created_at")), "verification_collected_at": _iso(row.get("verification_collected_at")), } def _classify_non_success_failure(row: dict[str, Any]) -> str: combined = " ".join( str(row.get(key) or "") for key in ("auto_error", "post_state_text", "evidence_summary") ).lower() if "unsupported scheme" in combined: return "unsupported_action_scheme" if "missing_query_parameter" in combined: return "verifier_missing_promql" if "empty_pod_name" in combined: return "verifier_target_missing_pod" if not bool(row.get("auto_success")): return "auto_repair_execution_failed" if "mcp:ssh_diagnose" in combined or "ssh_diagnose" in combined: return "observe_only_playbook" result = str(row.get("verification_result") or "").lower() if result in {"failed", "timeout"}: return f"verification_{result}" return "verification_degraded" def _remediation_for_failure_class(failure_class: str) -> dict[str, str]: """Map a non-success verification class to a read-only remediation work item. This is dashboard triage metadata only. It does not auto-close incidents, replay repairs, or approve write actions. """ if failure_class == "unsupported_action_scheme": return { "status": "ready_for_replay", "action": "replay_with_supported_executor", "owner": "auto_repair_executor", "reason": "executor_gateway_available_after_t23", } if failure_class == "verifier_missing_promql": return { "status": "ready_for_reverify", "action": "reverify_with_promql_template", "owner": "post_execution_verifier", "reason": "promql_template_available_after_t23", } if failure_class == "verifier_target_missing_pod": return { "status": "needs_target_mapping", "action": "map_target_and_reverify", "owner": "post_execution_verifier", "reason": "verifier_target_missing", } if failure_class == "auto_repair_execution_failed": return { "status": "needs_playbook_ticket", "action": "create_playbook_ticket", "owner": "solver_or_operator", "reason": "execution_failed_after_route_normalization", } if failure_class == "observe_only_playbook": return { "status": "needs_playbook_ticket", "action": "promote_diagnostic_to_repair_playbook", "owner": "solver_or_operator", "reason": "auto_repair_only_collected_evidence", } if failure_class in {"verification_failed", "verification_timeout"}: return { "status": "manual_review", "action": "escalate_verification_failure", "owner": "sre_operator", "reason": "verifier_returned_hard_failure", } return { "status": "manual_review", "action": "inspect_degraded_evidence", "owner": "sre_operator", "reason": "degraded_evidence_requires_human_context", } def _next_step_for_failure_class(failure_class: str) -> str: if failure_class == "unsupported_action_scheme": return "normalize_playbook_executor" if failure_class == "verifier_missing_promql": return "add_verifier_query_template" if failure_class == "verifier_target_missing_pod": return "map_verifier_target" if failure_class == "auto_repair_execution_failed": return "review_auto_repair_execution" if failure_class == "observe_only_playbook": return "author_mutating_repair_step" if failure_class in {"verification_failed", "verification_timeout"}: return "escalate_verification_failure" return "review_degraded_verification" def _remediation_queue_payload(recent_non_success: list[dict[str, Any]]) -> dict[str, Any]: items: list[dict[str, Any]] = [] for item in recent_non_success: items.append({ "work_item_id": ( f"verification:{item.get('incident_id')}:{item.get('auto_repair_id')}" ), "incident_id": item.get("incident_id"), "auto_repair_id": item.get("auto_repair_id"), "alertname": item.get("alertname"), "playbook_id": item.get("playbook_id"), "failure_class": item.get("failure_class"), "verification_result": item.get("verification_result"), "remediation_status": item.get("remediation_status"), "remediation_action": item.get("remediation_action"), "remediation_owner": item.get("remediation_owner"), "remediation_reason": item.get("remediation_reason"), "source": "adr100_verification_coverage", "auto_created_at": item.get("auto_created_at"), "verification_collected_at": item.get("verification_collected_at"), }) ready_for_ai = sum( 1 for item in items if item.get("remediation_status") in {"ready_for_replay", "ready_for_reverify"} ) needs_human = sum( 1 for item in items if item.get("remediation_status") in { "needs_target_mapping", "needs_playbook_ticket", "manual_review", } ) return { "schema_version": "adr100_remediation_queue_v1", "source": "recent_non_success_read_model", "total": len(items), "ready_for_ai": ready_for_ai, "needs_human": needs_human, "items": items, "by_status": _count_breakdown( item.get("remediation_status") for item in items ), "by_action": _count_breakdown( item.get("remediation_action") for item in items ), } def _count_breakdown(values: Any) -> list[dict[str, Any]]: counts: dict[str, int] = {} for value in values: key = str(value or "unknown") counts[key] = counts.get(key, 0) + 1 return [ {"name": name, "count": count} for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0])) ] def _short_text(value: Any, limit: int) -> str | None: if value is None: return None text = " ".join(str(value).split()) if not text: return None return text[:limit] def _iso(value: Any) -> str | None: return value.isoformat() if hasattr(value, "isoformat") else None def _int_or_none(value: Any) -> int | None: return int(value) if value is not None else None def _overall_status( metrics: list[dict[str, Any]], evaluable: list[dict[str, Any]], verification_coverage: dict[str, Any] | None = None, ) -> str: if any(metric.get("status") == "violated" for metric in metrics): return "violated" if verification_coverage and verification_coverage.get("status") in {"violated", "warning"}: return str(verification_coverage["status"]) if any(metric.get("status") == "warning" for metric in metrics): return "warning" if evaluable and any(metric.get("status") == "skipped_low_volume" for metric in metrics): return "partial" if evaluable: return "ok" if any(metric.get("status") == "no_data" for metric in metrics): return "no_data" return "skipped_low_volume" _adr100_slo_status_service: Adr100SloStatusService | None = None def get_adr100_slo_status_service() -> Adr100SloStatusService: global _adr100_slo_status_service if _adr100_slo_status_service is None: _adr100_slo_status_service = Adr100SloStatusService() return _adr100_slo_status_service