765 lines
27 KiB
Python
765 lines
27 KiB
Python
"""
|
|
Read-only ADR-100 SLO status snapshot.
|
|
|
|
GovernanceAgent.check_slo_compliance() can emit governance alerts when an SLO is
|
|
violated. This service is intentionally read-only so dashboards can show the
|
|
same Prometheus-backed state without producing Telegram/DB side effects.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import structlog
|
|
from sqlalchemy import text
|
|
|
|
from src.core.config import settings
|
|
from src.db.base import get_db_context
|
|
from src.utils.timezone import now_taipei_iso
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Adr100SloDefinition:
|
|
name: str
|
|
query: str
|
|
target: float
|
|
hard_red_line: float
|
|
direction: str
|
|
unit: str
|
|
window: str
|
|
denominator_query: str | None = None
|
|
denominator_window_seconds: int = 0
|
|
minimum_events: float = 1.0
|
|
|
|
|
|
ADR100_SLO_DEFINITIONS: tuple[Adr100SloDefinition, ...] = (
|
|
Adr100SloDefinition(
|
|
name="autonomy_rate",
|
|
query="sli:autonomy_rate:5m",
|
|
target=0.80,
|
|
hard_red_line=0.70,
|
|
direction="above",
|
|
unit="percent",
|
|
window="5m",
|
|
denominator_query="sum(rate(automation_operation_log_total[5m]))",
|
|
denominator_window_seconds=300,
|
|
),
|
|
Adr100SloDefinition(
|
|
name="decision_accuracy",
|
|
query="sli:decision_accuracy:5m",
|
|
target=0.90,
|
|
hard_red_line=0.85,
|
|
direction="above",
|
|
unit="percent",
|
|
window="5m",
|
|
denominator_query='sum(rate(automation_operation_log_total{outcome="auto_executed"}[5m]))',
|
|
denominator_window_seconds=300,
|
|
),
|
|
Adr100SloDefinition(
|
|
name="confidence_calibration",
|
|
query="sli:confidence_calibration:1h",
|
|
target=0.80,
|
|
hard_red_line=0.70,
|
|
direction="above",
|
|
unit="percent",
|
|
window="1h",
|
|
denominator_query="sum(rate(approval_records_high_confidence_total[1h]))",
|
|
denominator_window_seconds=3600,
|
|
),
|
|
Adr100SloDefinition(
|
|
name="km_growth_rate",
|
|
query="max(knowledge_entries_created_24h) or max(sli:km_growth_rate:24h)",
|
|
target=20.0,
|
|
hard_red_line=5.0,
|
|
direction="above",
|
|
unit="count",
|
|
window="24h",
|
|
),
|
|
Adr100SloDefinition(
|
|
name="truth_chain_quality_summary_latency",
|
|
query='max(awooop_truth_chain_quality_summary_last_duration_seconds{project_id="awoooi",limit="8",success="true"})',
|
|
target=2.0,
|
|
hard_red_line=8.0,
|
|
direction="below",
|
|
unit="seconds",
|
|
window="last_observation",
|
|
minimum_events=0.0,
|
|
),
|
|
)
|
|
|
|
|
|
class Adr100SloStatusService:
|
|
"""Fetch ADR-100 SLO status from Prometheus without writing governance events."""
|
|
|
|
async def fetch_report(self) -> dict[str, Any]:
|
|
prom_url = getattr(
|
|
settings,
|
|
"PROMETHEUS_URL",
|
|
"http://prometheus.observability.svc:9090",
|
|
).rstrip("/")
|
|
metrics: list[dict[str, Any]] = []
|
|
|
|
async with httpx.AsyncClient(timeout=5.0) as client:
|
|
for definition in ADR100_SLO_DEFINITIONS:
|
|
metrics.append(await self._fetch_metric(client, prom_url, definition))
|
|
|
|
evaluable = [metric for metric in metrics if metric.get("evaluable")]
|
|
ok_count = sum(1 for metric in evaluable if metric.get("status") == "ok")
|
|
overall_compliance = (ok_count / len(evaluable)) if evaluable else None
|
|
verification_coverage = await self._fetch_verification_coverage()
|
|
overall_status = _overall_status(metrics, evaluable, verification_coverage)
|
|
|
|
return {
|
|
"schema_version": "adr100_slo_status_v1",
|
|
"source": "prometheus+postgresql",
|
|
"evaluated_at": now_taipei_iso(),
|
|
"overall_status": overall_status,
|
|
"overall_compliance": overall_compliance,
|
|
"evaluable_count": len(evaluable),
|
|
"metric_count": len(metrics),
|
|
"metrics": metrics,
|
|
"verification_coverage": verification_coverage,
|
|
}
|
|
|
|
async def _fetch_metric(
|
|
self,
|
|
client: httpx.AsyncClient,
|
|
prom_url: str,
|
|
definition: Adr100SloDefinition,
|
|
) -> dict[str, Any]:
|
|
denominator_value: float | None = None
|
|
sample_count: float | None = None
|
|
|
|
if definition.denominator_query:
|
|
denominator_result = await _query_prometheus_value(
|
|
client,
|
|
prom_url,
|
|
definition.denominator_query,
|
|
)
|
|
if denominator_result["status"] != "ok":
|
|
return _metric_payload(
|
|
definition,
|
|
value=None,
|
|
status="no_data",
|
|
reason=denominator_result["reason"],
|
|
denominator_value=None,
|
|
sample_count=None,
|
|
)
|
|
|
|
denominator_value = float(denominator_result["value"])
|
|
sample_count = denominator_value * definition.denominator_window_seconds
|
|
if sample_count < definition.minimum_events:
|
|
return _metric_payload(
|
|
definition,
|
|
value=None,
|
|
status="skipped_low_volume",
|
|
reason="denominator_below_minimum_events",
|
|
denominator_value=denominator_value,
|
|
sample_count=sample_count,
|
|
)
|
|
|
|
value_result = await _query_prometheus_value(client, prom_url, definition.query)
|
|
if value_result["status"] != "ok":
|
|
status = (
|
|
"skipped_low_volume"
|
|
if value_result["reason"] == "prometheus_nan_or_inf"
|
|
else "no_data"
|
|
)
|
|
return _metric_payload(
|
|
definition,
|
|
value=None,
|
|
status=status,
|
|
reason=value_result["reason"],
|
|
denominator_value=denominator_value,
|
|
sample_count=sample_count,
|
|
)
|
|
|
|
value = float(value_result["value"])
|
|
status = _classify_status(value, definition)
|
|
return _metric_payload(
|
|
definition,
|
|
value=value,
|
|
status=status,
|
|
reason=None,
|
|
denominator_value=denominator_value,
|
|
sample_count=sample_count if sample_count is not None else value,
|
|
)
|
|
|
|
async def _fetch_verification_coverage(self) -> dict[str, Any]:
|
|
"""Summarize whether recent auto-repair executions have verifier evidence."""
|
|
try:
|
|
async with get_db_context() as db:
|
|
summary_row = (
|
|
await db.execute(text(_VERIFICATION_COVERAGE_SQL))
|
|
).mappings().one()
|
|
recent_rows = (
|
|
await db.execute(text(_VERIFICATION_COVERAGE_RECENT_SQL))
|
|
).mappings().all()
|
|
recent_non_success_rows = (
|
|
await db.execute(text(_VERIFICATION_COVERAGE_NON_SUCCESS_SQL))
|
|
).mappings().all()
|
|
except Exception as exc:
|
|
logger.warning("adr100_verification_coverage_query_error", error=str(exc))
|
|
return {
|
|
"schema_version": "adr100_verification_coverage_v1",
|
|
"source": "postgresql",
|
|
"window": "24h",
|
|
"status": "error",
|
|
"reason": "postgresql_query_error",
|
|
"evaluable": False,
|
|
"total_auto": 0,
|
|
"successful_auto": 0,
|
|
"verified_auto": 0,
|
|
"verified_success": 0,
|
|
"verified_non_success": 0,
|
|
"unverified_auto": 0,
|
|
"coverage_rate": None,
|
|
"verification_success_rate": None,
|
|
"last_auto_at": None,
|
|
"last_verified_auto_at": None,
|
|
"last_verification_evidence_at": None,
|
|
"latest_auto_age_seconds": None,
|
|
"last_verified_auto_age_seconds": None,
|
|
"recent_unverified": [],
|
|
"recent_non_success": [],
|
|
"non_success_breakdown": {
|
|
"by_verification_result": [],
|
|
"by_failure_class": [],
|
|
},
|
|
"remediation_queue": _remediation_queue_payload([]),
|
|
}
|
|
|
|
return _build_verification_coverage_payload(
|
|
summary_row,
|
|
recent_rows,
|
|
recent_non_success_rows,
|
|
)
|
|
|
|
|
|
_VERIFICATION_COVERAGE_SQL = """
|
|
WITH recent_auto AS (
|
|
SELECT id, incident_id, success, created_at
|
|
FROM auto_repair_executions
|
|
WHERE created_at >= NOW() - INTERVAL '24 hours'
|
|
),
|
|
per_auto AS (
|
|
SELECT
|
|
are.id,
|
|
are.incident_id,
|
|
are.success,
|
|
are.created_at,
|
|
latest.verification_result,
|
|
latest.collected_at AS verification_collected_at,
|
|
latest.self_healing_score
|
|
FROM recent_auto are
|
|
LEFT JOIN LATERAL (
|
|
SELECT ev.verification_result, ev.collected_at, ev.self_healing_score
|
|
FROM incident_evidence ev
|
|
WHERE ev.incident_id = are.incident_id
|
|
AND ev.verification_result IS NOT NULL
|
|
ORDER BY ev.collected_at DESC
|
|
LIMIT 1
|
|
) latest ON TRUE
|
|
)
|
|
SELECT
|
|
count(*)::int AS total_auto,
|
|
count(*) FILTER (WHERE success)::int AS successful_auto,
|
|
count(*) FILTER (WHERE verification_result IS NOT NULL)::int AS verified_auto,
|
|
count(*) FILTER (WHERE verification_result = 'success')::int AS verified_success,
|
|
count(*) FILTER (WHERE verification_result IN ('degraded','failed','timeout'))::int AS verified_non_success,
|
|
count(*) FILTER (WHERE verification_result IS NULL)::int AS unverified_auto,
|
|
max(created_at) AS last_auto_at,
|
|
max(created_at) FILTER (WHERE verification_result IS NOT NULL) AS last_verified_auto_at,
|
|
max(verification_collected_at) AS last_verification_evidence_at,
|
|
EXTRACT(EPOCH FROM (NOW() - max(created_at)))::int AS latest_auto_age_seconds,
|
|
EXTRACT(EPOCH FROM (NOW() - (max(created_at) FILTER (WHERE verification_result IS NOT NULL))))::int
|
|
AS last_verified_auto_age_seconds
|
|
FROM per_auto
|
|
"""
|
|
|
|
|
|
_VERIFICATION_COVERAGE_RECENT_SQL = """
|
|
WITH recent_auto AS (
|
|
SELECT id, incident_id, success, created_at
|
|
FROM auto_repair_executions
|
|
WHERE created_at >= NOW() - INTERVAL '24 hours'
|
|
),
|
|
per_auto AS (
|
|
SELECT
|
|
are.id,
|
|
are.incident_id,
|
|
are.success,
|
|
are.created_at,
|
|
latest.verification_result
|
|
FROM recent_auto are
|
|
LEFT JOIN LATERAL (
|
|
SELECT ev.verification_result
|
|
FROM incident_evidence ev
|
|
WHERE ev.incident_id = are.incident_id
|
|
AND ev.verification_result IS NOT NULL
|
|
ORDER BY ev.collected_at DESC
|
|
LIMIT 1
|
|
) latest ON TRUE
|
|
)
|
|
SELECT id, incident_id, success, created_at
|
|
FROM per_auto
|
|
WHERE verification_result IS NULL
|
|
ORDER BY created_at DESC
|
|
LIMIT 5
|
|
"""
|
|
|
|
|
|
_VERIFICATION_COVERAGE_NON_SUCCESS_SQL = """
|
|
WITH recent_auto AS (
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
success,
|
|
playbook_id,
|
|
playbook_name,
|
|
triggered_by,
|
|
risk_level,
|
|
error_message,
|
|
created_at
|
|
FROM auto_repair_executions
|
|
WHERE created_at >= NOW() - INTERVAL '24 hours'
|
|
),
|
|
per_auto AS (
|
|
SELECT
|
|
are.id AS auto_repair_id,
|
|
are.incident_id,
|
|
are.success AS auto_success,
|
|
are.playbook_id,
|
|
are.playbook_name,
|
|
are.triggered_by,
|
|
are.risk_level,
|
|
left(coalesce(are.error_message, ''), 240) AS auto_error,
|
|
are.created_at AS auto_created_at,
|
|
latest.verification_result,
|
|
latest.collected_at AS verification_collected_at,
|
|
left(coalesce(latest.post_execution_state::text, ''), 700) AS post_state_text,
|
|
left(coalesce(latest.evidence_summary, ''), 300) AS evidence_summary
|
|
FROM recent_auto are
|
|
LEFT JOIN LATERAL (
|
|
SELECT
|
|
ev.verification_result,
|
|
ev.collected_at,
|
|
ev.post_execution_state,
|
|
ev.evidence_summary
|
|
FROM incident_evidence ev
|
|
WHERE ev.incident_id = are.incident_id
|
|
AND ev.verification_result IS NOT NULL
|
|
ORDER BY ev.collected_at DESC
|
|
LIMIT 1
|
|
) latest ON TRUE
|
|
)
|
|
SELECT
|
|
p.*,
|
|
i.status::text AS incident_status,
|
|
i.severity::text AS incident_severity,
|
|
i.alert_category,
|
|
i.alertname
|
|
FROM per_auto p
|
|
LEFT JOIN incidents i ON i.incident_id = p.incident_id
|
|
WHERE p.verification_result IS NOT NULL
|
|
AND p.verification_result <> 'success'
|
|
ORDER BY p.auto_created_at DESC
|
|
LIMIT 8
|
|
"""
|
|
|
|
|
|
async def _query_prometheus_value(
|
|
client: httpx.AsyncClient,
|
|
prom_url: str,
|
|
query: str,
|
|
) -> dict[str, Any]:
|
|
try:
|
|
response = await client.get(
|
|
f"{prom_url}/api/v1/query",
|
|
params={"query": query},
|
|
)
|
|
data = response.json()
|
|
if data.get("status") != "success":
|
|
return {"status": "error", "reason": "prometheus_query_failed"}
|
|
|
|
results = data.get("data", {}).get("result", [])
|
|
if not results:
|
|
return {
|
|
"status": "no_data",
|
|
"reason": "prometheus_empty_result_metric_not_emitted",
|
|
}
|
|
|
|
raw_value = results[0]["value"][1]
|
|
value = float(raw_value)
|
|
if not math.isfinite(value):
|
|
return {
|
|
"status": "skipped",
|
|
"reason": "prometheus_nan_or_inf",
|
|
"raw_value": raw_value,
|
|
}
|
|
return {"status": "ok", "value": value}
|
|
except Exception as exc:
|
|
logger.warning("adr100_slo_prometheus_query_error", query=query, error=str(exc))
|
|
return {"status": "error", "reason": "prometheus_query_error"}
|
|
|
|
|
|
def _metric_payload(
|
|
definition: Adr100SloDefinition,
|
|
*,
|
|
value: float | None,
|
|
status: str,
|
|
reason: str | None,
|
|
denominator_value: float | None,
|
|
sample_count: float | None,
|
|
) -> dict[str, Any]:
|
|
return {
|
|
"name": definition.name,
|
|
"query": definition.query,
|
|
"value": value,
|
|
"target": definition.target,
|
|
"hard_red_line": definition.hard_red_line,
|
|
"direction": definition.direction,
|
|
"unit": definition.unit,
|
|
"window": definition.window,
|
|
"status": status,
|
|
"evaluable": status in {"ok", "warning", "violated"},
|
|
"reason": reason,
|
|
"denominator_query": definition.denominator_query,
|
|
"denominator_value": denominator_value,
|
|
"sample_count": sample_count,
|
|
}
|
|
|
|
|
|
def _classify_status(value: float, definition: Adr100SloDefinition) -> str:
|
|
if definition.direction == "above":
|
|
if value < definition.hard_red_line:
|
|
return "violated"
|
|
if value < definition.target:
|
|
return "warning"
|
|
return "ok"
|
|
|
|
if value > definition.hard_red_line:
|
|
return "violated"
|
|
if value > definition.target:
|
|
return "warning"
|
|
return "ok"
|
|
|
|
|
|
def _build_verification_coverage_payload(
|
|
summary_row: Any,
|
|
recent_unverified_rows: Any,
|
|
recent_non_success_rows: Any = (),
|
|
) -> dict[str, Any]:
|
|
row = dict(summary_row)
|
|
total_auto = int(row.get("total_auto") or 0)
|
|
verified_auto = int(row.get("verified_auto") or 0)
|
|
verified_success = int(row.get("verified_success") or 0)
|
|
verified_non_success = int(row.get("verified_non_success") or 0)
|
|
unverified_auto = int(row.get("unverified_auto") or 0)
|
|
|
|
if total_auto == 0:
|
|
status = "skipped_low_volume"
|
|
reason = "no_auto_repair_executions_24h"
|
|
evaluable = False
|
|
elif unverified_auto > 0:
|
|
status = "warning"
|
|
reason = "verification_backlog_present"
|
|
evaluable = True
|
|
elif verified_non_success > 0:
|
|
status = "warning"
|
|
reason = "non_success_verification_present"
|
|
evaluable = True
|
|
else:
|
|
status = "ok"
|
|
reason = None
|
|
evaluable = True
|
|
|
|
coverage_rate = (verified_auto / total_auto) if total_auto else None
|
|
verification_success_rate = (verified_success / verified_auto) if verified_auto else None
|
|
recent_non_success = [
|
|
_non_success_finding_payload(dict(raw))
|
|
for raw in recent_non_success_rows
|
|
]
|
|
remediation_queue = _remediation_queue_payload(recent_non_success)
|
|
|
|
return {
|
|
"schema_version": "adr100_verification_coverage_v1",
|
|
"source": "postgresql",
|
|
"window": "24h",
|
|
"status": status,
|
|
"reason": reason,
|
|
"evaluable": evaluable,
|
|
"total_auto": total_auto,
|
|
"successful_auto": int(row.get("successful_auto") or 0),
|
|
"verified_auto": verified_auto,
|
|
"verified_success": verified_success,
|
|
"verified_non_success": verified_non_success,
|
|
"unverified_auto": unverified_auto,
|
|
"coverage_rate": coverage_rate,
|
|
"verification_success_rate": verification_success_rate,
|
|
"last_auto_at": _iso(row.get("last_auto_at")),
|
|
"last_verified_auto_at": _iso(row.get("last_verified_auto_at")),
|
|
"last_verification_evidence_at": _iso(row.get("last_verification_evidence_at")),
|
|
"latest_auto_age_seconds": _int_or_none(row.get("latest_auto_age_seconds")),
|
|
"last_verified_auto_age_seconds": _int_or_none(row.get("last_verified_auto_age_seconds")),
|
|
"recent_unverified": [
|
|
{
|
|
"id": str(item.get("id")),
|
|
"incident_id": str(item.get("incident_id")),
|
|
"success": bool(item.get("success")),
|
|
"created_at": _iso(item.get("created_at")),
|
|
}
|
|
for item in (dict(raw) for raw in recent_unverified_rows)
|
|
],
|
|
"recent_non_success": recent_non_success,
|
|
"non_success_breakdown": {
|
|
"by_verification_result": _count_breakdown(
|
|
item["verification_result"] for item in recent_non_success
|
|
),
|
|
"by_failure_class": _count_breakdown(
|
|
item["failure_class"] for item in recent_non_success
|
|
),
|
|
"by_remediation_status": _count_breakdown(
|
|
item["remediation_status"] for item in remediation_queue["items"]
|
|
),
|
|
},
|
|
"remediation_queue": remediation_queue,
|
|
}
|
|
|
|
|
|
def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]:
|
|
failure_class = _classify_non_success_failure(row)
|
|
remediation = _remediation_for_failure_class(failure_class)
|
|
return {
|
|
"auto_repair_id": str(row.get("auto_repair_id")),
|
|
"incident_id": str(row.get("incident_id")),
|
|
"incident_status": str(row.get("incident_status") or "unknown"),
|
|
"incident_severity": str(row.get("incident_severity") or "unknown"),
|
|
"alert_category": row.get("alert_category"),
|
|
"alertname": row.get("alertname"),
|
|
"auto_success": bool(row.get("auto_success")),
|
|
"playbook_id": row.get("playbook_id"),
|
|
"playbook_name": row.get("playbook_name"),
|
|
"triggered_by": row.get("triggered_by"),
|
|
"risk_level": row.get("risk_level"),
|
|
"verification_result": str(row.get("verification_result") or "unknown"),
|
|
"failure_class": failure_class,
|
|
"next_step": _next_step_for_failure_class(failure_class),
|
|
"remediation_status": remediation["status"],
|
|
"remediation_action": remediation["action"],
|
|
"remediation_owner": remediation["owner"],
|
|
"remediation_reason": remediation["reason"],
|
|
"auto_error_excerpt": _short_text(row.get("auto_error"), 180),
|
|
"evidence_excerpt": _short_text(row.get("evidence_summary"), 180),
|
|
"auto_created_at": _iso(row.get("auto_created_at")),
|
|
"verification_collected_at": _iso(row.get("verification_collected_at")),
|
|
}
|
|
|
|
|
|
def _classify_non_success_failure(row: dict[str, Any]) -> str:
|
|
combined = " ".join(
|
|
str(row.get(key) or "")
|
|
for key in ("auto_error", "post_state_text", "evidence_summary")
|
|
).lower()
|
|
if "unsupported scheme" in combined:
|
|
return "unsupported_action_scheme"
|
|
if "missing_query_parameter" in combined:
|
|
return "verifier_missing_promql"
|
|
if "empty_pod_name" in combined:
|
|
return "verifier_target_missing_pod"
|
|
if not bool(row.get("auto_success")):
|
|
return "auto_repair_execution_failed"
|
|
if "mcp:ssh_diagnose" in combined or "ssh_diagnose" in combined:
|
|
return "observe_only_playbook"
|
|
|
|
result = str(row.get("verification_result") or "").lower()
|
|
if result in {"failed", "timeout"}:
|
|
return f"verification_{result}"
|
|
return "verification_degraded"
|
|
|
|
|
|
def _remediation_for_failure_class(failure_class: str) -> dict[str, str]:
|
|
"""Map a non-success verification class to a read-only remediation work item.
|
|
|
|
This is dashboard triage metadata only. It does not auto-close incidents,
|
|
replay repairs, or approve write actions.
|
|
"""
|
|
if failure_class == "unsupported_action_scheme":
|
|
return {
|
|
"status": "ready_for_replay",
|
|
"action": "replay_with_supported_executor",
|
|
"owner": "auto_repair_executor",
|
|
"reason": "executor_gateway_available_after_t23",
|
|
}
|
|
if failure_class == "verifier_missing_promql":
|
|
return {
|
|
"status": "ready_for_reverify",
|
|
"action": "reverify_with_promql_template",
|
|
"owner": "post_execution_verifier",
|
|
"reason": "promql_template_available_after_t23",
|
|
}
|
|
if failure_class == "verifier_target_missing_pod":
|
|
return {
|
|
"status": "needs_target_mapping",
|
|
"action": "map_target_and_reverify",
|
|
"owner": "post_execution_verifier",
|
|
"reason": "verifier_target_missing",
|
|
}
|
|
if failure_class == "auto_repair_execution_failed":
|
|
return {
|
|
"status": "needs_playbook_ticket",
|
|
"action": "create_playbook_ticket",
|
|
"owner": "solver_or_operator",
|
|
"reason": "execution_failed_after_route_normalization",
|
|
}
|
|
if failure_class == "observe_only_playbook":
|
|
return {
|
|
"status": "needs_playbook_ticket",
|
|
"action": "promote_diagnostic_to_repair_playbook",
|
|
"owner": "solver_or_operator",
|
|
"reason": "auto_repair_only_collected_evidence",
|
|
}
|
|
if failure_class in {"verification_failed", "verification_timeout"}:
|
|
return {
|
|
"status": "manual_review",
|
|
"action": "escalate_verification_failure",
|
|
"owner": "sre_operator",
|
|
"reason": "verifier_returned_hard_failure",
|
|
}
|
|
return {
|
|
"status": "manual_review",
|
|
"action": "inspect_degraded_evidence",
|
|
"owner": "sre_operator",
|
|
"reason": "degraded_evidence_requires_human_context",
|
|
}
|
|
|
|
|
|
def _next_step_for_failure_class(failure_class: str) -> str:
|
|
if failure_class == "unsupported_action_scheme":
|
|
return "normalize_playbook_executor"
|
|
if failure_class == "verifier_missing_promql":
|
|
return "add_verifier_query_template"
|
|
if failure_class == "verifier_target_missing_pod":
|
|
return "map_verifier_target"
|
|
if failure_class == "auto_repair_execution_failed":
|
|
return "review_auto_repair_execution"
|
|
if failure_class == "observe_only_playbook":
|
|
return "author_mutating_repair_step"
|
|
if failure_class in {"verification_failed", "verification_timeout"}:
|
|
return "escalate_verification_failure"
|
|
return "review_degraded_verification"
|
|
|
|
|
|
def _remediation_queue_payload(recent_non_success: list[dict[str, Any]]) -> dict[str, Any]:
|
|
items: list[dict[str, Any]] = []
|
|
for item in recent_non_success:
|
|
items.append({
|
|
"work_item_id": (
|
|
f"verification:{item.get('incident_id')}:{item.get('auto_repair_id')}"
|
|
),
|
|
"incident_id": item.get("incident_id"),
|
|
"auto_repair_id": item.get("auto_repair_id"),
|
|
"alertname": item.get("alertname"),
|
|
"playbook_id": item.get("playbook_id"),
|
|
"failure_class": item.get("failure_class"),
|
|
"verification_result": item.get("verification_result"),
|
|
"remediation_status": item.get("remediation_status"),
|
|
"remediation_action": item.get("remediation_action"),
|
|
"remediation_owner": item.get("remediation_owner"),
|
|
"remediation_reason": item.get("remediation_reason"),
|
|
"source": "adr100_verification_coverage",
|
|
"auto_created_at": item.get("auto_created_at"),
|
|
"verification_collected_at": item.get("verification_collected_at"),
|
|
})
|
|
|
|
ready_for_ai = sum(
|
|
1 for item in items
|
|
if item.get("remediation_status") in {"ready_for_replay", "ready_for_reverify"}
|
|
)
|
|
needs_human = sum(
|
|
1 for item in items
|
|
if item.get("remediation_status") in {
|
|
"needs_target_mapping",
|
|
"needs_playbook_ticket",
|
|
"manual_review",
|
|
}
|
|
)
|
|
|
|
return {
|
|
"schema_version": "adr100_remediation_queue_v1",
|
|
"source": "recent_non_success_read_model",
|
|
"total": len(items),
|
|
"ready_for_ai": ready_for_ai,
|
|
"needs_human": needs_human,
|
|
"items": items,
|
|
"by_status": _count_breakdown(
|
|
item.get("remediation_status") for item in items
|
|
),
|
|
"by_action": _count_breakdown(
|
|
item.get("remediation_action") for item in items
|
|
),
|
|
}
|
|
|
|
|
|
def _count_breakdown(values: Any) -> list[dict[str, Any]]:
|
|
counts: dict[str, int] = {}
|
|
for value in values:
|
|
key = str(value or "unknown")
|
|
counts[key] = counts.get(key, 0) + 1
|
|
return [
|
|
{"name": name, "count": count}
|
|
for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
|
|
]
|
|
|
|
|
|
def _short_text(value: Any, limit: int) -> str | None:
|
|
if value is None:
|
|
return None
|
|
text = " ".join(str(value).split())
|
|
if not text:
|
|
return None
|
|
return text[:limit]
|
|
|
|
|
|
def _iso(value: Any) -> str | None:
|
|
return value.isoformat() if hasattr(value, "isoformat") else None
|
|
|
|
|
|
def _int_or_none(value: Any) -> int | None:
|
|
return int(value) if value is not None else None
|
|
|
|
|
|
def _overall_status(
|
|
metrics: list[dict[str, Any]],
|
|
evaluable: list[dict[str, Any]],
|
|
verification_coverage: dict[str, Any] | None = None,
|
|
) -> str:
|
|
if any(metric.get("status") == "violated" for metric in metrics):
|
|
return "violated"
|
|
if verification_coverage and verification_coverage.get("status") in {"violated", "warning"}:
|
|
return str(verification_coverage["status"])
|
|
if any(metric.get("status") == "warning" for metric in metrics):
|
|
return "warning"
|
|
if evaluable and any(metric.get("status") == "skipped_low_volume" for metric in metrics):
|
|
return "partial"
|
|
if evaluable:
|
|
return "ok"
|
|
if any(metric.get("status") == "no_data" for metric in metrics):
|
|
return "no_data"
|
|
return "skipped_low_volume"
|
|
|
|
|
|
_adr100_slo_status_service: Adr100SloStatusService | None = None
|
|
|
|
|
|
def get_adr100_slo_status_service() -> Adr100SloStatusService:
|
|
global _adr100_slo_status_service
|
|
if _adr100_slo_status_service is None:
|
|
_adr100_slo_status_service = Adr100SloStatusService()
|
|
return _adr100_slo_status_service
|