Files
awoooi/apps/api/src/services/adr100_slo_status_service.py
Your Name 7f3722c7f7
Some checks failed
CD Pipeline / tests (push) Successful in 1m22s
Code Review / ai-code-review (push) Successful in 13s
CD Pipeline / build-and-deploy (push) Successful in 4m10s
CD Pipeline / post-deploy-checks (push) Has been cancelled
fix(ai): improve docker repair verification signals
2026-06-01 19:27:36 +08:00

765 lines
27 KiB
Python

"""
Read-only ADR-100 SLO status snapshot.
GovernanceAgent.check_slo_compliance() can emit governance alerts when an SLO is
violated. This service is intentionally read-only so dashboards can show the
same Prometheus-backed state without producing Telegram/DB side effects.
"""
from __future__ import annotations
import math
from dataclasses import dataclass
from typing import Any
import httpx
import structlog
from sqlalchemy import text
from src.core.config import settings
from src.db.base import get_db_context
from src.utils.timezone import now_taipei_iso
logger = structlog.get_logger(__name__)
@dataclass(frozen=True)
class Adr100SloDefinition:
name: str
query: str
target: float
hard_red_line: float
direction: str
unit: str
window: str
denominator_query: str | None = None
denominator_window_seconds: int = 0
minimum_events: float = 1.0
ADR100_SLO_DEFINITIONS: tuple[Adr100SloDefinition, ...] = (
Adr100SloDefinition(
name="autonomy_rate",
query="sli:autonomy_rate:5m",
target=0.80,
hard_red_line=0.70,
direction="above",
unit="percent",
window="5m",
denominator_query="sum(rate(automation_operation_log_total[5m]))",
denominator_window_seconds=300,
),
Adr100SloDefinition(
name="decision_accuracy",
query="sli:decision_accuracy:5m",
target=0.90,
hard_red_line=0.85,
direction="above",
unit="percent",
window="5m",
denominator_query='sum(rate(automation_operation_log_total{outcome="auto_executed"}[5m]))',
denominator_window_seconds=300,
),
Adr100SloDefinition(
name="confidence_calibration",
query="sli:confidence_calibration:1h",
target=0.80,
hard_red_line=0.70,
direction="above",
unit="percent",
window="1h",
denominator_query="sum(rate(approval_records_high_confidence_total[1h]))",
denominator_window_seconds=3600,
),
Adr100SloDefinition(
name="km_growth_rate",
query="max(knowledge_entries_created_24h) or max(sli:km_growth_rate:24h)",
target=20.0,
hard_red_line=5.0,
direction="above",
unit="count",
window="24h",
),
Adr100SloDefinition(
name="truth_chain_quality_summary_latency",
query='max(awooop_truth_chain_quality_summary_last_duration_seconds{project_id="awoooi",limit="8",success="true"})',
target=2.0,
hard_red_line=8.0,
direction="below",
unit="seconds",
window="last_observation",
minimum_events=0.0,
),
)
class Adr100SloStatusService:
"""Fetch ADR-100 SLO status from Prometheus without writing governance events."""
async def fetch_report(self) -> dict[str, Any]:
prom_url = getattr(
settings,
"PROMETHEUS_URL",
"http://prometheus.observability.svc:9090",
).rstrip("/")
metrics: list[dict[str, Any]] = []
async with httpx.AsyncClient(timeout=5.0) as client:
for definition in ADR100_SLO_DEFINITIONS:
metrics.append(await self._fetch_metric(client, prom_url, definition))
evaluable = [metric for metric in metrics if metric.get("evaluable")]
ok_count = sum(1 for metric in evaluable if metric.get("status") == "ok")
overall_compliance = (ok_count / len(evaluable)) if evaluable else None
verification_coverage = await self._fetch_verification_coverage()
overall_status = _overall_status(metrics, evaluable, verification_coverage)
return {
"schema_version": "adr100_slo_status_v1",
"source": "prometheus+postgresql",
"evaluated_at": now_taipei_iso(),
"overall_status": overall_status,
"overall_compliance": overall_compliance,
"evaluable_count": len(evaluable),
"metric_count": len(metrics),
"metrics": metrics,
"verification_coverage": verification_coverage,
}
async def _fetch_metric(
self,
client: httpx.AsyncClient,
prom_url: str,
definition: Adr100SloDefinition,
) -> dict[str, Any]:
denominator_value: float | None = None
sample_count: float | None = None
if definition.denominator_query:
denominator_result = await _query_prometheus_value(
client,
prom_url,
definition.denominator_query,
)
if denominator_result["status"] != "ok":
return _metric_payload(
definition,
value=None,
status="no_data",
reason=denominator_result["reason"],
denominator_value=None,
sample_count=None,
)
denominator_value = float(denominator_result["value"])
sample_count = denominator_value * definition.denominator_window_seconds
if sample_count < definition.minimum_events:
return _metric_payload(
definition,
value=None,
status="skipped_low_volume",
reason="denominator_below_minimum_events",
denominator_value=denominator_value,
sample_count=sample_count,
)
value_result = await _query_prometheus_value(client, prom_url, definition.query)
if value_result["status"] != "ok":
status = (
"skipped_low_volume"
if value_result["reason"] == "prometheus_nan_or_inf"
else "no_data"
)
return _metric_payload(
definition,
value=None,
status=status,
reason=value_result["reason"],
denominator_value=denominator_value,
sample_count=sample_count,
)
value = float(value_result["value"])
status = _classify_status(value, definition)
return _metric_payload(
definition,
value=value,
status=status,
reason=None,
denominator_value=denominator_value,
sample_count=sample_count if sample_count is not None else value,
)
async def _fetch_verification_coverage(self) -> dict[str, Any]:
"""Summarize whether recent auto-repair executions have verifier evidence."""
try:
async with get_db_context() as db:
summary_row = (
await db.execute(text(_VERIFICATION_COVERAGE_SQL))
).mappings().one()
recent_rows = (
await db.execute(text(_VERIFICATION_COVERAGE_RECENT_SQL))
).mappings().all()
recent_non_success_rows = (
await db.execute(text(_VERIFICATION_COVERAGE_NON_SUCCESS_SQL))
).mappings().all()
except Exception as exc:
logger.warning("adr100_verification_coverage_query_error", error=str(exc))
return {
"schema_version": "adr100_verification_coverage_v1",
"source": "postgresql",
"window": "24h",
"status": "error",
"reason": "postgresql_query_error",
"evaluable": False,
"total_auto": 0,
"successful_auto": 0,
"verified_auto": 0,
"verified_success": 0,
"verified_non_success": 0,
"unverified_auto": 0,
"coverage_rate": None,
"verification_success_rate": None,
"last_auto_at": None,
"last_verified_auto_at": None,
"last_verification_evidence_at": None,
"latest_auto_age_seconds": None,
"last_verified_auto_age_seconds": None,
"recent_unverified": [],
"recent_non_success": [],
"non_success_breakdown": {
"by_verification_result": [],
"by_failure_class": [],
},
"remediation_queue": _remediation_queue_payload([]),
}
return _build_verification_coverage_payload(
summary_row,
recent_rows,
recent_non_success_rows,
)
_VERIFICATION_COVERAGE_SQL = """
WITH recent_auto AS (
SELECT id, incident_id, success, created_at
FROM auto_repair_executions
WHERE created_at >= NOW() - INTERVAL '24 hours'
),
per_auto AS (
SELECT
are.id,
are.incident_id,
are.success,
are.created_at,
latest.verification_result,
latest.collected_at AS verification_collected_at,
latest.self_healing_score
FROM recent_auto are
LEFT JOIN LATERAL (
SELECT ev.verification_result, ev.collected_at, ev.self_healing_score
FROM incident_evidence ev
WHERE ev.incident_id = are.incident_id
AND ev.verification_result IS NOT NULL
ORDER BY ev.collected_at DESC
LIMIT 1
) latest ON TRUE
)
SELECT
count(*)::int AS total_auto,
count(*) FILTER (WHERE success)::int AS successful_auto,
count(*) FILTER (WHERE verification_result IS NOT NULL)::int AS verified_auto,
count(*) FILTER (WHERE verification_result = 'success')::int AS verified_success,
count(*) FILTER (WHERE verification_result IN ('degraded','failed','timeout'))::int AS verified_non_success,
count(*) FILTER (WHERE verification_result IS NULL)::int AS unverified_auto,
max(created_at) AS last_auto_at,
max(created_at) FILTER (WHERE verification_result IS NOT NULL) AS last_verified_auto_at,
max(verification_collected_at) AS last_verification_evidence_at,
EXTRACT(EPOCH FROM (NOW() - max(created_at)))::int AS latest_auto_age_seconds,
EXTRACT(EPOCH FROM (NOW() - (max(created_at) FILTER (WHERE verification_result IS NOT NULL))))::int
AS last_verified_auto_age_seconds
FROM per_auto
"""
_VERIFICATION_COVERAGE_RECENT_SQL = """
WITH recent_auto AS (
SELECT id, incident_id, success, created_at
FROM auto_repair_executions
WHERE created_at >= NOW() - INTERVAL '24 hours'
),
per_auto AS (
SELECT
are.id,
are.incident_id,
are.success,
are.created_at,
latest.verification_result
FROM recent_auto are
LEFT JOIN LATERAL (
SELECT ev.verification_result
FROM incident_evidence ev
WHERE ev.incident_id = are.incident_id
AND ev.verification_result IS NOT NULL
ORDER BY ev.collected_at DESC
LIMIT 1
) latest ON TRUE
)
SELECT id, incident_id, success, created_at
FROM per_auto
WHERE verification_result IS NULL
ORDER BY created_at DESC
LIMIT 5
"""
_VERIFICATION_COVERAGE_NON_SUCCESS_SQL = """
WITH recent_auto AS (
SELECT
id,
incident_id,
success,
playbook_id,
playbook_name,
triggered_by,
risk_level,
error_message,
created_at
FROM auto_repair_executions
WHERE created_at >= NOW() - INTERVAL '24 hours'
),
per_auto AS (
SELECT
are.id AS auto_repair_id,
are.incident_id,
are.success AS auto_success,
are.playbook_id,
are.playbook_name,
are.triggered_by,
are.risk_level,
left(coalesce(are.error_message, ''), 240) AS auto_error,
are.created_at AS auto_created_at,
latest.verification_result,
latest.collected_at AS verification_collected_at,
left(coalesce(latest.post_execution_state::text, ''), 700) AS post_state_text,
left(coalesce(latest.evidence_summary, ''), 300) AS evidence_summary
FROM recent_auto are
LEFT JOIN LATERAL (
SELECT
ev.verification_result,
ev.collected_at,
ev.post_execution_state,
ev.evidence_summary
FROM incident_evidence ev
WHERE ev.incident_id = are.incident_id
AND ev.verification_result IS NOT NULL
ORDER BY ev.collected_at DESC
LIMIT 1
) latest ON TRUE
)
SELECT
p.*,
i.status::text AS incident_status,
i.severity::text AS incident_severity,
i.alert_category,
i.alertname
FROM per_auto p
LEFT JOIN incidents i ON i.incident_id = p.incident_id
WHERE p.verification_result IS NOT NULL
AND p.verification_result <> 'success'
ORDER BY p.auto_created_at DESC
LIMIT 8
"""
async def _query_prometheus_value(
client: httpx.AsyncClient,
prom_url: str,
query: str,
) -> dict[str, Any]:
try:
response = await client.get(
f"{prom_url}/api/v1/query",
params={"query": query},
)
data = response.json()
if data.get("status") != "success":
return {"status": "error", "reason": "prometheus_query_failed"}
results = data.get("data", {}).get("result", [])
if not results:
return {
"status": "no_data",
"reason": "prometheus_empty_result_metric_not_emitted",
}
raw_value = results[0]["value"][1]
value = float(raw_value)
if not math.isfinite(value):
return {
"status": "skipped",
"reason": "prometheus_nan_or_inf",
"raw_value": raw_value,
}
return {"status": "ok", "value": value}
except Exception as exc:
logger.warning("adr100_slo_prometheus_query_error", query=query, error=str(exc))
return {"status": "error", "reason": "prometheus_query_error"}
def _metric_payload(
definition: Adr100SloDefinition,
*,
value: float | None,
status: str,
reason: str | None,
denominator_value: float | None,
sample_count: float | None,
) -> dict[str, Any]:
return {
"name": definition.name,
"query": definition.query,
"value": value,
"target": definition.target,
"hard_red_line": definition.hard_red_line,
"direction": definition.direction,
"unit": definition.unit,
"window": definition.window,
"status": status,
"evaluable": status in {"ok", "warning", "violated"},
"reason": reason,
"denominator_query": definition.denominator_query,
"denominator_value": denominator_value,
"sample_count": sample_count,
}
def _classify_status(value: float, definition: Adr100SloDefinition) -> str:
if definition.direction == "above":
if value < definition.hard_red_line:
return "violated"
if value < definition.target:
return "warning"
return "ok"
if value > definition.hard_red_line:
return "violated"
if value > definition.target:
return "warning"
return "ok"
def _build_verification_coverage_payload(
summary_row: Any,
recent_unverified_rows: Any,
recent_non_success_rows: Any = (),
) -> dict[str, Any]:
row = dict(summary_row)
total_auto = int(row.get("total_auto") or 0)
verified_auto = int(row.get("verified_auto") or 0)
verified_success = int(row.get("verified_success") or 0)
verified_non_success = int(row.get("verified_non_success") or 0)
unverified_auto = int(row.get("unverified_auto") or 0)
if total_auto == 0:
status = "skipped_low_volume"
reason = "no_auto_repair_executions_24h"
evaluable = False
elif unverified_auto > 0:
status = "warning"
reason = "verification_backlog_present"
evaluable = True
elif verified_non_success > 0:
status = "warning"
reason = "non_success_verification_present"
evaluable = True
else:
status = "ok"
reason = None
evaluable = True
coverage_rate = (verified_auto / total_auto) if total_auto else None
verification_success_rate = (verified_success / verified_auto) if verified_auto else None
recent_non_success = [
_non_success_finding_payload(dict(raw))
for raw in recent_non_success_rows
]
remediation_queue = _remediation_queue_payload(recent_non_success)
return {
"schema_version": "adr100_verification_coverage_v1",
"source": "postgresql",
"window": "24h",
"status": status,
"reason": reason,
"evaluable": evaluable,
"total_auto": total_auto,
"successful_auto": int(row.get("successful_auto") or 0),
"verified_auto": verified_auto,
"verified_success": verified_success,
"verified_non_success": verified_non_success,
"unverified_auto": unverified_auto,
"coverage_rate": coverage_rate,
"verification_success_rate": verification_success_rate,
"last_auto_at": _iso(row.get("last_auto_at")),
"last_verified_auto_at": _iso(row.get("last_verified_auto_at")),
"last_verification_evidence_at": _iso(row.get("last_verification_evidence_at")),
"latest_auto_age_seconds": _int_or_none(row.get("latest_auto_age_seconds")),
"last_verified_auto_age_seconds": _int_or_none(row.get("last_verified_auto_age_seconds")),
"recent_unverified": [
{
"id": str(item.get("id")),
"incident_id": str(item.get("incident_id")),
"success": bool(item.get("success")),
"created_at": _iso(item.get("created_at")),
}
for item in (dict(raw) for raw in recent_unverified_rows)
],
"recent_non_success": recent_non_success,
"non_success_breakdown": {
"by_verification_result": _count_breakdown(
item["verification_result"] for item in recent_non_success
),
"by_failure_class": _count_breakdown(
item["failure_class"] for item in recent_non_success
),
"by_remediation_status": _count_breakdown(
item["remediation_status"] for item in remediation_queue["items"]
),
},
"remediation_queue": remediation_queue,
}
def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]:
failure_class = _classify_non_success_failure(row)
remediation = _remediation_for_failure_class(failure_class)
return {
"auto_repair_id": str(row.get("auto_repair_id")),
"incident_id": str(row.get("incident_id")),
"incident_status": str(row.get("incident_status") or "unknown"),
"incident_severity": str(row.get("incident_severity") or "unknown"),
"alert_category": row.get("alert_category"),
"alertname": row.get("alertname"),
"auto_success": bool(row.get("auto_success")),
"playbook_id": row.get("playbook_id"),
"playbook_name": row.get("playbook_name"),
"triggered_by": row.get("triggered_by"),
"risk_level": row.get("risk_level"),
"verification_result": str(row.get("verification_result") or "unknown"),
"failure_class": failure_class,
"next_step": _next_step_for_failure_class(failure_class),
"remediation_status": remediation["status"],
"remediation_action": remediation["action"],
"remediation_owner": remediation["owner"],
"remediation_reason": remediation["reason"],
"auto_error_excerpt": _short_text(row.get("auto_error"), 180),
"evidence_excerpt": _short_text(row.get("evidence_summary"), 180),
"auto_created_at": _iso(row.get("auto_created_at")),
"verification_collected_at": _iso(row.get("verification_collected_at")),
}
def _classify_non_success_failure(row: dict[str, Any]) -> str:
combined = " ".join(
str(row.get(key) or "")
for key in ("auto_error", "post_state_text", "evidence_summary")
).lower()
if "unsupported scheme" in combined:
return "unsupported_action_scheme"
if "missing_query_parameter" in combined:
return "verifier_missing_promql"
if "empty_pod_name" in combined:
return "verifier_target_missing_pod"
if not bool(row.get("auto_success")):
return "auto_repair_execution_failed"
if "mcp:ssh_diagnose" in combined or "ssh_diagnose" in combined:
return "observe_only_playbook"
result = str(row.get("verification_result") or "").lower()
if result in {"failed", "timeout"}:
return f"verification_{result}"
return "verification_degraded"
def _remediation_for_failure_class(failure_class: str) -> dict[str, str]:
"""Map a non-success verification class to a read-only remediation work item.
This is dashboard triage metadata only. It does not auto-close incidents,
replay repairs, or approve write actions.
"""
if failure_class == "unsupported_action_scheme":
return {
"status": "ready_for_replay",
"action": "replay_with_supported_executor",
"owner": "auto_repair_executor",
"reason": "executor_gateway_available_after_t23",
}
if failure_class == "verifier_missing_promql":
return {
"status": "ready_for_reverify",
"action": "reverify_with_promql_template",
"owner": "post_execution_verifier",
"reason": "promql_template_available_after_t23",
}
if failure_class == "verifier_target_missing_pod":
return {
"status": "needs_target_mapping",
"action": "map_target_and_reverify",
"owner": "post_execution_verifier",
"reason": "verifier_target_missing",
}
if failure_class == "auto_repair_execution_failed":
return {
"status": "needs_playbook_ticket",
"action": "create_playbook_ticket",
"owner": "solver_or_operator",
"reason": "execution_failed_after_route_normalization",
}
if failure_class == "observe_only_playbook":
return {
"status": "needs_playbook_ticket",
"action": "promote_diagnostic_to_repair_playbook",
"owner": "solver_or_operator",
"reason": "auto_repair_only_collected_evidence",
}
if failure_class in {"verification_failed", "verification_timeout"}:
return {
"status": "manual_review",
"action": "escalate_verification_failure",
"owner": "sre_operator",
"reason": "verifier_returned_hard_failure",
}
return {
"status": "manual_review",
"action": "inspect_degraded_evidence",
"owner": "sre_operator",
"reason": "degraded_evidence_requires_human_context",
}
def _next_step_for_failure_class(failure_class: str) -> str:
if failure_class == "unsupported_action_scheme":
return "normalize_playbook_executor"
if failure_class == "verifier_missing_promql":
return "add_verifier_query_template"
if failure_class == "verifier_target_missing_pod":
return "map_verifier_target"
if failure_class == "auto_repair_execution_failed":
return "review_auto_repair_execution"
if failure_class == "observe_only_playbook":
return "author_mutating_repair_step"
if failure_class in {"verification_failed", "verification_timeout"}:
return "escalate_verification_failure"
return "review_degraded_verification"
def _remediation_queue_payload(recent_non_success: list[dict[str, Any]]) -> dict[str, Any]:
items: list[dict[str, Any]] = []
for item in recent_non_success:
items.append({
"work_item_id": (
f"verification:{item.get('incident_id')}:{item.get('auto_repair_id')}"
),
"incident_id": item.get("incident_id"),
"auto_repair_id": item.get("auto_repair_id"),
"alertname": item.get("alertname"),
"playbook_id": item.get("playbook_id"),
"failure_class": item.get("failure_class"),
"verification_result": item.get("verification_result"),
"remediation_status": item.get("remediation_status"),
"remediation_action": item.get("remediation_action"),
"remediation_owner": item.get("remediation_owner"),
"remediation_reason": item.get("remediation_reason"),
"source": "adr100_verification_coverage",
"auto_created_at": item.get("auto_created_at"),
"verification_collected_at": item.get("verification_collected_at"),
})
ready_for_ai = sum(
1 for item in items
if item.get("remediation_status") in {"ready_for_replay", "ready_for_reverify"}
)
needs_human = sum(
1 for item in items
if item.get("remediation_status") in {
"needs_target_mapping",
"needs_playbook_ticket",
"manual_review",
}
)
return {
"schema_version": "adr100_remediation_queue_v1",
"source": "recent_non_success_read_model",
"total": len(items),
"ready_for_ai": ready_for_ai,
"needs_human": needs_human,
"items": items,
"by_status": _count_breakdown(
item.get("remediation_status") for item in items
),
"by_action": _count_breakdown(
item.get("remediation_action") for item in items
),
}
def _count_breakdown(values: Any) -> list[dict[str, Any]]:
counts: dict[str, int] = {}
for value in values:
key = str(value or "unknown")
counts[key] = counts.get(key, 0) + 1
return [
{"name": name, "count": count}
for name, count in sorted(counts.items(), key=lambda item: (-item[1], item[0]))
]
def _short_text(value: Any, limit: int) -> str | None:
if value is None:
return None
text = " ".join(str(value).split())
if not text:
return None
return text[:limit]
def _iso(value: Any) -> str | None:
return value.isoformat() if hasattr(value, "isoformat") else None
def _int_or_none(value: Any) -> int | None:
return int(value) if value is not None else None
def _overall_status(
metrics: list[dict[str, Any]],
evaluable: list[dict[str, Any]],
verification_coverage: dict[str, Any] | None = None,
) -> str:
if any(metric.get("status") == "violated" for metric in metrics):
return "violated"
if verification_coverage and verification_coverage.get("status") in {"violated", "warning"}:
return str(verification_coverage["status"])
if any(metric.get("status") == "warning" for metric in metrics):
return "warning"
if evaluable and any(metric.get("status") == "skipped_low_volume" for metric in metrics):
return "partial"
if evaluable:
return "ok"
if any(metric.get("status") == "no_data" for metric in metrics):
return "no_data"
return "skipped_low_volume"
_adr100_slo_status_service: Adr100SloStatusService | None = None
def get_adr100_slo_status_service() -> Adr100SloStatusService:
global _adr100_slo_status_service
if _adr100_slo_status_service is None:
_adr100_slo_status_service = Adr100SloStatusService()
return _adr100_slo_status_service