446 lines
17 KiB
Python
446 lines
17 KiB
Python
"""
|
|
ADR-100 SLO metrics emitter.
|
|
|
|
Prometheus recording rules for the AI flywheel SLOs expect a small set of
|
|
counter-like metrics. The source of truth already lives in PostgreSQL, so this
|
|
read-side emitter exposes DB totals on /metrics without changing runtime write
|
|
paths or introducing another state store.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass, field
|
|
from time import time
|
|
|
|
from sqlalchemy import text
|
|
|
|
from src.db.base import get_db_context
|
|
from src.services.awooop_truth_chain_service import get_quality_summary_observations
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class AutomationOperationSample:
|
|
outcome: str
|
|
operation_type: str
|
|
count: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class VerificationSample:
|
|
outcome: str
|
|
count: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class QualitySummaryObservation:
|
|
project_id: str
|
|
hours: int
|
|
limit: int
|
|
cache_status: str
|
|
success: bool
|
|
duration_seconds: float
|
|
observed_at: float
|
|
error: str | None = None
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Adr100SloMetricsSnapshot:
|
|
automation_operations: list[AutomationOperationSample] = field(default_factory=list)
|
|
automation_operations_24h: list[AutomationOperationSample] = field(default_factory=list)
|
|
post_execution_verifications: list[VerificationSample] = field(default_factory=list)
|
|
post_execution_verifications_24h: list[VerificationSample] = field(default_factory=list)
|
|
knowledge_entries_total: int = 0
|
|
knowledge_entries_created_24h: int = 0
|
|
high_confidence_total: int = 0
|
|
high_confidence_success_total: int = 0
|
|
quality_summary_observations: list[QualitySummaryObservation] = field(default_factory=list)
|
|
emitted_at: float = field(default_factory=time)
|
|
|
|
|
|
class Adr100SloMetricsService:
|
|
"""Build ADR-100 Prometheus samples from production DB state."""
|
|
|
|
async def to_prometheus_lines(self) -> str:
|
|
snapshot = await self.fetch_snapshot()
|
|
return render_adr100_slo_metrics(snapshot)
|
|
|
|
async def fetch_snapshot(self) -> Adr100SloMetricsSnapshot:
|
|
async with get_db_context() as db:
|
|
automation_rows = (
|
|
await db.execute(text(_AUTOMATION_OPERATION_SQL))
|
|
).fetchall()
|
|
automation_24h_rows = (
|
|
await db.execute(text(_AUTOMATION_OPERATION_24H_SQL))
|
|
).fetchall()
|
|
verification_rows = (
|
|
await db.execute(text(_POST_EXECUTION_VERIFICATION_SQL))
|
|
).fetchall()
|
|
verification_24h_rows = (
|
|
await db.execute(text(_POST_EXECUTION_VERIFICATION_24H_SQL))
|
|
).fetchall()
|
|
knowledge_total = int(
|
|
(await db.execute(text("SELECT count(*) FROM knowledge_entries"))).scalar()
|
|
or 0
|
|
)
|
|
knowledge_created_24h = int(
|
|
(
|
|
await db.execute(
|
|
text(
|
|
"""
|
|
SELECT count(*)
|
|
FROM knowledge_entries
|
|
WHERE created_at >= NOW() - INTERVAL '24 hours'
|
|
"""
|
|
)
|
|
)
|
|
).scalar()
|
|
or 0
|
|
)
|
|
confidence_row = (
|
|
await db.execute(text(_HIGH_CONFIDENCE_APPROVAL_SQL))
|
|
).one()
|
|
|
|
return Adr100SloMetricsSnapshot(
|
|
automation_operations=[
|
|
AutomationOperationSample(
|
|
outcome=str(row.outcome),
|
|
operation_type=str(row.operation_type),
|
|
count=int(row.count or 0),
|
|
)
|
|
for row in automation_rows
|
|
],
|
|
automation_operations_24h=[
|
|
AutomationOperationSample(
|
|
outcome=str(row.outcome),
|
|
operation_type=str(row.operation_type),
|
|
count=int(row.count or 0),
|
|
)
|
|
for row in automation_24h_rows
|
|
],
|
|
post_execution_verifications=[
|
|
VerificationSample(
|
|
outcome=str(row.outcome),
|
|
count=int(row.count or 0),
|
|
)
|
|
for row in verification_rows
|
|
],
|
|
post_execution_verifications_24h=[
|
|
VerificationSample(
|
|
outcome=str(row.outcome),
|
|
count=int(row.count or 0),
|
|
)
|
|
for row in verification_24h_rows
|
|
],
|
|
knowledge_entries_total=knowledge_total,
|
|
knowledge_entries_created_24h=knowledge_created_24h,
|
|
high_confidence_total=int(confidence_row.high_confidence_total or 0),
|
|
high_confidence_success_total=int(
|
|
confidence_row.high_confidence_success_total or 0
|
|
),
|
|
quality_summary_observations=[
|
|
QualitySummaryObservation(
|
|
project_id=str(row.get("project_id") or "awoooi"),
|
|
hours=int(row.get("hours") or 0),
|
|
limit=int(row.get("limit") or 0),
|
|
cache_status=str(row.get("cache_status") or "unknown"),
|
|
success=bool(row.get("success")),
|
|
duration_seconds=float(row.get("duration_seconds") or 0.0),
|
|
observed_at=float(row.get("observed_at") or 0.0),
|
|
error=(
|
|
str(row.get("error"))
|
|
if row.get("error") is not None
|
|
else None
|
|
),
|
|
)
|
|
for row in get_quality_summary_observations()
|
|
],
|
|
)
|
|
|
|
|
|
def render_adr100_slo_metrics(snapshot: Adr100SloMetricsSnapshot) -> str:
|
|
"""Render ADR-100 SLO metrics in Prometheus text exposition format."""
|
|
lines: list[str] = [
|
|
"",
|
|
"# HELP automation_operation_log_total DB-derived AI automation operation count for ADR-100 SLOs",
|
|
"# TYPE automation_operation_log_total counter",
|
|
]
|
|
if snapshot.automation_operations:
|
|
for sample in snapshot.automation_operations:
|
|
lines.append(
|
|
"automation_operation_log_total"
|
|
f'{{outcome="{_escape_label(sample.outcome)}",'
|
|
f'operation_type="{_escape_label(sample.operation_type)}"}} '
|
|
f"{sample.count}"
|
|
)
|
|
else:
|
|
lines.append(
|
|
'automation_operation_log_total{outcome="none",operation_type="none"} 0'
|
|
)
|
|
|
|
lines.extend([
|
|
"# HELP automation_operation_created_24h DB-derived AI automation operation count created in the last 24 hours for ADR-100 SLO dashboards",
|
|
"# TYPE automation_operation_created_24h gauge",
|
|
])
|
|
if snapshot.automation_operations_24h:
|
|
for sample in snapshot.automation_operations_24h:
|
|
lines.append(
|
|
"automation_operation_created_24h"
|
|
f'{{outcome="{_escape_label(sample.outcome)}",'
|
|
f'operation_type="{_escape_label(sample.operation_type)}"}} '
|
|
f"{sample.count}"
|
|
)
|
|
else:
|
|
lines.append(
|
|
'automation_operation_created_24h{outcome="none",operation_type="none"} 0'
|
|
)
|
|
|
|
lines.extend([
|
|
"# HELP post_execution_verification_total DB-derived post execution verification result count for ADR-100 SLOs",
|
|
"# TYPE post_execution_verification_total counter",
|
|
])
|
|
if snapshot.post_execution_verifications:
|
|
for sample in snapshot.post_execution_verifications:
|
|
lines.append(
|
|
"post_execution_verification_total"
|
|
f'{{outcome="{_escape_label(sample.outcome)}"}} {sample.count}'
|
|
)
|
|
else:
|
|
lines.append('post_execution_verification_total{outcome="none"} 0')
|
|
|
|
lines.extend([
|
|
"# HELP post_execution_verification_created_24h DB-derived post execution verification result count created in the last 24 hours for ADR-100 SLO dashboards",
|
|
"# TYPE post_execution_verification_created_24h gauge",
|
|
])
|
|
if snapshot.post_execution_verifications_24h:
|
|
for sample in snapshot.post_execution_verifications_24h:
|
|
lines.append(
|
|
"post_execution_verification_created_24h"
|
|
f'{{outcome="{_escape_label(sample.outcome)}"}} {sample.count}'
|
|
)
|
|
else:
|
|
lines.append('post_execution_verification_created_24h{outcome="none"} 0')
|
|
|
|
lines.extend([
|
|
"# HELP knowledge_entries_total DB-derived knowledge entry count for ADR-100 SLOs",
|
|
"# TYPE knowledge_entries_total counter",
|
|
f"knowledge_entries_total {snapshot.knowledge_entries_total}",
|
|
"# HELP knowledge_entries_created_24h DB-derived knowledge entries created in the last 24 hours for ADR-100 SLOs",
|
|
"# TYPE knowledge_entries_created_24h gauge",
|
|
f"knowledge_entries_created_24h {snapshot.knowledge_entries_created_24h}",
|
|
"# HELP approval_records_high_confidence_total DB-derived high confidence approval decisions for ADR-100 SLOs",
|
|
"# TYPE approval_records_high_confidence_total counter",
|
|
f"approval_records_high_confidence_total {snapshot.high_confidence_total}",
|
|
"# HELP approval_records_high_confidence_success_total DB-derived high confidence approval decisions with successful verification for ADR-100 SLOs",
|
|
"# TYPE approval_records_high_confidence_success_total counter",
|
|
(
|
|
"approval_records_high_confidence_success_total "
|
|
f"{snapshot.high_confidence_success_total}"
|
|
),
|
|
"# HELP adr100_slo_emitter_last_success_timestamp Last successful ADR-100 DB metrics emission timestamp",
|
|
"# TYPE adr100_slo_emitter_last_success_timestamp gauge",
|
|
f"adr100_slo_emitter_last_success_timestamp {snapshot.emitted_at:.0f}",
|
|
])
|
|
lines.extend([
|
|
"# HELP awooop_truth_chain_quality_summary_last_duration_seconds Last observed AwoooP truth-chain quality summary aggregation duration",
|
|
"# TYPE awooop_truth_chain_quality_summary_last_duration_seconds gauge",
|
|
])
|
|
if snapshot.quality_summary_observations:
|
|
for observation in snapshot.quality_summary_observations:
|
|
labels = _quality_summary_labels(observation)
|
|
lines.append(
|
|
"awooop_truth_chain_quality_summary_last_duration_seconds"
|
|
f"{labels} {observation.duration_seconds:.6f}"
|
|
)
|
|
else:
|
|
lines.append(
|
|
'awooop_truth_chain_quality_summary_last_duration_seconds{project_id="none",hours="0",limit="0",cache_status="none",success="false"} 0'
|
|
)
|
|
|
|
lines.extend([
|
|
"# HELP awooop_truth_chain_quality_summary_last_success Last observed AwoooP truth-chain quality summary success flag",
|
|
"# TYPE awooop_truth_chain_quality_summary_last_success gauge",
|
|
])
|
|
if snapshot.quality_summary_observations:
|
|
for observation in snapshot.quality_summary_observations:
|
|
labels = _quality_summary_labels(observation)
|
|
lines.append(
|
|
"awooop_truth_chain_quality_summary_last_success"
|
|
f"{labels} {1 if observation.success else 0}"
|
|
)
|
|
else:
|
|
lines.append(
|
|
'awooop_truth_chain_quality_summary_last_success{project_id="none",hours="0",limit="0",cache_status="none",success="false"} 0'
|
|
)
|
|
|
|
lines.extend([
|
|
"# HELP awooop_truth_chain_quality_summary_observed_timestamp Last observed AwoooP truth-chain quality summary timestamp",
|
|
"# TYPE awooop_truth_chain_quality_summary_observed_timestamp gauge",
|
|
])
|
|
if snapshot.quality_summary_observations:
|
|
for observation in snapshot.quality_summary_observations:
|
|
labels = _quality_summary_labels(observation)
|
|
lines.append(
|
|
"awooop_truth_chain_quality_summary_observed_timestamp"
|
|
f"{labels} {observation.observed_at:.0f}"
|
|
)
|
|
else:
|
|
lines.append(
|
|
'awooop_truth_chain_quality_summary_observed_timestamp{project_id="none",hours="0",limit="0",cache_status="none",success="false"} 0'
|
|
)
|
|
|
|
lines.append("")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _escape_label(value: str) -> str:
|
|
return value.replace("\\", "\\\\").replace("\n", "\\n").replace('"', '\\"')
|
|
|
|
|
|
def _quality_summary_labels(observation: QualitySummaryObservation) -> str:
|
|
return (
|
|
"{"
|
|
f'project_id="{_escape_label(observation.project_id)}",'
|
|
f'hours="{observation.hours}",'
|
|
f'limit="{observation.limit}",'
|
|
f'cache_status="{_escape_label(observation.cache_status)}",'
|
|
f'success="{"true" if observation.success else "false"}"'
|
|
"}"
|
|
)
|
|
|
|
|
|
_AUTOMATION_OPERATION_SQL = """
|
|
WITH automation_scope AS (
|
|
SELECT
|
|
CASE
|
|
WHEN status <> 'success' THEN status
|
|
WHEN actor = 'approval_execution'
|
|
AND COALESCE(input->>'requested_by', '') NOT ILIKE 'auto%%'
|
|
THEN 'human_required'
|
|
ELSE 'auto_executed'
|
|
END AS outcome,
|
|
operation_type
|
|
FROM automation_operation_log
|
|
WHERE operation_type IN (
|
|
'playbook_executed',
|
|
'remediation_executed',
|
|
'remediation_verified',
|
|
'remediation_rolled_back',
|
|
'self_correction_attempted'
|
|
)
|
|
UNION ALL
|
|
SELECT
|
|
CASE WHEN success THEN 'auto_executed' ELSE 'failed' END AS outcome,
|
|
'auto_repair_executed' AS operation_type
|
|
FROM auto_repair_executions
|
|
)
|
|
SELECT
|
|
outcome,
|
|
operation_type,
|
|
count(*) AS count
|
|
FROM automation_scope
|
|
GROUP BY outcome, operation_type
|
|
ORDER BY outcome, operation_type
|
|
"""
|
|
|
|
|
|
_AUTOMATION_OPERATION_24H_SQL = """
|
|
WITH automation_scope AS (
|
|
SELECT
|
|
CASE
|
|
WHEN status <> 'success' THEN status
|
|
WHEN actor = 'approval_execution'
|
|
AND COALESCE(input->>'requested_by', '') NOT ILIKE 'auto%%'
|
|
THEN 'human_required'
|
|
ELSE 'auto_executed'
|
|
END AS outcome,
|
|
operation_type
|
|
FROM automation_operation_log
|
|
WHERE operation_type IN (
|
|
'playbook_executed',
|
|
'remediation_executed',
|
|
'remediation_verified',
|
|
'remediation_rolled_back',
|
|
'self_correction_attempted'
|
|
)
|
|
AND created_at >= NOW() - INTERVAL '24 hours'
|
|
UNION ALL
|
|
SELECT
|
|
CASE WHEN success THEN 'auto_executed' ELSE 'failed' END AS outcome,
|
|
'auto_repair_executed' AS operation_type
|
|
FROM auto_repair_executions
|
|
WHERE created_at >= NOW() - INTERVAL '24 hours'
|
|
)
|
|
SELECT
|
|
outcome,
|
|
operation_type,
|
|
count(*) AS count
|
|
FROM automation_scope
|
|
GROUP BY outcome, operation_type
|
|
ORDER BY outcome, operation_type
|
|
"""
|
|
|
|
|
|
_POST_EXECUTION_VERIFICATION_SQL = """
|
|
SELECT verification_result AS outcome, count(*) AS count
|
|
FROM incident_evidence
|
|
WHERE verification_result IS NOT NULL
|
|
GROUP BY verification_result
|
|
ORDER BY verification_result
|
|
"""
|
|
|
|
|
|
_POST_EXECUTION_VERIFICATION_24H_SQL = """
|
|
SELECT verification_result AS outcome, count(*) AS count
|
|
FROM incident_evidence
|
|
WHERE verification_result IS NOT NULL
|
|
AND collected_at >= NOW() - INTERVAL '24 hours'
|
|
GROUP BY verification_result
|
|
ORDER BY verification_result
|
|
"""
|
|
|
|
|
|
_HIGH_CONFIDENCE_APPROVAL_SQL = """
|
|
WITH approval_confidence AS (
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
COALESCE(
|
|
CASE
|
|
WHEN extra_metadata->>'confidence_score' ~ '^[0-9]+(\\.[0-9]+)?$'
|
|
THEN (extra_metadata->>'confidence_score')::numeric
|
|
ELSE NULL
|
|
END,
|
|
CASE
|
|
WHEN extra_metadata->>'confidence' ~ '^[0-9]+(\\.[0-9]+)?$'
|
|
THEN (extra_metadata->>'confidence')::numeric
|
|
ELSE NULL
|
|
END,
|
|
composite_score,
|
|
0
|
|
) AS confidence
|
|
FROM approval_records
|
|
)
|
|
SELECT
|
|
count(*) FILTER (WHERE confidence >= 0.8) AS high_confidence_total,
|
|
count(*) FILTER (
|
|
WHERE confidence >= 0.8
|
|
AND EXISTS (
|
|
SELECT 1
|
|
FROM incident_evidence ev
|
|
WHERE ev.incident_id = approval_confidence.incident_id
|
|
AND ev.verification_result = 'success'
|
|
)
|
|
) AS high_confidence_success_total
|
|
FROM approval_confidence
|
|
"""
|
|
|
|
|
|
_adr100_slo_metrics_service: Adr100SloMetricsService | None = None
|
|
|
|
|
|
def get_adr100_slo_metrics_service() -> Adr100SloMetricsService:
|
|
global _adr100_slo_metrics_service
|
|
if _adr100_slo_metrics_service is None:
|
|
_adr100_slo_metrics_service = Adr100SloMetricsService()
|
|
return _adr100_slo_metrics_service
|