feat(sre): add 24h automation SLO scorecard
This commit is contained in:
Your Name
@@ -346,6 +346,7 @@ from src.services.ai_technology_radar_readback import (
|
||||
from src.services.ai_technology_report_cadence_readback import (
|
||||
load_latest_ai_technology_report_cadence_readback,
|
||||
)
|
||||
from src.services.automation_slo_scorecard import read_automation_slo_scorecard
|
||||
from src.services.awoooi_gitea_onboarding_warning_step_runtime_enablement_gate import (
|
||||
load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate,
|
||||
)
|
||||
@@ -1248,6 +1249,41 @@ async def get_sre_k3s_controlled_automation_work_items() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/sre-automation-slo-scorecard-24h",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得最近 24 小時 SRE AI 自動化成效 scorecard",
|
||||
description=(
|
||||
"唯讀彙總 PostgreSQL incident、run 與 provider budget receipts，計算 "
|
||||
"MTTA、MTTR、誤判、復發、人工介入、verifier、rollback、freshness、"
|
||||
"asset coverage 與模型成本。缺 evidence 時保持 partial，不執行修復、"
|
||||
"不呼叫模型、不讀 secret。"
|
||||
),
|
||||
)
|
||||
async def get_sre_automation_slo_scorecard_24h(
|
||||
x_project_id: str = Header(default="awoooi", alias="X-Project-ID"),
|
||||
) -> dict[str, Any]:
|
||||
"""回傳一個 project-scoped、no-false-green 的 24h scorecard。"""
|
||||
|
||||
try:
|
||||
payload = await read_automation_slo_scorecard(project_id=x_project_id)
|
||||
return redact_public_lan_topology(payload)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"sre_automation_slo_scorecard_read_failed",
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="SRE AI 自動化 24h scorecard 暫時無法讀取",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/portfolio-infrastructure-asset-reconciliation",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
404
apps/api/src/services/automation_slo_scorecard.py
Normal file
404
apps/api/src/services/automation_slo_scorecard.py
Normal file
@@ -0,0 +1,404 @@
|
||||
"""Read-only 24-hour SRE automation outcome scorecard."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.db.base import get_db_context
|
||||
from src.services.ai_automation_runtime_contract import (
|
||||
AI_AUTOMATION_REQUIRED_OUTCOME_METRICS,
|
||||
)
|
||||
|
||||
SCHEMA_VERSION = "sre_automation_slo_scorecard_v1"
|
||||
_PROJECT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
|
||||
|
||||
_INCIDENT_SAMPLES_SQL = text(
|
||||
"""
|
||||
WITH scoped_events AS (
|
||||
SELECT
|
||||
COALESCE(operation.incident_id, operation.id::text) AS sample_id,
|
||||
operation.event_type::text AS event_type,
|
||||
operation.created_at,
|
||||
operation.context,
|
||||
incident.resolved_at AS incident_resolved_at,
|
||||
COALESCE(
|
||||
NULLIF(operation.context ->> 'typed_domain', ''),
|
||||
NULLIF(operation.context ->> 'alert_category', ''),
|
||||
NULLIF(incident.alert_category, ''),
|
||||
'unclassified'
|
||||
) AS lane
|
||||
FROM alert_operation_log operation
|
||||
LEFT JOIN incidents incident
|
||||
ON incident.incident_id = operation.incident_id
|
||||
WHERE operation.created_at >= :window_start
|
||||
AND operation.created_at < :window_end
|
||||
AND COALESCE(
|
||||
incident.project_id,
|
||||
NULLIF(operation.context ->> 'project_id', ''),
|
||||
'awoooi'
|
||||
) = :project_id
|
||||
), per_incident AS (
|
||||
SELECT
|
||||
sample_id,
|
||||
COALESCE(
|
||||
min(lane) FILTER (WHERE lane <> 'unclassified'),
|
||||
'unclassified'
|
||||
) AS lane,
|
||||
min(created_at) FILTER (
|
||||
WHERE event_type = 'ALERT_RECEIVED'
|
||||
) AS received_at,
|
||||
min(created_at) FILTER (
|
||||
WHERE event_type IN (
|
||||
'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED',
|
||||
'PRE_FLIGHT_PASSED', 'PRE_FLIGHT_FAILED',
|
||||
'GUARDRAIL_BLOCKED', 'USER_ACTION', 'ESCALATED'
|
||||
)
|
||||
) AS first_action_at,
|
||||
COALESCE(
|
||||
min(created_at) FILTER (WHERE event_type = 'RESOLVED'),
|
||||
max(incident_resolved_at)
|
||||
) AS resolved_at,
|
||||
bool_or(context ? 'false_positive') AS false_positive_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'false_positive', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
) AS false_positive,
|
||||
bool_or(
|
||||
context ? 'recurrence' OR context ? 'recurrence_fingerprint'
|
||||
) AS recurrence_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'recurrence', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
OR NULLIF(context ->> 'recurrence_fingerprint', '') IS NOT NULL
|
||||
) AS recurrence,
|
||||
bool_or(event_type IN ('USER_ACTION', 'MANUAL_FIX_RECORDED'))
|
||||
AS human_intervention,
|
||||
bool_or(event_type IN (
|
||||
'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED',
|
||||
'EXECUTION_COMPLETED', 'CHANGE_APPLIED'
|
||||
)) AS execution_observed,
|
||||
bool_or(
|
||||
context ? 'verifier_passed'
|
||||
OR context ? 'verification_result'
|
||||
) AS verifier_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'verifier_passed', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
OR lower(COALESCE(context ->> 'verification_result', ''))
|
||||
IN ('success', 'passed', 'verified')
|
||||
) AS verifier_passed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'rollback_performed', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
) AS rollback_performed,
|
||||
bool_or(context ? 'signal_freshness_slo_pass')
|
||||
AS freshness_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(
|
||||
context ->> 'signal_freshness_slo_pass', 'false'
|
||||
)) IN ('true', '1', 'yes')
|
||||
) AS freshness_passed,
|
||||
bool_or(
|
||||
NULLIF(context ->> 'canonical_asset_id', '') IS NOT NULL
|
||||
AND COALESCE(context ->> 'asset_identity_status', 'resolved')
|
||||
<> 'asset_identity_unresolved'
|
||||
) AS asset_identity_resolved
|
||||
FROM scoped_events
|
||||
GROUP BY sample_id
|
||||
)
|
||||
SELECT *
|
||||
FROM per_incident
|
||||
WHERE received_at IS NOT NULL
|
||||
ORDER BY received_at, sample_id
|
||||
"""
|
||||
)
|
||||
|
||||
_RUN_SUMMARY_SQL = text(
|
||||
"""
|
||||
SELECT
|
||||
count(*)::int AS total_runs,
|
||||
count(*) FILTER (WHERE is_shadow IS FALSE)::int AS non_shadow_runs,
|
||||
count(*) FILTER (
|
||||
WHERE is_shadow IS FALSE AND step_count > 0
|
||||
)::int AS controlled_runs_with_steps,
|
||||
count(*) FILTER (WHERE state = 'completed')::int AS completed_runs,
|
||||
count(*) FILTER (
|
||||
WHERE state IN ('failed', 'cancelled', 'timeout')
|
||||
)::int AS failed_runs
|
||||
FROM awooop_run_state
|
||||
WHERE project_id = :project_id
|
||||
AND created_at >= :window_start
|
||||
AND created_at < :window_end
|
||||
"""
|
||||
)
|
||||
|
||||
_PROVIDER_COST_SQL = text(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(provider, 'unattributed') AS provider,
|
||||
COALESCE(model, 'unattributed') AS model,
|
||||
count(*)::int AS call_count,
|
||||
COALESCE(sum(prompt_tokens), 0)::bigint AS prompt_tokens,
|
||||
COALESCE(sum(completion_tokens), 0)::bigint AS completion_tokens,
|
||||
COALESCE(sum(cost_usd), 0) AS cost_usd
|
||||
FROM budget_ledger
|
||||
WHERE project_id = :project_id
|
||||
AND recorded_at >= :window_start
|
||||
AND recorded_at < :window_end
|
||||
GROUP BY provider, model
|
||||
ORDER BY provider, model
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _flag(value: object) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value or "").strip().lower() in {"true", "1", "yes"}
|
||||
|
||||
|
||||
def _as_datetime(value: object) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
elif isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _rate(numerator: int, denominator: int) -> float | None:
|
||||
return round(numerator / denominator, 4) if denominator else None
|
||||
|
||||
|
||||
def _average(values: Sequence[float]) -> float | None:
|
||||
return round(sum(values) / len(values), 2) if values else None
|
||||
|
||||
|
||||
def _metrics_for(
|
||||
samples: Sequence[Mapping[str, Any]],
|
||||
) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
mtta: list[float] = []
|
||||
mttr: list[float] = []
|
||||
for sample in samples:
|
||||
received_at = _as_datetime(sample.get("received_at"))
|
||||
first_action_at = _as_datetime(sample.get("first_action_at"))
|
||||
resolved_at = _as_datetime(sample.get("resolved_at"))
|
||||
if received_at and first_action_at and first_action_at >= received_at:
|
||||
mtta.append((first_action_at - received_at).total_seconds())
|
||||
if received_at and resolved_at and resolved_at >= received_at:
|
||||
mttr.append((resolved_at - received_at).total_seconds())
|
||||
|
||||
total = len(samples)
|
||||
false_observed = sum(_flag(row.get("false_positive_observed")) for row in samples)
|
||||
recurrence_observed = sum(_flag(row.get("recurrence_observed")) for row in samples)
|
||||
execution_observed = sum(_flag(row.get("execution_observed")) for row in samples)
|
||||
verifier_observed = sum(_flag(row.get("verifier_observed")) for row in samples)
|
||||
freshness_observed = sum(_flag(row.get("freshness_observed")) for row in samples)
|
||||
denominators = {
|
||||
"alert_count": total,
|
||||
"mtta_sample_count": len(mtta),
|
||||
"mttr_sample_count": len(mttr),
|
||||
"false_positive_classified_count": false_observed,
|
||||
"recurrence_classified_count": recurrence_observed,
|
||||
"execution_observed_count": execution_observed,
|
||||
"verifier_observed_count": verifier_observed,
|
||||
"freshness_observed_count": freshness_observed,
|
||||
}
|
||||
metrics = {
|
||||
"mtta_seconds": _average(mtta),
|
||||
"mttr_seconds": _average(mttr),
|
||||
"false_positive_rate": _rate(
|
||||
sum(_flag(row.get("false_positive")) for row in samples),
|
||||
false_observed,
|
||||
),
|
||||
"recurrence_rate": _rate(
|
||||
sum(_flag(row.get("recurrence")) for row in samples),
|
||||
recurrence_observed,
|
||||
),
|
||||
"human_intervention_rate": _rate(
|
||||
sum(_flag(row.get("human_intervention")) for row in samples),
|
||||
total,
|
||||
),
|
||||
"verifier_pass_rate": _rate(
|
||||
sum(_flag(row.get("verifier_passed")) for row in samples),
|
||||
verifier_observed,
|
||||
),
|
||||
"rollback_rate": _rate(
|
||||
sum(_flag(row.get("rollback_performed")) for row in samples),
|
||||
execution_observed,
|
||||
),
|
||||
"signal_freshness_slo_pass_rate": _rate(
|
||||
sum(_flag(row.get("freshness_passed")) for row in samples),
|
||||
freshness_observed,
|
||||
),
|
||||
"asset_coverage_percent": (
|
||||
round(
|
||||
sum(_flag(row.get("asset_identity_resolved")) for row in samples)
|
||||
/ total
|
||||
* 100,
|
||||
2,
|
||||
)
|
||||
if total
|
||||
else None
|
||||
),
|
||||
}
|
||||
return metrics, denominators
|
||||
|
||||
|
||||
def build_automation_slo_scorecard(
|
||||
*,
|
||||
project_id: str,
|
||||
incident_samples: Sequence[Mapping[str, Any]],
|
||||
run_summary: Mapping[str, Any] | None,
|
||||
provider_cost_rows: Sequence[Mapping[str, Any]],
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a no-false-green scorecard from sanitized aggregate rows."""
|
||||
|
||||
metrics, denominators = _metrics_for(incident_samples)
|
||||
lanes: dict[str, list[Mapping[str, Any]]] = {}
|
||||
for sample in incident_samples:
|
||||
lane = str(sample.get("lane") or "unclassified")[:64]
|
||||
lanes.setdefault(lane, []).append(sample)
|
||||
lane_scorecards = []
|
||||
for lane, samples in sorted(lanes.items()):
|
||||
lane_metrics, lane_denominators = _metrics_for(samples)
|
||||
lane_scorecards.append(
|
||||
{
|
||||
"lane": lane,
|
||||
"metrics": lane_metrics,
|
||||
"denominators": lane_denominators,
|
||||
}
|
||||
)
|
||||
|
||||
missing_metrics = sorted(
|
||||
metric
|
||||
for metric in AI_AUTOMATION_REQUIRED_OUTCOME_METRICS
|
||||
if metrics.get(metric) is None
|
||||
)
|
||||
coverage_blockers = []
|
||||
alert_count = denominators["alert_count"]
|
||||
for key in (
|
||||
"mtta_sample_count",
|
||||
"mttr_sample_count",
|
||||
"false_positive_classified_count",
|
||||
"recurrence_classified_count",
|
||||
"freshness_observed_count",
|
||||
):
|
||||
if denominators[key] < alert_count:
|
||||
coverage_blockers.append(f"{key}_below_alert_count")
|
||||
if (
|
||||
denominators["verifier_observed_count"]
|
||||
< denominators["execution_observed_count"]
|
||||
):
|
||||
coverage_blockers.append("verifier_observed_count_below_execution_count")
|
||||
|
||||
costs = []
|
||||
for row in provider_cost_rows:
|
||||
cost = Decimal(str(row.get("cost_usd") or "0"))
|
||||
costs.append(
|
||||
{
|
||||
"provider": str(row.get("provider") or "unattributed")[:32],
|
||||
"model": str(row.get("model") or "unattributed")[:64],
|
||||
"call_count": int(row.get("call_count") or 0),
|
||||
"prompt_tokens": int(row.get("prompt_tokens") or 0),
|
||||
"completion_tokens": int(row.get("completion_tokens") or 0),
|
||||
"cost_usd": f"{cost:.4f}",
|
||||
}
|
||||
)
|
||||
total_cost = sum((Decimal(row["cost_usd"]) for row in costs), Decimal("0"))
|
||||
status = (
|
||||
"no_runtime_sample_fail_closed"
|
||||
if not incident_samples
|
||||
else "partial_missing_receipt_coverage"
|
||||
if missing_metrics or coverage_blockers
|
||||
else "runtime_aggregate_ready_same_run_verification_pending"
|
||||
)
|
||||
safe_runs = {
|
||||
key: int((run_summary or {}).get(key) or 0)
|
||||
for key in (
|
||||
"total_runs",
|
||||
"non_shadow_runs",
|
||||
"controlled_runs_with_steps",
|
||||
"completed_runs",
|
||||
"failed_runs",
|
||||
)
|
||||
}
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"project_id": project_id,
|
||||
"window": {
|
||||
"hours": 24,
|
||||
"start": window_start.astimezone(UTC).isoformat(),
|
||||
"end": window_end.astimezone(UTC).isoformat(),
|
||||
},
|
||||
"status": status,
|
||||
"runtime_closed": False,
|
||||
"metrics": metrics,
|
||||
"denominators": denominators,
|
||||
"lane_scorecards": lane_scorecards,
|
||||
"runs": safe_runs,
|
||||
"provider_usage": {
|
||||
"rows": costs,
|
||||
"call_count": sum(row["call_count"] for row in costs),
|
||||
"prompt_tokens": sum(row["prompt_tokens"] for row in costs),
|
||||
"completion_tokens": sum(row["completion_tokens"] for row in costs),
|
||||
"cost_usd": f"{total_cost:.4f}",
|
||||
},
|
||||
"missing_metrics": missing_metrics,
|
||||
"coverage_blockers": coverage_blockers,
|
||||
"evidence": {
|
||||
"source": "postgresql_read_only_aggregate",
|
||||
"same_run_receipt_verification": "pending",
|
||||
"no_false_green": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def read_automation_slo_scorecard(
|
||||
*,
|
||||
project_id: str = "awoooi",
|
||||
window_end: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Read the latest 24 hours without mutating runtime state."""
|
||||
|
||||
if not _PROJECT_ID.fullmatch(project_id):
|
||||
raise ValueError("project_id_invalid")
|
||||
end = window_end or datetime.now(UTC)
|
||||
if end.tzinfo is None:
|
||||
end = end.replace(tzinfo=UTC)
|
||||
end = end.astimezone(UTC)
|
||||
start = end - timedelta(hours=24)
|
||||
parameters = {
|
||||
"project_id": project_id,
|
||||
"window_start": start,
|
||||
"window_end": end,
|
||||
}
|
||||
async with get_db_context(project_id) as db:
|
||||
incident_result = await db.execute(_INCIDENT_SAMPLES_SQL, parameters)
|
||||
run_result = await db.execute(_RUN_SUMMARY_SQL, parameters)
|
||||
cost_result = await db.execute(_PROVIDER_COST_SQL, parameters)
|
||||
incident_samples = [dict(row) for row in incident_result.mappings().all()]
|
||||
run_row = run_result.mappings().first()
|
||||
provider_cost_rows = [dict(row) for row in cost_result.mappings().all()]
|
||||
return build_automation_slo_scorecard(
|
||||
project_id=project_id,
|
||||
incident_samples=incident_samples,
|
||||
run_summary=dict(run_row) if run_row else {},
|
||||
provider_cost_rows=provider_cost_rows,
|
||||
window_start=start,
|
||||
window_end=end,
|
||||
)
|
||||
Reference in New Issue
Block a user