feat(sre): add 24h automation SLO scorecard
This commit is contained in:
Your Name
@@ -346,6 +346,7 @@ from src.services.ai_technology_radar_readback import (
|
||||
from src.services.ai_technology_report_cadence_readback import (
|
||||
load_latest_ai_technology_report_cadence_readback,
|
||||
)
|
||||
from src.services.automation_slo_scorecard import read_automation_slo_scorecard
|
||||
from src.services.awoooi_gitea_onboarding_warning_step_runtime_enablement_gate import (
|
||||
load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate,
|
||||
)
|
||||
@@ -1248,6 +1249,41 @@ async def get_sre_k3s_controlled_automation_work_items() -> dict[str, Any]:
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/sre-automation-slo-scorecard-24h",
|
||||
response_model=dict[str, Any],
|
||||
summary="取得最近 24 小時 SRE AI 自動化成效 scorecard",
|
||||
description=(
|
||||
"唯讀彙總 PostgreSQL incident、run 與 provider budget receipts，計算 "
|
||||
"MTTA、MTTR、誤判、復發、人工介入、verifier、rollback、freshness、"
|
||||
"asset coverage 與模型成本。缺 evidence 時保持 partial，不執行修復、"
|
||||
"不呼叫模型、不讀 secret。"
|
||||
),
|
||||
)
|
||||
async def get_sre_automation_slo_scorecard_24h(
|
||||
x_project_id: str = Header(default="awoooi", alias="X-Project-ID"),
|
||||
) -> dict[str, Any]:
|
||||
"""回傳一個 project-scoped、no-false-green 的 24h scorecard。"""
|
||||
|
||||
try:
|
||||
payload = await read_automation_slo_scorecard(project_id=x_project_id)
|
||||
return redact_public_lan_topology(payload)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail=str(exc),
|
||||
) from exc
|
||||
except Exception as exc:
|
||||
logger.error(
|
||||
"sre_automation_slo_scorecard_read_failed",
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="SRE AI 自動化 24h scorecard 暫時無法讀取",
|
||||
) from exc
|
||||
|
||||
|
||||
@router.get(
|
||||
"/portfolio-infrastructure-asset-reconciliation",
|
||||
response_model=dict[str, Any],
|
||||
|
||||
404
apps/api/src/services/automation_slo_scorecard.py
Normal file
404
apps/api/src/services/automation_slo_scorecard.py
Normal file
@@ -0,0 +1,404 @@
|
||||
"""Read-only 24-hour SRE automation outcome scorecard."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections.abc import Mapping, Sequence
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import text
|
||||
|
||||
from src.db.base import get_db_context
|
||||
from src.services.ai_automation_runtime_contract import (
|
||||
AI_AUTOMATION_REQUIRED_OUTCOME_METRICS,
|
||||
)
|
||||
|
||||
SCHEMA_VERSION = "sre_automation_slo_scorecard_v1"
|
||||
_PROJECT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
|
||||
|
||||
_INCIDENT_SAMPLES_SQL = text(
|
||||
"""
|
||||
WITH scoped_events AS (
|
||||
SELECT
|
||||
COALESCE(operation.incident_id, operation.id::text) AS sample_id,
|
||||
operation.event_type::text AS event_type,
|
||||
operation.created_at,
|
||||
operation.context,
|
||||
incident.resolved_at AS incident_resolved_at,
|
||||
COALESCE(
|
||||
NULLIF(operation.context ->> 'typed_domain', ''),
|
||||
NULLIF(operation.context ->> 'alert_category', ''),
|
||||
NULLIF(incident.alert_category, ''),
|
||||
'unclassified'
|
||||
) AS lane
|
||||
FROM alert_operation_log operation
|
||||
LEFT JOIN incidents incident
|
||||
ON incident.incident_id = operation.incident_id
|
||||
WHERE operation.created_at >= :window_start
|
||||
AND operation.created_at < :window_end
|
||||
AND COALESCE(
|
||||
incident.project_id,
|
||||
NULLIF(operation.context ->> 'project_id', ''),
|
||||
'awoooi'
|
||||
) = :project_id
|
||||
), per_incident AS (
|
||||
SELECT
|
||||
sample_id,
|
||||
COALESCE(
|
||||
min(lane) FILTER (WHERE lane <> 'unclassified'),
|
||||
'unclassified'
|
||||
) AS lane,
|
||||
min(created_at) FILTER (
|
||||
WHERE event_type = 'ALERT_RECEIVED'
|
||||
) AS received_at,
|
||||
min(created_at) FILTER (
|
||||
WHERE event_type IN (
|
||||
'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED',
|
||||
'PRE_FLIGHT_PASSED', 'PRE_FLIGHT_FAILED',
|
||||
'GUARDRAIL_BLOCKED', 'USER_ACTION', 'ESCALATED'
|
||||
)
|
||||
) AS first_action_at,
|
||||
COALESCE(
|
||||
min(created_at) FILTER (WHERE event_type = 'RESOLVED'),
|
||||
max(incident_resolved_at)
|
||||
) AS resolved_at,
|
||||
bool_or(context ? 'false_positive') AS false_positive_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'false_positive', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
) AS false_positive,
|
||||
bool_or(
|
||||
context ? 'recurrence' OR context ? 'recurrence_fingerprint'
|
||||
) AS recurrence_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'recurrence', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
OR NULLIF(context ->> 'recurrence_fingerprint', '') IS NOT NULL
|
||||
) AS recurrence,
|
||||
bool_or(event_type IN ('USER_ACTION', 'MANUAL_FIX_RECORDED'))
|
||||
AS human_intervention,
|
||||
bool_or(event_type IN (
|
||||
'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED',
|
||||
'EXECUTION_COMPLETED', 'CHANGE_APPLIED'
|
||||
)) AS execution_observed,
|
||||
bool_or(
|
||||
context ? 'verifier_passed'
|
||||
OR context ? 'verification_result'
|
||||
) AS verifier_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'verifier_passed', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
OR lower(COALESCE(context ->> 'verification_result', ''))
|
||||
IN ('success', 'passed', 'verified')
|
||||
) AS verifier_passed,
|
||||
bool_or(
|
||||
lower(COALESCE(context ->> 'rollback_performed', 'false'))
|
||||
IN ('true', '1', 'yes')
|
||||
) AS rollback_performed,
|
||||
bool_or(context ? 'signal_freshness_slo_pass')
|
||||
AS freshness_observed,
|
||||
bool_or(
|
||||
lower(COALESCE(
|
||||
context ->> 'signal_freshness_slo_pass', 'false'
|
||||
)) IN ('true', '1', 'yes')
|
||||
) AS freshness_passed,
|
||||
bool_or(
|
||||
NULLIF(context ->> 'canonical_asset_id', '') IS NOT NULL
|
||||
AND COALESCE(context ->> 'asset_identity_status', 'resolved')
|
||||
<> 'asset_identity_unresolved'
|
||||
) AS asset_identity_resolved
|
||||
FROM scoped_events
|
||||
GROUP BY sample_id
|
||||
)
|
||||
SELECT *
|
||||
FROM per_incident
|
||||
WHERE received_at IS NOT NULL
|
||||
ORDER BY received_at, sample_id
|
||||
"""
|
||||
)
|
||||
|
||||
_RUN_SUMMARY_SQL = text(
|
||||
"""
|
||||
SELECT
|
||||
count(*)::int AS total_runs,
|
||||
count(*) FILTER (WHERE is_shadow IS FALSE)::int AS non_shadow_runs,
|
||||
count(*) FILTER (
|
||||
WHERE is_shadow IS FALSE AND step_count > 0
|
||||
)::int AS controlled_runs_with_steps,
|
||||
count(*) FILTER (WHERE state = 'completed')::int AS completed_runs,
|
||||
count(*) FILTER (
|
||||
WHERE state IN ('failed', 'cancelled', 'timeout')
|
||||
)::int AS failed_runs
|
||||
FROM awooop_run_state
|
||||
WHERE project_id = :project_id
|
||||
AND created_at >= :window_start
|
||||
AND created_at < :window_end
|
||||
"""
|
||||
)
|
||||
|
||||
_PROVIDER_COST_SQL = text(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(provider, 'unattributed') AS provider,
|
||||
COALESCE(model, 'unattributed') AS model,
|
||||
count(*)::int AS call_count,
|
||||
COALESCE(sum(prompt_tokens), 0)::bigint AS prompt_tokens,
|
||||
COALESCE(sum(completion_tokens), 0)::bigint AS completion_tokens,
|
||||
COALESCE(sum(cost_usd), 0) AS cost_usd
|
||||
FROM budget_ledger
|
||||
WHERE project_id = :project_id
|
||||
AND recorded_at >= :window_start
|
||||
AND recorded_at < :window_end
|
||||
GROUP BY provider, model
|
||||
ORDER BY provider, model
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
def _flag(value: object) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
return str(value or "").strip().lower() in {"true", "1", "yes"}
|
||||
|
||||
|
||||
def _as_datetime(value: object) -> datetime | None:
|
||||
if isinstance(value, datetime):
|
||||
parsed = value
|
||||
elif isinstance(value, str) and value.strip():
|
||||
try:
|
||||
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
return None
|
||||
else:
|
||||
return None
|
||||
if parsed.tzinfo is None:
|
||||
parsed = parsed.replace(tzinfo=UTC)
|
||||
return parsed.astimezone(UTC)
|
||||
|
||||
|
||||
def _rate(numerator: int, denominator: int) -> float | None:
|
||||
return round(numerator / denominator, 4) if denominator else None
|
||||
|
||||
|
||||
def _average(values: Sequence[float]) -> float | None:
|
||||
return round(sum(values) / len(values), 2) if values else None
|
||||
|
||||
|
||||
def _metrics_for(
|
||||
samples: Sequence[Mapping[str, Any]],
|
||||
) -> tuple[dict[str, Any], dict[str, int]]:
|
||||
mtta: list[float] = []
|
||||
mttr: list[float] = []
|
||||
for sample in samples:
|
||||
received_at = _as_datetime(sample.get("received_at"))
|
||||
first_action_at = _as_datetime(sample.get("first_action_at"))
|
||||
resolved_at = _as_datetime(sample.get("resolved_at"))
|
||||
if received_at and first_action_at and first_action_at >= received_at:
|
||||
mtta.append((first_action_at - received_at).total_seconds())
|
||||
if received_at and resolved_at and resolved_at >= received_at:
|
||||
mttr.append((resolved_at - received_at).total_seconds())
|
||||
|
||||
total = len(samples)
|
||||
false_observed = sum(_flag(row.get("false_positive_observed")) for row in samples)
|
||||
recurrence_observed = sum(_flag(row.get("recurrence_observed")) for row in samples)
|
||||
execution_observed = sum(_flag(row.get("execution_observed")) for row in samples)
|
||||
verifier_observed = sum(_flag(row.get("verifier_observed")) for row in samples)
|
||||
freshness_observed = sum(_flag(row.get("freshness_observed")) for row in samples)
|
||||
denominators = {
|
||||
"alert_count": total,
|
||||
"mtta_sample_count": len(mtta),
|
||||
"mttr_sample_count": len(mttr),
|
||||
"false_positive_classified_count": false_observed,
|
||||
"recurrence_classified_count": recurrence_observed,
|
||||
"execution_observed_count": execution_observed,
|
||||
"verifier_observed_count": verifier_observed,
|
||||
"freshness_observed_count": freshness_observed,
|
||||
}
|
||||
metrics = {
|
||||
"mtta_seconds": _average(mtta),
|
||||
"mttr_seconds": _average(mttr),
|
||||
"false_positive_rate": _rate(
|
||||
sum(_flag(row.get("false_positive")) for row in samples),
|
||||
false_observed,
|
||||
),
|
||||
"recurrence_rate": _rate(
|
||||
sum(_flag(row.get("recurrence")) for row in samples),
|
||||
recurrence_observed,
|
||||
),
|
||||
"human_intervention_rate": _rate(
|
||||
sum(_flag(row.get("human_intervention")) for row in samples),
|
||||
total,
|
||||
),
|
||||
"verifier_pass_rate": _rate(
|
||||
sum(_flag(row.get("verifier_passed")) for row in samples),
|
||||
verifier_observed,
|
||||
),
|
||||
"rollback_rate": _rate(
|
||||
sum(_flag(row.get("rollback_performed")) for row in samples),
|
||||
execution_observed,
|
||||
),
|
||||
"signal_freshness_slo_pass_rate": _rate(
|
||||
sum(_flag(row.get("freshness_passed")) for row in samples),
|
||||
freshness_observed,
|
||||
),
|
||||
"asset_coverage_percent": (
|
||||
round(
|
||||
sum(_flag(row.get("asset_identity_resolved")) for row in samples)
|
||||
/ total
|
||||
* 100,
|
||||
2,
|
||||
)
|
||||
if total
|
||||
else None
|
||||
),
|
||||
}
|
||||
return metrics, denominators
|
||||
|
||||
|
||||
def build_automation_slo_scorecard(
|
||||
*,
|
||||
project_id: str,
|
||||
incident_samples: Sequence[Mapping[str, Any]],
|
||||
run_summary: Mapping[str, Any] | None,
|
||||
provider_cost_rows: Sequence[Mapping[str, Any]],
|
||||
window_start: datetime,
|
||||
window_end: datetime,
|
||||
) -> dict[str, Any]:
|
||||
"""Build a no-false-green scorecard from sanitized aggregate rows."""
|
||||
|
||||
metrics, denominators = _metrics_for(incident_samples)
|
||||
lanes: dict[str, list[Mapping[str, Any]]] = {}
|
||||
for sample in incident_samples:
|
||||
lane = str(sample.get("lane") or "unclassified")[:64]
|
||||
lanes.setdefault(lane, []).append(sample)
|
||||
lane_scorecards = []
|
||||
for lane, samples in sorted(lanes.items()):
|
||||
lane_metrics, lane_denominators = _metrics_for(samples)
|
||||
lane_scorecards.append(
|
||||
{
|
||||
"lane": lane,
|
||||
"metrics": lane_metrics,
|
||||
"denominators": lane_denominators,
|
||||
}
|
||||
)
|
||||
|
||||
missing_metrics = sorted(
|
||||
metric
|
||||
for metric in AI_AUTOMATION_REQUIRED_OUTCOME_METRICS
|
||||
if metrics.get(metric) is None
|
||||
)
|
||||
coverage_blockers = []
|
||||
alert_count = denominators["alert_count"]
|
||||
for key in (
|
||||
"mtta_sample_count",
|
||||
"mttr_sample_count",
|
||||
"false_positive_classified_count",
|
||||
"recurrence_classified_count",
|
||||
"freshness_observed_count",
|
||||
):
|
||||
if denominators[key] < alert_count:
|
||||
coverage_blockers.append(f"{key}_below_alert_count")
|
||||
if (
|
||||
denominators["verifier_observed_count"]
|
||||
< denominators["execution_observed_count"]
|
||||
):
|
||||
coverage_blockers.append("verifier_observed_count_below_execution_count")
|
||||
|
||||
costs = []
|
||||
for row in provider_cost_rows:
|
||||
cost = Decimal(str(row.get("cost_usd") or "0"))
|
||||
costs.append(
|
||||
{
|
||||
"provider": str(row.get("provider") or "unattributed")[:32],
|
||||
"model": str(row.get("model") or "unattributed")[:64],
|
||||
"call_count": int(row.get("call_count") or 0),
|
||||
"prompt_tokens": int(row.get("prompt_tokens") or 0),
|
||||
"completion_tokens": int(row.get("completion_tokens") or 0),
|
||||
"cost_usd": f"{cost:.4f}",
|
||||
}
|
||||
)
|
||||
total_cost = sum((Decimal(row["cost_usd"]) for row in costs), Decimal("0"))
|
||||
status = (
|
||||
"no_runtime_sample_fail_closed"
|
||||
if not incident_samples
|
||||
else "partial_missing_receipt_coverage"
|
||||
if missing_metrics or coverage_blockers
|
||||
else "runtime_aggregate_ready_same_run_verification_pending"
|
||||
)
|
||||
safe_runs = {
|
||||
key: int((run_summary or {}).get(key) or 0)
|
||||
for key in (
|
||||
"total_runs",
|
||||
"non_shadow_runs",
|
||||
"controlled_runs_with_steps",
|
||||
"completed_runs",
|
||||
"failed_runs",
|
||||
)
|
||||
}
|
||||
return {
|
||||
"schema_version": SCHEMA_VERSION,
|
||||
"project_id": project_id,
|
||||
"window": {
|
||||
"hours": 24,
|
||||
"start": window_start.astimezone(UTC).isoformat(),
|
||||
"end": window_end.astimezone(UTC).isoformat(),
|
||||
},
|
||||
"status": status,
|
||||
"runtime_closed": False,
|
||||
"metrics": metrics,
|
||||
"denominators": denominators,
|
||||
"lane_scorecards": lane_scorecards,
|
||||
"runs": safe_runs,
|
||||
"provider_usage": {
|
||||
"rows": costs,
|
||||
"call_count": sum(row["call_count"] for row in costs),
|
||||
"prompt_tokens": sum(row["prompt_tokens"] for row in costs),
|
||||
"completion_tokens": sum(row["completion_tokens"] for row in costs),
|
||||
"cost_usd": f"{total_cost:.4f}",
|
||||
},
|
||||
"missing_metrics": missing_metrics,
|
||||
"coverage_blockers": coverage_blockers,
|
||||
"evidence": {
|
||||
"source": "postgresql_read_only_aggregate",
|
||||
"same_run_receipt_verification": "pending",
|
||||
"no_false_green": True,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
async def read_automation_slo_scorecard(
|
||||
*,
|
||||
project_id: str = "awoooi",
|
||||
window_end: datetime | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Read the latest 24 hours without mutating runtime state."""
|
||||
|
||||
if not _PROJECT_ID.fullmatch(project_id):
|
||||
raise ValueError("project_id_invalid")
|
||||
end = window_end or datetime.now(UTC)
|
||||
if end.tzinfo is None:
|
||||
end = end.replace(tzinfo=UTC)
|
||||
end = end.astimezone(UTC)
|
||||
start = end - timedelta(hours=24)
|
||||
parameters = {
|
||||
"project_id": project_id,
|
||||
"window_start": start,
|
||||
"window_end": end,
|
||||
}
|
||||
async with get_db_context(project_id) as db:
|
||||
incident_result = await db.execute(_INCIDENT_SAMPLES_SQL, parameters)
|
||||
run_result = await db.execute(_RUN_SUMMARY_SQL, parameters)
|
||||
cost_result = await db.execute(_PROVIDER_COST_SQL, parameters)
|
||||
incident_samples = [dict(row) for row in incident_result.mappings().all()]
|
||||
run_row = run_result.mappings().first()
|
||||
provider_cost_rows = [dict(row) for row in cost_result.mappings().all()]
|
||||
return build_automation_slo_scorecard(
|
||||
project_id=project_id,
|
||||
incident_samples=incident_samples,
|
||||
run_summary=dict(run_row) if run_row else {},
|
||||
provider_cost_rows=provider_cost_rows,
|
||||
window_start=start,
|
||||
window_end=end,
|
||||
)
|
||||
275
apps/api/tests/test_automation_slo_scorecard.py
Normal file
275
apps/api/tests/test_automation_slo_scorecard.py
Normal file
@@ -0,0 +1,275 @@
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from unittest.mock import AsyncMock
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from src.api.v1 import agents
|
||||
from src.services import automation_slo_scorecard as scorecard_module
|
||||
from src.services.automation_slo_scorecard import (
|
||||
build_automation_slo_scorecard,
|
||||
read_automation_slo_scorecard,
|
||||
)
|
||||
|
||||
|
||||
def _sample(
|
||||
start: datetime,
|
||||
*,
|
||||
lane: str,
|
||||
action_seconds: int,
|
||||
resolve_seconds: int,
|
||||
positive: bool,
|
||||
) -> dict:
|
||||
return {
|
||||
"lane": lane,
|
||||
"received_at": start,
|
||||
"first_action_at": start + timedelta(seconds=action_seconds),
|
||||
"resolved_at": start + timedelta(seconds=resolve_seconds),
|
||||
"false_positive_observed": True,
|
||||
"false_positive": positive,
|
||||
"recurrence_observed": True,
|
||||
"recurrence": positive,
|
||||
"human_intervention": positive,
|
||||
"execution_observed": True,
|
||||
"verifier_observed": True,
|
||||
"verifier_passed": not positive,
|
||||
"rollback_performed": positive,
|
||||
"freshness_observed": True,
|
||||
"freshness_passed": not positive,
|
||||
"asset_identity_resolved": not positive,
|
||||
}
|
||||
|
||||
|
||||
def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
|
||||
end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC)
|
||||
start = end - timedelta(hours=24)
|
||||
scorecard = build_automation_slo_scorecard(
|
||||
project_id="awoooi",
|
||||
incident_samples=[
|
||||
_sample(
|
||||
start + timedelta(hours=1),
|
||||
lane="kubernetes_workload",
|
||||
action_seconds=10,
|
||||
resolve_seconds=100,
|
||||
positive=False,
|
||||
),
|
||||
_sample(
|
||||
start + timedelta(hours=2),
|
||||
lane="windows_vmware",
|
||||
action_seconds=20,
|
||||
resolve_seconds=200,
|
||||
positive=True,
|
||||
),
|
||||
],
|
||||
run_summary={
|
||||
"total_runs": 5,
|
||||
"non_shadow_runs": 3,
|
||||
"controlled_runs_with_steps": 2,
|
||||
"completed_runs": 4,
|
||||
"failed_runs": 1,
|
||||
},
|
||||
provider_cost_rows=[
|
||||
{
|
||||
"provider": "ollama_gcp_a",
|
||||
"model": "qwen3",
|
||||
"call_count": 3,
|
||||
"prompt_tokens": 1200,
|
||||
"completion_tokens": 300,
|
||||
"cost_usd": "0",
|
||||
},
|
||||
{
|
||||
"provider": "gemini",
|
||||
"model": "gemini-critic",
|
||||
"call_count": 1,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"cost_usd": "0.0123",
|
||||
},
|
||||
],
|
||||
window_start=start,
|
||||
window_end=end,
|
||||
)
|
||||
|
||||
assert scorecard["status"] == (
|
||||
"runtime_aggregate_ready_same_run_verification_pending"
|
||||
)
|
||||
assert scorecard["runtime_closed"] is False
|
||||
assert scorecard["metrics"] == {
|
||||
"mtta_seconds": 15.0,
|
||||
"mttr_seconds": 150.0,
|
||||
"false_positive_rate": 0.5,
|
||||
"recurrence_rate": 0.5,
|
||||
"human_intervention_rate": 0.5,
|
||||
"verifier_pass_rate": 0.5,
|
||||
"rollback_rate": 0.5,
|
||||
"signal_freshness_slo_pass_rate": 0.5,
|
||||
"asset_coverage_percent": 50.0,
|
||||
}
|
||||
assert scorecard["missing_metrics"] == []
|
||||
assert scorecard["coverage_blockers"] == []
|
||||
assert len(scorecard["lane_scorecards"]) == 2
|
||||
assert scorecard["runs"]["controlled_runs_with_steps"] == 2
|
||||
assert scorecard["provider_usage"] == {
|
||||
"rows": [
|
||||
{
|
||||
"provider": "ollama_gcp_a",
|
||||
"model": "qwen3",
|
||||
"call_count": 3,
|
||||
"prompt_tokens": 1200,
|
||||
"completion_tokens": 300,
|
||||
"cost_usd": "0.0000",
|
||||
},
|
||||
{
|
||||
"provider": "gemini",
|
||||
"model": "gemini-critic",
|
||||
"call_count": 1,
|
||||
"prompt_tokens": 100,
|
||||
"completion_tokens": 50,
|
||||
"cost_usd": "0.0123",
|
||||
},
|
||||
],
|
||||
"call_count": 4,
|
||||
"prompt_tokens": 1300,
|
||||
"completion_tokens": 350,
|
||||
"cost_usd": "0.0123",
|
||||
}
|
||||
assert scorecard["evidence"] == {
|
||||
"source": "postgresql_read_only_aggregate",
|
||||
"same_run_receipt_verification": "pending",
|
||||
"no_false_green": True,
|
||||
}
|
||||
|
||||
|
||||
def test_scorecard_does_not_false_green_missing_runtime_samples() -> None:
|
||||
end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC)
|
||||
scorecard = build_automation_slo_scorecard(
|
||||
project_id="awoooi",
|
||||
incident_samples=[],
|
||||
run_summary={},
|
||||
provider_cost_rows=[],
|
||||
window_start=end - timedelta(hours=24),
|
||||
window_end=end,
|
||||
)
|
||||
|
||||
assert scorecard["status"] == "no_runtime_sample_fail_closed"
|
||||
assert scorecard["runtime_closed"] is False
|
||||
assert set(scorecard["missing_metrics"]) == {
|
||||
"asset_coverage_percent",
|
||||
"false_positive_rate",
|
||||
"human_intervention_rate",
|
||||
"mtta_seconds",
|
||||
"mttr_seconds",
|
||||
"recurrence_rate",
|
||||
"rollback_rate",
|
||||
"signal_freshness_slo_pass_rate",
|
||||
"verifier_pass_rate",
|
||||
}
|
||||
assert scorecard["provider_usage"]["cost_usd"] == "0.0000"
|
||||
|
||||
|
||||
def test_scorecard_endpoint_is_project_scoped_and_read_only(monkeypatch) -> None:
|
||||
readback = AsyncMock(
|
||||
return_value={
|
||||
"schema_version": "sre_automation_slo_scorecard_v1",
|
||||
"project_id": "awoooi",
|
||||
"status": "partial_missing_receipt_coverage",
|
||||
"runtime_closed": False,
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(agents, "read_automation_slo_scorecard", readback)
|
||||
app = FastAPI()
|
||||
app.include_router(agents.router, prefix="/api/v1")
|
||||
|
||||
response = TestClient(app).get(
|
||||
"/api/v1/agents/sre-automation-slo-scorecard-24h",
|
||||
headers={"X-Project-ID": "awoooi"},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["runtime_closed"] is False
|
||||
readback.assert_awaited_once_with(project_id="awoooi")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_runtime_reader_executes_only_three_bounded_read_queries(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC)
|
||||
sample = _sample(
|
||||
end - timedelta(hours=1),
|
||||
lane="kubernetes_workload",
|
||||
action_seconds=10,
|
||||
resolve_seconds=100,
|
||||
positive=False,
|
||||
)
|
||||
|
||||
class Result:
|
||||
def __init__(self, rows):
|
||||
self.rows = rows
|
||||
|
||||
def mappings(self):
|
||||
return self
|
||||
|
||||
def all(self):
|
||||
return self.rows
|
||||
|
||||
def first(self):
|
||||
return self.rows[0] if self.rows else None
|
||||
|
||||
class DB:
|
||||
def __init__(self):
|
||||
self.statements = []
|
||||
self.results = [
|
||||
Result([sample]),
|
||||
Result(
|
||||
[
|
||||
{
|
||||
"total_runs": 1,
|
||||
"non_shadow_runs": 1,
|
||||
"controlled_runs_with_steps": 1,
|
||||
"completed_runs": 1,
|
||||
"failed_runs": 0,
|
||||
}
|
||||
]
|
||||
),
|
||||
Result([]),
|
||||
]
|
||||
|
||||
async def execute(self, statement, _parameters):
|
||||
self.statements.append(str(statement))
|
||||
return self.results[len(self.statements) - 1]
|
||||
|
||||
class Context:
|
||||
def __init__(self, db):
|
||||
self.db = db
|
||||
|
||||
async def __aenter__(self):
|
||||
return self.db
|
||||
|
||||
async def __aexit__(self, *_args):
|
||||
return False
|
||||
|
||||
db = DB()
|
||||
monkeypatch.setattr(
|
||||
scorecard_module,
|
||||
"get_db_context",
|
||||
lambda project_id: Context(db),
|
||||
)
|
||||
|
||||
scorecard = await read_automation_slo_scorecard(
|
||||
project_id="awoooi",
|
||||
window_end=end,
|
||||
)
|
||||
|
||||
assert len(db.statements) == 3
|
||||
assert all(
|
||||
statement.lstrip().startswith(("WITH", "SELECT")) for statement in db.statements
|
||||
)
|
||||
assert all(
|
||||
keyword not in " ".join(statement.upper().split())
|
||||
for statement in db.statements
|
||||
for keyword in (" INSERT ", " UPDATE ", " DELETE ", " ALTER ", " DROP ")
|
||||
)
|
||||
assert scorecard["metrics"]["mtta_seconds"] == 10.0
|
||||
assert scorecard["runtime_closed"] is False
|
||||
@@ -116,10 +116,10 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() -
|
||||
"total_items": 18,
|
||||
"by_priority": {"P0": 16, "P1": 2},
|
||||
"by_status": {
|
||||
"source_implemented_runtime_pending": 15,
|
||||
"planned": 3,
|
||||
"source_implemented_runtime_pending": 16,
|
||||
"planned": 2,
|
||||
},
|
||||
"source_implemented_items": 15,
|
||||
"source_implemented_items": 16,
|
||||
"runtime_closed_items": 0,
|
||||
"program_completion_percent": 0,
|
||||
"asset_coverage_status": "partial",
|
||||
@@ -158,6 +158,12 @@ def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None:
|
||||
items["AIA-SRE-014"]["confirmed_truth"]
|
||||
)
|
||||
assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"])
|
||||
assert items["AIA-SRE-016"]["status"] == (
|
||||
"source_implemented_runtime_pending"
|
||||
)
|
||||
assert "three project-scoped aggregate queries" in " ".join(
|
||||
items["AIA-SRE-016"]["confirmed_truth"]
|
||||
)
|
||||
assert items["AIA-SRE-009"]["status"] == (
|
||||
"source_implemented_runtime_pending"
|
||||
)
|
||||
@@ -200,7 +206,7 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None:
|
||||
payload = load_sre_k3s_controlled_automation_work_items()
|
||||
projection = build_sre_k3s_program_projection(payload)
|
||||
|
||||
assert projection["rollups"]["source_implemented_items"] == 15
|
||||
assert projection["rollups"]["source_implemented_items"] == 16
|
||||
assert projection["rollups"]["runtime_closed_items"] == 0
|
||||
assert projection["rollups"]["program_completion_percent"] == 0
|
||||
assert projection["domain_count"] == 8
|
||||
|
||||
@@ -799,16 +799,29 @@
|
||||
"title": "24h replay、MTTA/MTTR、誤判、復發與成本 scorecard",
|
||||
"owner_lane": "AutomationSLO",
|
||||
"risk": "low",
|
||||
"status": "planned",
|
||||
"status": "source_implemented_runtime_pending",
|
||||
"dependencies": [
|
||||
"AIA-SRE-015"
|
||||
],
|
||||
"source_refs": [],
|
||||
"source_refs": [
|
||||
"apps/api/src/services/automation_slo_scorecard.py",
|
||||
"apps/api/src/services/ai_automation_runtime_contract.py",
|
||||
"apps/api/src/api/v1/agents.py"
|
||||
],
|
||||
"executor": "offline replay worker",
|
||||
"verifier": "production aggregate versus same-run receipts",
|
||||
"rollback": "read-only scorecard",
|
||||
"exit_condition": "all lanes publish MTTA, MTTR, false positive, recurrence, human intervention, verifier and rollback metrics",
|
||||
"next_action": "replay the latest 24h alert/runs corpus after phase 1 deploy"
|
||||
"confirmed_truth": [
|
||||
"read-only scorecard executes three project-scoped aggregate queries for incident lifecycle, AwoooP runs and provider budget receipts",
|
||||
"global and per-lane projections publish MTTA, MTTR, false-positive, recurrence, human-intervention, verifier-pass, rollback, freshness and asset-coverage metrics",
|
||||
"run projection separates total, non-shadow, controlled-with-steps, completed and failed runs while provider projection preserves call, token and cost totals",
|
||||
"missing samples, classifications, verifier evidence or freshness denominators remain partial/no-runtime-sample and cannot report runtime closure"
|
||||
],
|
||||
"runtime_gaps": [
|
||||
"the exact source revision is not deployed and no production 24h aggregate has been compared with same-run receipts or recipient-visible scorecard readback"
|
||||
],
|
||||
"next_action": "deploy one exact source revision through the authorized release lane, read /api/v1/agents/sre-automation-slo-scorecard-24h after a complete 24h window, compare aggregate counts with same-run receipts and keep any missing denominator partial"
|
||||
},
|
||||
{
|
||||
"id": "AIA-SRE-017",
|
||||
@@ -897,10 +910,10 @@
|
||||
"P1": 2
|
||||
},
|
||||
"by_status": {
|
||||
"source_implemented_runtime_pending": 15,
|
||||
"planned": 3
|
||||
"source_implemented_runtime_pending": 16,
|
||||
"planned": 2
|
||||
},
|
||||
"source_implemented_items": 15,
|
||||
"source_implemented_items": 16,
|
||||
"runtime_closed_items": 0,
|
||||
"program_completion_percent": 0,
|
||||
"asset_coverage_status": "partial",
|
||||
|
||||
Reference in New Issue
Block a user