From 8cd01cb278f01a8c4acf586ba725acc26f8d83f8 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 01:47:35 +0800 Subject: [PATCH] feat(sre): add 24h automation SLO scorecard --- apps/api/src/api/v1/agents.py | 36 ++ .../src/services/automation_slo_scorecard.py | 404 ++++++++++++++++++ .../tests/test_automation_slo_scorecard.py | 275 ++++++++++++ ...3s_controlled_automation_work_items_api.py | 14 +- ...rolled-automation-work-items.snapshot.json | 25 +- 5 files changed, 744 insertions(+), 10 deletions(-) create mode 100644 apps/api/src/services/automation_slo_scorecard.py create mode 100644 apps/api/tests/test_automation_slo_scorecard.py diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 0d8dcec57..2afe56411 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -346,6 +346,7 @@ from src.services.ai_technology_radar_readback import ( from src.services.ai_technology_report_cadence_readback import ( load_latest_ai_technology_report_cadence_readback, ) +from src.services.automation_slo_scorecard import read_automation_slo_scorecard from src.services.awoooi_gitea_onboarding_warning_step_runtime_enablement_gate import ( load_latest_awoooi_gitea_onboarding_warning_step_runtime_enablement_gate, ) @@ -1248,6 +1249,41 @@ async def get_sre_k3s_controlled_automation_work_items() -> dict[str, Any]: ) from exc +@router.get( + "/sre-automation-slo-scorecard-24h", + response_model=dict[str, Any], + summary="取得最近 24 小時 SRE AI 自動化成效 scorecard", + description=( + "唯讀彙總 PostgreSQL incident、run 與 provider budget receipts，計算 " + "MTTA、MTTR、誤判、復發、人工介入、verifier、rollback、freshness、" + "asset coverage 與模型成本。缺 evidence 時保持 partial，不執行修復、" + "不呼叫模型、不讀 secret。" + ), +) +async def get_sre_automation_slo_scorecard_24h( + x_project_id: str = Header(default="awoooi", alias="X-Project-ID"), +) -> dict[str, Any]: + """回傳一個 project-scoped、no-false-green 的 24h scorecard。""" + + try: + payload = await read_automation_slo_scorecard(project_id=x_project_id) + return redact_public_lan_topology(payload) + except ValueError as exc: + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail=str(exc), + ) from exc + except Exception as exc: + logger.error( + "sre_automation_slo_scorecard_read_failed", + error_type=type(exc).__name__, + ) + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="SRE AI 自動化 24h scorecard 暫時無法讀取", + ) from exc + + @router.get( "/portfolio-infrastructure-asset-reconciliation", response_model=dict[str, Any], diff --git a/apps/api/src/services/automation_slo_scorecard.py b/apps/api/src/services/automation_slo_scorecard.py new file mode 100644 index 000000000..970e79709 --- /dev/null +++ b/apps/api/src/services/automation_slo_scorecard.py @@ -0,0 +1,404 @@ +"""Read-only 24-hour SRE automation outcome scorecard.""" + +from __future__ import annotations + +import re +from collections.abc import Mapping, Sequence +from datetime import UTC, datetime, timedelta +from decimal import Decimal +from typing import Any + +from sqlalchemy import text + +from src.db.base import get_db_context +from src.services.ai_automation_runtime_contract import ( + AI_AUTOMATION_REQUIRED_OUTCOME_METRICS, +) + +SCHEMA_VERSION = "sre_automation_slo_scorecard_v1" +_PROJECT_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$") + +_INCIDENT_SAMPLES_SQL = text( + """ + WITH scoped_events AS ( + SELECT + COALESCE(operation.incident_id, operation.id::text) AS sample_id, + operation.event_type::text AS event_type, + operation.created_at, + operation.context, + incident.resolved_at AS incident_resolved_at, + COALESCE( + NULLIF(operation.context ->> 'typed_domain', ''), + NULLIF(operation.context ->> 'alert_category', ''), + NULLIF(incident.alert_category, ''), + 'unclassified' + ) AS lane + FROM alert_operation_log operation + LEFT JOIN incidents incident + ON incident.incident_id = operation.incident_id + WHERE operation.created_at >= :window_start + AND operation.created_at < :window_end + AND COALESCE( + incident.project_id, + NULLIF(operation.context ->> 'project_id', ''), + 'awoooi' + ) = :project_id + ), per_incident AS ( + SELECT + sample_id, + COALESCE( + min(lane) FILTER (WHERE lane <> 'unclassified'), + 'unclassified' + ) AS lane, + min(created_at) FILTER ( + WHERE event_type = 'ALERT_RECEIVED' + ) AS received_at, + min(created_at) FILTER ( + WHERE event_type IN ( + 'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED', + 'PRE_FLIGHT_PASSED', 'PRE_FLIGHT_FAILED', + 'GUARDRAIL_BLOCKED', 'USER_ACTION', 'ESCALATED' + ) + ) AS first_action_at, + COALESCE( + min(created_at) FILTER (WHERE event_type = 'RESOLVED'), + max(incident_resolved_at) + ) AS resolved_at, + bool_or(context ? 'false_positive') AS false_positive_observed, + bool_or( + lower(COALESCE(context ->> 'false_positive', 'false')) + IN ('true', '1', 'yes') + ) AS false_positive, + bool_or( + context ? 'recurrence' OR context ? 'recurrence_fingerprint' + ) AS recurrence_observed, + bool_or( + lower(COALESCE(context ->> 'recurrence', 'false')) + IN ('true', '1', 'yes') + OR NULLIF(context ->> 'recurrence_fingerprint', '') IS NOT NULL + ) AS recurrence, + bool_or(event_type IN ('USER_ACTION', 'MANUAL_FIX_RECORDED')) + AS human_intervention, + bool_or(event_type IN ( + 'AUTO_REPAIR_TRIGGERED', 'EXECUTION_STARTED', + 'EXECUTION_COMPLETED', 'CHANGE_APPLIED' + )) AS execution_observed, + bool_or( + context ? 'verifier_passed' + OR context ? 'verification_result' + ) AS verifier_observed, + bool_or( + lower(COALESCE(context ->> 'verifier_passed', 'false')) + IN ('true', '1', 'yes') + OR lower(COALESCE(context ->> 'verification_result', '')) + IN ('success', 'passed', 'verified') + ) AS verifier_passed, + bool_or( + lower(COALESCE(context ->> 'rollback_performed', 'false')) + IN ('true', '1', 'yes') + ) AS rollback_performed, + bool_or(context ? 'signal_freshness_slo_pass') + AS freshness_observed, + bool_or( + lower(COALESCE( + context ->> 'signal_freshness_slo_pass', 'false' + )) IN ('true', '1', 'yes') + ) AS freshness_passed, + bool_or( + NULLIF(context ->> 'canonical_asset_id', '') IS NOT NULL + AND COALESCE(context ->> 'asset_identity_status', 'resolved') + <> 'asset_identity_unresolved' + ) AS asset_identity_resolved + FROM scoped_events + GROUP BY sample_id + ) + SELECT * + FROM per_incident + WHERE received_at IS NOT NULL + ORDER BY received_at, sample_id + """ +) + +_RUN_SUMMARY_SQL = text( + """ + SELECT + count(*)::int AS total_runs, + count(*) FILTER (WHERE is_shadow IS FALSE)::int AS non_shadow_runs, + count(*) FILTER ( + WHERE is_shadow IS FALSE AND step_count > 0 + )::int AS controlled_runs_with_steps, + count(*) FILTER (WHERE state = 'completed')::int AS completed_runs, + count(*) FILTER ( + WHERE state IN ('failed', 'cancelled', 'timeout') + )::int AS failed_runs + FROM awooop_run_state + WHERE project_id = :project_id + AND created_at >= :window_start + AND created_at < :window_end + """ +) + +_PROVIDER_COST_SQL = text( + """ + SELECT + COALESCE(provider, 'unattributed') AS provider, + COALESCE(model, 'unattributed') AS model, + count(*)::int AS call_count, + COALESCE(sum(prompt_tokens), 0)::bigint AS prompt_tokens, + COALESCE(sum(completion_tokens), 0)::bigint AS completion_tokens, + COALESCE(sum(cost_usd), 0) AS cost_usd + FROM budget_ledger + WHERE project_id = :project_id + AND recorded_at >= :window_start + AND recorded_at < :window_end + GROUP BY provider, model + ORDER BY provider, model + """ +) + + +def _flag(value: object) -> bool: + if isinstance(value, bool): + return value + return str(value or "").strip().lower() in {"true", "1", "yes"} + + +def _as_datetime(value: object) -> datetime | None: + if isinstance(value, datetime): + parsed = value + elif isinstance(value, str) and value.strip(): + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + else: + return None + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=UTC) + return parsed.astimezone(UTC) + + +def _rate(numerator: int, denominator: int) -> float | None: + return round(numerator / denominator, 4) if denominator else None + + +def _average(values: Sequence[float]) -> float | None: + return round(sum(values) / len(values), 2) if values else None + + +def _metrics_for( + samples: Sequence[Mapping[str, Any]], +) -> tuple[dict[str, Any], dict[str, int]]: + mtta: list[float] = [] + mttr: list[float] = [] + for sample in samples: + received_at = _as_datetime(sample.get("received_at")) + first_action_at = _as_datetime(sample.get("first_action_at")) + resolved_at = _as_datetime(sample.get("resolved_at")) + if received_at and first_action_at and first_action_at >= received_at: + mtta.append((first_action_at - received_at).total_seconds()) + if received_at and resolved_at and resolved_at >= received_at: + mttr.append((resolved_at - received_at).total_seconds()) + + total = len(samples) + false_observed = sum(_flag(row.get("false_positive_observed")) for row in samples) + recurrence_observed = sum(_flag(row.get("recurrence_observed")) for row in samples) + execution_observed = sum(_flag(row.get("execution_observed")) for row in samples) + verifier_observed = sum(_flag(row.get("verifier_observed")) for row in samples) + freshness_observed = sum(_flag(row.get("freshness_observed")) for row in samples) + denominators = { + "alert_count": total, + "mtta_sample_count": len(mtta), + "mttr_sample_count": len(mttr), + "false_positive_classified_count": false_observed, + "recurrence_classified_count": recurrence_observed, + "execution_observed_count": execution_observed, + "verifier_observed_count": verifier_observed, + "freshness_observed_count": freshness_observed, + } + metrics = { + "mtta_seconds": _average(mtta), + "mttr_seconds": _average(mttr), + "false_positive_rate": _rate( + sum(_flag(row.get("false_positive")) for row in samples), + false_observed, + ), + "recurrence_rate": _rate( + sum(_flag(row.get("recurrence")) for row in samples), + recurrence_observed, + ), + "human_intervention_rate": _rate( + sum(_flag(row.get("human_intervention")) for row in samples), + total, + ), + "verifier_pass_rate": _rate( + sum(_flag(row.get("verifier_passed")) for row in samples), + verifier_observed, + ), + "rollback_rate": _rate( + sum(_flag(row.get("rollback_performed")) for row in samples), + execution_observed, + ), + "signal_freshness_slo_pass_rate": _rate( + sum(_flag(row.get("freshness_passed")) for row in samples), + freshness_observed, + ), + "asset_coverage_percent": ( + round( + sum(_flag(row.get("asset_identity_resolved")) for row in samples) + / total + * 100, + 2, + ) + if total + else None + ), + } + return metrics, denominators + + +def build_automation_slo_scorecard( + *, + project_id: str, + incident_samples: Sequence[Mapping[str, Any]], + run_summary: Mapping[str, Any] | None, + provider_cost_rows: Sequence[Mapping[str, Any]], + window_start: datetime, + window_end: datetime, +) -> dict[str, Any]: + """Build a no-false-green scorecard from sanitized aggregate rows.""" + + metrics, denominators = _metrics_for(incident_samples) + lanes: dict[str, list[Mapping[str, Any]]] = {} + for sample in incident_samples: + lane = str(sample.get("lane") or "unclassified")[:64] + lanes.setdefault(lane, []).append(sample) + lane_scorecards = [] + for lane, samples in sorted(lanes.items()): + lane_metrics, lane_denominators = _metrics_for(samples) + lane_scorecards.append( + { + "lane": lane, + "metrics": lane_metrics, + "denominators": lane_denominators, + } + ) + + missing_metrics = sorted( + metric + for metric in AI_AUTOMATION_REQUIRED_OUTCOME_METRICS + if metrics.get(metric) is None + ) + coverage_blockers = [] + alert_count = denominators["alert_count"] + for key in ( + "mtta_sample_count", + "mttr_sample_count", + "false_positive_classified_count", + "recurrence_classified_count", + "freshness_observed_count", + ): + if denominators[key] < alert_count: + coverage_blockers.append(f"{key}_below_alert_count") + if ( + denominators["verifier_observed_count"] + < denominators["execution_observed_count"] + ): + coverage_blockers.append("verifier_observed_count_below_execution_count") + + costs = [] + for row in provider_cost_rows: + cost = Decimal(str(row.get("cost_usd") or "0")) + costs.append( + { + "provider": str(row.get("provider") or "unattributed")[:32], + "model": str(row.get("model") or "unattributed")[:64], + "call_count": int(row.get("call_count") or 0), + "prompt_tokens": int(row.get("prompt_tokens") or 0), + "completion_tokens": int(row.get("completion_tokens") or 0), + "cost_usd": f"{cost:.4f}", + } + ) + total_cost = sum((Decimal(row["cost_usd"]) for row in costs), Decimal("0")) + status = ( + "no_runtime_sample_fail_closed" + if not incident_samples + else "partial_missing_receipt_coverage" + if missing_metrics or coverage_blockers + else "runtime_aggregate_ready_same_run_verification_pending" + ) + safe_runs = { + key: int((run_summary or {}).get(key) or 0) + for key in ( + "total_runs", + "non_shadow_runs", + "controlled_runs_with_steps", + "completed_runs", + "failed_runs", + ) + } + return { + "schema_version": SCHEMA_VERSION, + "project_id": project_id, + "window": { + "hours": 24, + "start": window_start.astimezone(UTC).isoformat(), + "end": window_end.astimezone(UTC).isoformat(), + }, + "status": status, + "runtime_closed": False, + "metrics": metrics, + "denominators": denominators, + "lane_scorecards": lane_scorecards, + "runs": safe_runs, + "provider_usage": { + "rows": costs, + "call_count": sum(row["call_count"] for row in costs), + "prompt_tokens": sum(row["prompt_tokens"] for row in costs), + "completion_tokens": sum(row["completion_tokens"] for row in costs), + "cost_usd": f"{total_cost:.4f}", + }, + "missing_metrics": missing_metrics, + "coverage_blockers": coverage_blockers, + "evidence": { + "source": "postgresql_read_only_aggregate", + "same_run_receipt_verification": "pending", + "no_false_green": True, + }, + } + + +async def read_automation_slo_scorecard( + *, + project_id: str = "awoooi", + window_end: datetime | None = None, +) -> dict[str, Any]: + """Read the latest 24 hours without mutating runtime state.""" + + if not _PROJECT_ID.fullmatch(project_id): + raise ValueError("project_id_invalid") + end = window_end or datetime.now(UTC) + if end.tzinfo is None: + end = end.replace(tzinfo=UTC) + end = end.astimezone(UTC) + start = end - timedelta(hours=24) + parameters = { + "project_id": project_id, + "window_start": start, + "window_end": end, + } + async with get_db_context(project_id) as db: + incident_result = await db.execute(_INCIDENT_SAMPLES_SQL, parameters) + run_result = await db.execute(_RUN_SUMMARY_SQL, parameters) + cost_result = await db.execute(_PROVIDER_COST_SQL, parameters) + incident_samples = [dict(row) for row in incident_result.mappings().all()] + run_row = run_result.mappings().first() + provider_cost_rows = [dict(row) for row in cost_result.mappings().all()] + return build_automation_slo_scorecard( + project_id=project_id, + incident_samples=incident_samples, + run_summary=dict(run_row) if run_row else {}, + provider_cost_rows=provider_cost_rows, + window_start=start, + window_end=end, + ) diff --git a/apps/api/tests/test_automation_slo_scorecard.py b/apps/api/tests/test_automation_slo_scorecard.py new file mode 100644 index 000000000..9cd28f951 --- /dev/null +++ b/apps/api/tests/test_automation_slo_scorecard.py @@ -0,0 +1,275 @@ +from datetime import UTC, datetime, timedelta +from unittest.mock import AsyncMock + +import pytest +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1 import agents +from src.services import automation_slo_scorecard as scorecard_module +from src.services.automation_slo_scorecard import ( + build_automation_slo_scorecard, + read_automation_slo_scorecard, +) + + +def _sample( + start: datetime, + *, + lane: str, + action_seconds: int, + resolve_seconds: int, + positive: bool, +) -> dict: + return { + "lane": lane, + "received_at": start, + "first_action_at": start + timedelta(seconds=action_seconds), + "resolved_at": start + timedelta(seconds=resolve_seconds), + "false_positive_observed": True, + "false_positive": positive, + "recurrence_observed": True, + "recurrence": positive, + "human_intervention": positive, + "execution_observed": True, + "verifier_observed": True, + "verifier_passed": not positive, + "rollback_performed": positive, + "freshness_observed": True, + "freshness_passed": not positive, + "asset_identity_resolved": not positive, + } + + +def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None: + end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC) + start = end - timedelta(hours=24) + scorecard = build_automation_slo_scorecard( + project_id="awoooi", + incident_samples=[ + _sample( + start + timedelta(hours=1), + lane="kubernetes_workload", + action_seconds=10, + resolve_seconds=100, + positive=False, + ), + _sample( + start + timedelta(hours=2), + lane="windows_vmware", + action_seconds=20, + resolve_seconds=200, + positive=True, + ), + ], + run_summary={ + "total_runs": 5, + "non_shadow_runs": 3, + "controlled_runs_with_steps": 2, + "completed_runs": 4, + "failed_runs": 1, + }, + provider_cost_rows=[ + { + "provider": "ollama_gcp_a", + "model": "qwen3", + "call_count": 3, + "prompt_tokens": 1200, + "completion_tokens": 300, + "cost_usd": "0", + }, + { + "provider": "gemini", + "model": "gemini-critic", + "call_count": 1, + "prompt_tokens": 100, + "completion_tokens": 50, + "cost_usd": "0.0123", + }, + ], + window_start=start, + window_end=end, + ) + + assert scorecard["status"] == ( + "runtime_aggregate_ready_same_run_verification_pending" + ) + assert scorecard["runtime_closed"] is False + assert scorecard["metrics"] == { + "mtta_seconds": 15.0, + "mttr_seconds": 150.0, + "false_positive_rate": 0.5, + "recurrence_rate": 0.5, + "human_intervention_rate": 0.5, + "verifier_pass_rate": 0.5, + "rollback_rate": 0.5, + "signal_freshness_slo_pass_rate": 0.5, + "asset_coverage_percent": 50.0, + } + assert scorecard["missing_metrics"] == [] + assert scorecard["coverage_blockers"] == [] + assert len(scorecard["lane_scorecards"]) == 2 + assert scorecard["runs"]["controlled_runs_with_steps"] == 2 + assert scorecard["provider_usage"] == { + "rows": [ + { + "provider": "ollama_gcp_a", + "model": "qwen3", + "call_count": 3, + "prompt_tokens": 1200, + "completion_tokens": 300, + "cost_usd": "0.0000", + }, + { + "provider": "gemini", + "model": "gemini-critic", + "call_count": 1, + "prompt_tokens": 100, + "completion_tokens": 50, + "cost_usd": "0.0123", + }, + ], + "call_count": 4, + "prompt_tokens": 1300, + "completion_tokens": 350, + "cost_usd": "0.0123", + } + assert scorecard["evidence"] == { + "source": "postgresql_read_only_aggregate", + "same_run_receipt_verification": "pending", + "no_false_green": True, + } + + +def test_scorecard_does_not_false_green_missing_runtime_samples() -> None: + end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC) + scorecard = build_automation_slo_scorecard( + project_id="awoooi", + incident_samples=[], + run_summary={}, + provider_cost_rows=[], + window_start=end - timedelta(hours=24), + window_end=end, + ) + + assert scorecard["status"] == "no_runtime_sample_fail_closed" + assert scorecard["runtime_closed"] is False + assert set(scorecard["missing_metrics"]) == { + "asset_coverage_percent", + "false_positive_rate", + "human_intervention_rate", + "mtta_seconds", + "mttr_seconds", + "recurrence_rate", + "rollback_rate", + "signal_freshness_slo_pass_rate", + "verifier_pass_rate", + } + assert scorecard["provider_usage"]["cost_usd"] == "0.0000" + + +def test_scorecard_endpoint_is_project_scoped_and_read_only(monkeypatch) -> None: + readback = AsyncMock( + return_value={ + "schema_version": "sre_automation_slo_scorecard_v1", + "project_id": "awoooi", + "status": "partial_missing_receipt_coverage", + "runtime_closed": False, + } + ) + monkeypatch.setattr(agents, "read_automation_slo_scorecard", readback) + app = FastAPI() + app.include_router(agents.router, prefix="/api/v1") + + response = TestClient(app).get( + "/api/v1/agents/sre-automation-slo-scorecard-24h", + headers={"X-Project-ID": "awoooi"}, + ) + + assert response.status_code == 200 + assert response.json()["runtime_closed"] is False + readback.assert_awaited_once_with(project_id="awoooi") + + +@pytest.mark.asyncio +async def test_runtime_reader_executes_only_three_bounded_read_queries( + monkeypatch, +) -> None: + end = datetime(2026, 7, 19, 4, 0, tzinfo=UTC) + sample = _sample( + end - timedelta(hours=1), + lane="kubernetes_workload", + action_seconds=10, + resolve_seconds=100, + positive=False, + ) + + class Result: + def __init__(self, rows): + self.rows = rows + + def mappings(self): + return self + + def all(self): + return self.rows + + def first(self): + return self.rows[0] if self.rows else None + + class DB: + def __init__(self): + self.statements = [] + self.results = [ + Result([sample]), + Result( + [ + { + "total_runs": 1, + "non_shadow_runs": 1, + "controlled_runs_with_steps": 1, + "completed_runs": 1, + "failed_runs": 0, + } + ] + ), + Result([]), + ] + + async def execute(self, statement, _parameters): + self.statements.append(str(statement)) + return self.results[len(self.statements) - 1] + + class Context: + def __init__(self, db): + self.db = db + + async def __aenter__(self): + return self.db + + async def __aexit__(self, *_args): + return False + + db = DB() + monkeypatch.setattr( + scorecard_module, + "get_db_context", + lambda project_id: Context(db), + ) + + scorecard = await read_automation_slo_scorecard( + project_id="awoooi", + window_end=end, + ) + + assert len(db.statements) == 3 + assert all( + statement.lstrip().startswith(("WITH", "SELECT")) for statement in db.statements + ) + assert all( + keyword not in " ".join(statement.upper().split()) + for statement in db.statements + for keyword in (" INSERT ", " UPDATE ", " DELETE ", " ALTER ", " DROP ") + ) + assert scorecard["metrics"]["mtta_seconds"] == 10.0 + assert scorecard["runtime_closed"] is False diff --git a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py index 36f046809..55d9aaaeb 100644 --- a/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py +++ b/apps/api/tests/test_sre_k3s_controlled_automation_work_items_api.py @@ -116,10 +116,10 @@ def test_loader_returns_fixed_architecture_provider_order_and_agent99_bridge() - "total_items": 18, "by_priority": {"P0": 16, "P1": 2}, "by_status": { - "source_implemented_runtime_pending": 15, - "planned": 3, + "source_implemented_runtime_pending": 16, + "planned": 2, }, - "source_implemented_items": 15, + "source_implemented_items": 16, "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial", @@ -158,6 +158,12 @@ def test_ledger_links_confirmed_runtime_gaps_without_false_closure() -> None: items["AIA-SRE-014"]["confirmed_truth"] ) assert "host99 Agent99" in " ".join(items["AIA-SRE-017"]["runtime_gaps"]) + assert items["AIA-SRE-016"]["status"] == ( + "source_implemented_runtime_pending" + ) + assert "three project-scoped aggregate queries" in " ".join( + items["AIA-SRE-016"]["confirmed_truth"] + ) assert items["AIA-SRE-009"]["status"] == ( "source_implemented_runtime_pending" ) @@ -200,7 +206,7 @@ def test_projection_keeps_program_asset_and_runtime_truth_separate() -> None: payload = load_sre_k3s_controlled_automation_work_items() projection = build_sre_k3s_program_projection(payload) - assert projection["rollups"]["source_implemented_items"] == 15 + assert projection["rollups"]["source_implemented_items"] == 16 assert projection["rollups"]["runtime_closed_items"] == 0 assert projection["rollups"]["program_completion_percent"] == 0 assert projection["domain_count"] == 8 diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json index c7cbbcdb1..0a75d51c1 100644 --- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json +++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json @@ -799,16 +799,29 @@ "title": "24h replay、MTTA/MTTR、誤判、復發與成本 scorecard", "owner_lane": "AutomationSLO", "risk": "low", - "status": "planned", + "status": "source_implemented_runtime_pending", "dependencies": [ "AIA-SRE-015" ], - "source_refs": [], + "source_refs": [ + "apps/api/src/services/automation_slo_scorecard.py", + "apps/api/src/services/ai_automation_runtime_contract.py", + "apps/api/src/api/v1/agents.py" + ], "executor": "offline replay worker", "verifier": "production aggregate versus same-run receipts", "rollback": "read-only scorecard", "exit_condition": "all lanes publish MTTA, MTTR, false positive, recurrence, human intervention, verifier and rollback metrics", - "next_action": "replay the latest 24h alert/runs corpus after phase 1 deploy" + "confirmed_truth": [ + "read-only scorecard executes three project-scoped aggregate queries for incident lifecycle, AwoooP runs and provider budget receipts", + "global and per-lane projections publish MTTA, MTTR, false-positive, recurrence, human-intervention, verifier-pass, rollback, freshness and asset-coverage metrics", + "run projection separates total, non-shadow, controlled-with-steps, completed and failed runs while provider projection preserves call, token and cost totals", + "missing samples, classifications, verifier evidence or freshness denominators remain partial/no-runtime-sample and cannot report runtime closure" + ], + "runtime_gaps": [ + "the exact source revision is not deployed and no production 24h aggregate has been compared with same-run receipts or recipient-visible scorecard readback" + ], + "next_action": "deploy one exact source revision through the authorized release lane, read /api/v1/agents/sre-automation-slo-scorecard-24h after a complete 24h window, compare aggregate counts with same-run receipts and keep any missing denominator partial" }, { "id": "AIA-SRE-017", @@ -897,10 +910,10 @@ "P1": 2 }, "by_status": { - "source_implemented_runtime_pending": 15, - "planned": 3 + "source_implemented_runtime_pending": 16, + "planned": 2 }, - "source_implemented_items": 15, + "source_implemented_items": 16, "runtime_closed_items": 0, "program_completion_percent": 0, "asset_coverage_status": "partial",