From 3b1a42941907e07577f39cbd18ecbb91ffacc7ef Mon Sep 17 00:00:00 2001
From: Your Name <you@example.com>
Date: Sun, 19 Jul 2026 03:34:10 +0800
Subject: [PATCH] fix(sre): separate controlled run outcomes

---
 .../src/services/automation_slo_scorecard.py  | 14 +++++++++++-
 .../tests/test_automation_slo_scorecard.py    | 22 +++++++++++++++++++
 ...rolled-automation-work-items.snapshot.json |  2 +-
 3 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/apps/api/src/services/automation_slo_scorecard.py b/apps/api/src/services/automation_slo_scorecard.py
index 970e79709..2d0979a51 100644
--- a/apps/api/src/services/automation_slo_scorecard.py
+++ b/apps/api/src/services/automation_slo_scorecard.py
@@ -130,7 +130,17 @@ _RUN_SUMMARY_SQL = text(
         count(*) FILTER (WHERE state = 'completed')::int AS completed_runs,
         count(*) FILTER (
             WHERE state IN ('failed', 'cancelled', 'timeout')
-        )::int AS failed_runs
+        )::int AS failed_runs,
+        count(*) FILTER (
+            WHERE is_shadow IS FALSE
+              AND step_count > 0
+              AND state = 'completed'
+        )::int AS controlled_completed_runs,
+        count(*) FILTER (
+            WHERE is_shadow IS FALSE
+              AND step_count > 0
+              AND state IN ('failed', 'cancelled', 'timeout')
+        )::int AS controlled_failed_runs
     FROM awooop_run_state
     WHERE project_id = :project_id
       AND created_at >= :window_start
@@ -335,6 +345,8 @@ def build_automation_slo_scorecard(
             "controlled_runs_with_steps",
             "completed_runs",
             "failed_runs",
+            "controlled_completed_runs",
+            "controlled_failed_runs",
         )
     }
     return {
diff --git a/apps/api/tests/test_automation_slo_scorecard.py b/apps/api/tests/test_automation_slo_scorecard.py
index 9cd28f951..b9ae11b4e 100644
--- a/apps/api/tests/test_automation_slo_scorecard.py
+++ b/apps/api/tests/test_automation_slo_scorecard.py
@@ -68,6 +68,8 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
             "controlled_runs_with_steps": 2,
             "completed_runs": 4,
             "failed_runs": 1,
+            "controlled_completed_runs": 1,
+            "controlled_failed_runs": 1,
         },
         provider_cost_rows=[
             {
@@ -110,6 +112,9 @@ def test_scorecard_computes_all_required_metrics_and_provider_cost() -> None:
     assert scorecard["coverage_blockers"] == []
     assert len(scorecard["lane_scorecards"]) == 2
     assert scorecard["runs"]["controlled_runs_with_steps"] == 2
+    assert scorecard["runs"]["completed_runs"] == 4
+    assert scorecard["runs"]["controlled_completed_runs"] == 1
+    assert scorecard["runs"]["controlled_failed_runs"] == 1
     assert scorecard["provider_usage"] == {
         "rows": [
             {
@@ -168,6 +173,20 @@ def test_scorecard_does_not_false_green_missing_runtime_samples() -> None:
     assert scorecard["provider_usage"]["cost_usd"] == "0.0000"
 
 
+def test_run_query_excludes_shadow_and_no_step_runs_from_controlled_outcomes() -> None:
+    statement = " ".join(str(scorecard_module._RUN_SUMMARY_SQL).split())
+
+    assert (
+        "WHERE is_shadow IS FALSE AND step_count > 0 AND state = 'completed'"
+        in statement
+    )
+    assert (
+        "WHERE is_shadow IS FALSE AND step_count > 0 "
+        "AND state IN ('failed', 'cancelled', 'timeout')"
+        in statement
+    )
+
+
 def test_scorecard_endpoint_is_project_scoped_and_read_only(monkeypatch) -> None:
     readback = AsyncMock(
         return_value={
@@ -230,6 +249,8 @@ async def test_runtime_reader_executes_only_three_bounded_read_queries(
                             "controlled_runs_with_steps": 1,
                             "completed_runs": 1,
                             "failed_runs": 0,
+                            "controlled_completed_runs": 1,
+                            "controlled_failed_runs": 0,
                         }
                     ]
                 ),
@@ -272,4 +293,5 @@ async def test_runtime_reader_executes_only_three_bounded_read_queries(
         for keyword in (" INSERT ", " UPDATE ", " DELETE ", " ALTER ", " DROP ")
     )
     assert scorecard["metrics"]["mtta_seconds"] == 10.0
+    assert scorecard["runs"]["controlled_completed_runs"] == 1
     assert scorecard["runtime_closed"] is False
diff --git a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json
index 5c3f681d5..9a76b233d 100644
--- a/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json
+++ b/docs/operations/sre-k3s-controlled-automation-work-items.snapshot.json
@@ -852,7 +852,7 @@
       "confirmed_truth": [
         "read-only scorecard executes three project-scoped aggregate queries for incident lifecycle, AwoooP runs and provider budget receipts",
         "global and per-lane projections publish MTTA, MTTR, false-positive, recurrence, human-intervention, verifier-pass, rollback, freshness and asset-coverage metrics",
-        "run projection separates total, non-shadow, controlled-with-steps, completed and failed runs while provider projection preserves call, token and cost totals",
+        "run projection preserves raw total/completed/failed counts while separately publishing non-shadow controlled-with-steps completed and failed counts, so shadow or no-step runs cannot inflate true automation outcomes; provider projection preserves call, token and cost totals",
         "missing samples, classifications, verifier evidence or freshness denominators remain partial/no-runtime-sample and cannot report runtime closure"
       ],
       "runtime_gaps": [