feat(sre): enforce typed controlled automation

scripts/generate_monitoring.py

@@ -45,7 +45,41 @@ EXPECTED_JOBS = {
     "prometheus": "Prometheus self-scrape",
     "blackbox-http": "Blackbox HTTP probe",
     "blackbox-tcp": "Blackbox TCP probe",
-    "github-actions": "GitHub Actions exporter",
+    "gitea-native": "Gitea native metrics",
+}
+
+# Jobs alone are not sufficient evidence for multi-target scrape jobs.  A
+# healthy website target can keep ``blackbox-http`` green while both AI
+# provider probes are absent.  Keep the exact cloud-edge identities here so
+# source/runtime reconciliation fails closed on missing, down, or duplicated
+# provider targets.
+EXPECTED_TARGETS = {
+    "ollama_gcp_a_cloud_edge": {
+        "description": "GCP-A Ollama cloud-edge /api/tags from host110",
+        "labels": {
+            "job": "blackbox-http",
+            "provider": "ollama_gcp_a",
+            "probe_origin": "host110",
+            "boundary": "cloud_edge_availability_only",
+            "instance": "http://34.143.170.20:11434/api/tags",
+        },
+    },
+    "ollama_gcp_b_cloud_edge": {
+        "description": "GCP-B Ollama cloud-edge /api/tags from host110",
+        "labels": {
+            "job": "blackbox-http",
+            "provider": "ollama_gcp_b",
+            "probe_origin": "host110",
+            "boundary": "cloud_edge_availability_only",
+            "instance": "http://34.21.145.224:11434/api/tags",
+        },
+    },
+}
+
+# GitHub is globally frozen.  A legacy exporter still appearing UP must be
+# reported as policy drift, not counted as harmless surplus coverage.
+FORBIDDEN_JOBS = {
+    "github-actions": "retired GitHub exporter must be removed from production",
 }
 
 # 允許 down 的 target (已知問題，不影響覆蓋率計算)
@@ -116,6 +150,42 @@ def analyze_targets(targets_data: dict) -> dict:
     return jobs
 
 
+def analyze_expected_targets(targets_data: dict) -> dict:
+    """Resolve exact multi-target identities without allowing job-level false green."""
+    active = targets_data.get("activeTargets", [])
+    target_status: dict[str, dict] = {}
+
+    for target_id, expected in EXPECTED_TARGETS.items():
+        expected_labels = expected["labels"]
+        matches = [
+            target
+            for target in active
+            if all(
+                target.get("labels", {}).get(label) == value
+                for label, value in expected_labels.items()
+            )
+        ]
+        health_values = [str(target.get("health", "unknown")) for target in matches]
+        if not matches:
+            state = "missing"
+        elif len(matches) != 1:
+            state = "ambiguous"
+        elif health_values == ["up"]:
+            state = "up"
+        else:
+            state = "down"
+
+        target_status[target_id] = {
+            "description": expected["description"],
+            "expected_labels": expected_labels,
+            "match_count": len(matches),
+            "health": health_values,
+            "state": state,
+        }
+
+    return target_status
+
+
 def build_report(jobs: dict) -> dict:
     """建立覆蓋率報告"""
     total_jobs = len(jobs)
@@ -132,6 +202,7 @@ def build_report(jobs: dict) -> dict:
 
     expected_covered = sum(1 for j in EXPECTED_JOBS if j in jobs and jobs[j]["up"])
     coverage_pct = round(expected_covered / len(EXPECTED_JOBS) * 100, 1)
+    forbidden_jobs_present = [job for job in FORBIDDEN_JOBS if job in jobs]
 
     return {
         "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
@@ -143,23 +214,65 @@ def build_report(jobs: dict) -> dict:
             "down_jobs": down_jobs,
             "real_down_jobs": len(real_down_jobs),
             "expected_coverage_pct": coverage_pct,
+            "forbidden_jobs_present": len(forbidden_jobs_present),
         },
         "jobs": jobs,
         "expected_jobs": EXPECTED_JOBS,
         "known_down": KNOWN_DOWN_TARGETS,
         "real_down_jobs": list(real_down_jobs.keys()),
         "missing_expected": [j for j in EXPECTED_JOBS if j not in jobs],
+        "forbidden_jobs": FORBIDDEN_JOBS,
+        "forbidden_jobs_present": forbidden_jobs_present,
     }
 
 
 def build_report_from_targets(targets_data: dict) -> dict:
     """從 Prometheus targets API payload 建立覆蓋率報告"""
-    return build_report(analyze_targets(targets_data))
+    report = build_report(analyze_targets(targets_data))
+    expected_targets = analyze_expected_targets(targets_data)
+    missing_expected_targets = [
+        target_id
+        for target_id, target in expected_targets.items()
+        if target["state"] == "missing"
+    ]
+    down_expected_targets = [
+        target_id
+        for target_id, target in expected_targets.items()
+        if target["state"] == "down"
+    ]
+    ambiguous_expected_targets = [
+        target_id
+        for target_id, target in expected_targets.items()
+        if target["state"] == "ambiguous"
+    ]
+    report["expected_targets"] = expected_targets
+    report["missing_expected_targets"] = missing_expected_targets
+    report["down_expected_targets"] = down_expected_targets
+    report["ambiguous_expected_targets"] = ambiguous_expected_targets
+    report["summary"].update(
+        {
+            "expected_target_count": len(expected_targets),
+            "expected_targets_up": sum(
+                target["state"] == "up" for target in expected_targets.values()
+            ),
+            "expected_targets_missing": len(missing_expected_targets),
+            "expected_targets_down": len(down_expected_targets),
+            "expected_targets_ambiguous": len(ambiguous_expected_targets),
+        }
+    )
+    return report
 
 
 def report_needs_stabilization(report: dict) -> bool:
     """是否需要重查，避免 post-deploy 瞬間 scrape 狀態造成 false red."""
-    return bool(report["real_down_jobs"] or report["missing_expected"])
+    return bool(
+        report["real_down_jobs"]
+        or report["missing_expected"]
+        or report["missing_expected_targets"]
+        or report["down_expected_targets"]
+        or report["ambiguous_expected_targets"]
+        or report["forbidden_jobs_present"]
+    )
 
 
 def stabilization_reason(report: dict) -> str:
@@ -168,6 +281,24 @@ def stabilization_reason(report: dict) -> str:
         parts.append(f"real_down={','.join(report['real_down_jobs'])}")
     if report["missing_expected"]:
         parts.append(f"missing_expected={','.join(report['missing_expected'])}")
+    if report["missing_expected_targets"]:
+        parts.append(
+            "missing_expected_targets="
+            f"{','.join(report['missing_expected_targets'])}"
+        )
+    if report["down_expected_targets"]:
+        parts.append(
+            f"down_expected_targets={','.join(report['down_expected_targets'])}"
+        )
+    if report["ambiguous_expected_targets"]:
+        parts.append(
+            "ambiguous_expected_targets="
+            f"{','.join(report['ambiguous_expected_targets'])}"
+        )
+    if report["forbidden_jobs_present"]:
+        parts.append(
+            f"forbidden_jobs_present={','.join(report['forbidden_jobs_present'])}"
+        )
     return "; ".join(parts) if parts else "stable"
 
 
@@ -226,18 +357,26 @@ def print_human_report(report: dict) -> None:
     """輸出人可讀格式報告"""
     s = report["summary"]
     print(f"\n{'='*60}")
-    print(f"  AWOOOI 監控覆蓋率報告")
+    print("  AWOOOI 監控覆蓋率報告")
     print(f"  生成時間: {report['generated_at']}")
     print(f"{'='*60}")
-    print(f"\n📊 總覽")
+    print("\n📊 總覽")
     print(f"  Jobs 總數:        {s['total_jobs']}")
     print(f"  全部 UP:          {s['up_jobs']}")
     print(f"  部分 UP:          {s['partial_jobs']}")
     print(f"  全部 DOWN:        {s['down_jobs']}")
     print(f"  真實問題 (非已知): {s['real_down_jobs']}")
     print(f"  預期覆蓋率:       {s['expected_coverage_pct']}% ({COVERAGE_THRESHOLD}% 門檻)")
+    print(
+        "  精確 Target:      "
+        f"{s['expected_targets_up']}/{s['expected_target_count']} UP "
+        f"(missing={s['expected_targets_missing']}, "
+        f"down={s['expected_targets_down']}, "
+        f"ambiguous={s['expected_targets_ambiguous']})"
+    )
+    print(f"  禁止 Job 漂移:   {s['forbidden_jobs_present']}")
 
-    print(f"\n✅ 預期服務狀態")
+    print("\n✅ 預期服務狀態")
     for job, desc in report["expected_jobs"].items():
         jobs = report["jobs"]
         if job not in jobs:
@@ -256,24 +395,42 @@ def print_human_report(report: dict) -> None:
         if job in report["jobs"] and report["jobs"][job]["down"]
     ]
     if known_down_present:
-        print(f"\n⚠️  已知 DOWN (不影響覆蓋率)")
+        print("\n⚠️  已知 DOWN (不影響覆蓋率)")
         for job, reason in known_down_present:
             print(f"  {job:<30} {reason}")
 
     if report["real_down_jobs"]:
-        print(f"\n🔴 需處理的 DOWN targets")
+        print("\n🔴 需處理的 DOWN targets")
         for job in report["real_down_jobs"]:
             instances = report["jobs"][job].get("down", [])
             print(f"  {job}: {', '.join(instances)}")
 
     if report["missing_expected"]:
-        print(f"\n🔴 缺少預期服務監控")
+        print("\n🔴 缺少預期服務監控")
         for job in report["missing_expected"]:
             print(f"  {job}: {report['expected_jobs'][job]}")
 
+    if report["forbidden_jobs_present"]:
+        print("\n🔴 禁止的 legacy monitoring job 仍存在")
+        for job in report["forbidden_jobs_present"]:
+            print(f"  {job}: {report['forbidden_jobs'][job]}")
+
+    if (
+        report["missing_expected_targets"]
+        or report["down_expected_targets"]
+        or report["ambiguous_expected_targets"]
+    ):
+        print("\n🔴 精確 Target identity/readback 未通過")
+        for target_id, target in report["expected_targets"].items():
+            if target["state"] != "up":
+                print(
+                    f"  {target_id}: {target['state']} "
+                    f"matches={target['match_count']} {target['description']}"
+                )
+
     stabilization = report.get("stabilization")
     if stabilization and stabilization["attempt"] > 1:
-        print(f"\n⏱️  Prometheus target 穩定化")
+        print("\n⏱️  Prometheus target 穩定化")
         print(
             "  "
             f"{stabilization['status']} after "
@@ -282,7 +439,19 @@ def print_human_report(report: dict) -> None:
 
     pct = s["expected_coverage_pct"]
     threshold = COVERAGE_THRESHOLD
-    if pct >= threshold and not report["real_down_jobs"]:
+    exact_targets_healthy = not (
+        report["missing_expected_targets"]
+        or report["down_expected_targets"]
+        or report["ambiguous_expected_targets"]
+    )
+    monitoring_contract_healthy = (
+        exact_targets_healthy and not report["forbidden_jobs_present"]
+    )
+    if (
+        pct >= threshold
+        and not report["real_down_jobs"]
+        and monitoring_contract_healthy
+    ):
         print(f"\n✅ 監控健康: 覆蓋率 {pct}% >= {threshold}%，無真實問題\n")
     elif pct >= threshold:
         print(f"\n⚠️  覆蓋率達標 ({pct}%)，但有 {s['real_down_jobs']} 個真實 DOWN 需處理\n")
@@ -335,7 +504,21 @@ def main() -> None:
     if args.check:
         pct = report["summary"]["expected_coverage_pct"]
         real_down = report["summary"]["real_down_jobs"]
-        if pct < COVERAGE_THRESHOLD or real_down > 0:
+        exact_target_failures = sum(
+            report["summary"][key]
+            for key in (
+                "expected_targets_missing",
+                "expected_targets_down",
+                "expected_targets_ambiguous",
+            )
+        )
+        forbidden_jobs = report["summary"]["forbidden_jobs_present"]
+        if (
+            pct < COVERAGE_THRESHOLD
+            or real_down > 0
+            or exact_target_failures > 0
+            or forbidden_jobs > 0
+        ):
             sys.exit(1)