fix(backup): cross-check bundle coverage with dev prod repos

apps/api/src/services/awoooi_priority_work_order_readback.py

@@ -3107,6 +3107,31 @@ def _enrich_from_current_readbacks(
         reboot_service_data_backup.get("gitea_repo_bundle_active_blocker_count")
         or gitea_bundle_backup.get("active_blocker_count")
     )
+    state["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int(
+        reboot_service_data_backup.get(
+            "gitea_repo_bundle_dev_prod_repo_truth_expected_count"
+        )
+        or gitea_bundle_backup.get("dev_prod_repo_truth_expected_count")
+    )
+    state["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = _int(
+        reboot_service_data_backup.get(
+            "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
+        )
+        or gitea_bundle_backup.get("dev_prod_repo_truth_backup_covered_count")
+    )
+    state["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int(
+        reboot_service_data_backup.get(
+            "gitea_repo_bundle_missing_dev_prod_repo_backup_count"
+        )
+        or gitea_bundle_backup.get("missing_dev_prod_repo_backup_count")
+    )
+    state["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = bool(
+        reboot_service_data_backup.get(
+            "gitea_repo_bundle_all_dev_prod_repos_have_backup"
+        )
+        is True
+        or gitea_bundle_backup.get("all_dev_prod_repos_have_bundle_backup") is True
+    )
     state["backup_core_green_false_but_gitea_repo_bundle_ready"] = bool(
         reboot_service_data_backup.get(
             "backup_core_green_false_but_gitea_repo_bundle_ready"
@@ -3352,6 +3377,20 @@ def _enrich_from_current_readbacks(
         evidence["gitea_repo_bundle_active_blocker_count"] = _int(
             state.get("gitea_repo_bundle_active_blocker_count")
         )
+        evidence["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
+        )
+        evidence[
+            "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
+        ] = _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
+        )
+        evidence["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int(
+            state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
+        )
+        evidence["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
+            state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
+        )
         evidence["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
             state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
         )
@@ -3965,6 +4004,18 @@ def _set_rollups_and_summary(
         "gitea_repo_bundle_active_blocker_count": _int(
             state.get("gitea_repo_bundle_active_blocker_count")
         ),
+        "gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
+        ),
+        "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
+        ),
+        "gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int(
+            state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
+        ),
+        "gitea_repo_bundle_all_dev_prod_repos_have_backup": (
+            state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
+        ),
         "backup_core_green_false_but_gitea_repo_bundle_ready": (
             state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
         ),
@@ -4073,6 +4124,18 @@ def _set_rollups_and_summary(
         "gitea_repo_bundle_active_blocker_count": _int(
             state.get("gitea_repo_bundle_active_blocker_count")
         ),
+        "gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
+        ),
+        "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int(
+            state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
+        ),
+        "gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int(
+            state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
+        ),
+        "gitea_repo_bundle_all_dev_prod_repos_have_backup": (
+            state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
+        ),
         "backup_core_green_false_but_gitea_repo_bundle_ready": (
             state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
         ),

apps/api/src/services/gitea_repo_bundle_backup_readback.py

@@ -9,17 +9,21 @@ from __future__ import annotations
 
 import json
 from datetime import datetime
+from pathlib import Path
 from typing import Any
 from urllib.parse import urlencode
 from urllib.request import Request, urlopen
 from zoneinfo import ZoneInfo
 
 from src.core.config import settings
+from src.services.snapshot_paths import default_operations_dir
 
 SCHEMA_VERSION = "gitea_repo_bundle_backup_readback_v1"
 TZ_TAIPEI = ZoneInfo("Asia/Taipei")
 DEFAULT_HOST = "188"
 PROMETHEUS_TIMEOUT_SECONDS = 4
+_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
+_DEV_PROD_REPO_READBACK_FILE = "gitea-all-product-dev-prod-repo-readback.snapshot.json"
 
 
 def build_gitea_repo_bundle_backup_readback(
@@ -28,6 +32,9 @@ def build_gitea_repo_bundle_backup_readback(
     generated_at: str | None = None,
     host: str = DEFAULT_HOST,
     metrics_error: str = "",
+    dev_prod_repo_refs: list[str] | None = None,
+    dev_prod_repo_truth_ref: str = "",
+    dev_prod_repo_truth_error: str = "",
 ) -> dict[str, Any]:
     """Build the readback payload from Prometheus vector samples."""
     now = generated_at or datetime.now(TZ_TAIPEI).isoformat(timespec="seconds")
@@ -39,6 +46,12 @@ def build_gitea_repo_bundle_backup_readback(
     metrics_present = bool(filtered) and not metrics_error
     scalar = _scalar_index(filtered)
     repo_rows = _repo_rows(filtered)
+    dev_prod_coverage = _dev_prod_repo_truth_coverage(
+        repo_rows=repo_rows,
+        expected_repo_refs=dev_prod_repo_refs,
+        source_ref=dev_prod_repo_truth_ref,
+        source_error=dev_prod_repo_truth_error,
+    )
 
     expected_repo_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_count", 0))
     missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_missing_count", 0))
@@ -72,6 +85,13 @@ def build_gitea_repo_bundle_backup_readback(
         blockers.append("gitea_bundle_sample_restore_dry_run_missing")
     elif metrics_present and not sample_restore_ok:
         blockers.append("gitea_bundle_sample_restore_dry_run_failed")
+    if dev_prod_coverage["source_error"]:
+        blockers.append("gitea_bundle_dev_prod_repo_truth_unavailable")
+    elif (
+        dev_prod_coverage["evaluated"]
+        and not dev_prod_coverage["all_dev_prod_repos_have_bundle_backup"]
+    ):
+        blockers.append("gitea_bundle_dev_prod_repo_truth_missing")
 
     ready = not blockers
     if ready:
@@ -86,6 +106,12 @@ def build_gitea_repo_bundle_backup_readback(
     elif "gitea_bundle_sample_restore_dry_run_missing" in blockers:
         status_value = "blocked_gitea_bundle_restore_dry_run_missing"
         safe_next_step = "run_gitea_bundle_sample_restore_dry_run_no_secret_no_prod_restore"
+    elif "gitea_bundle_dev_prod_repo_truth_missing" in blockers:
+        status_value = "blocked_gitea_bundle_dev_prod_repo_truth_gap"
+        safe_next_step = "run_gitea_bundle_backup_until_all_dev_prod_repos_are_covered"
+    elif "gitea_bundle_dev_prod_repo_truth_unavailable" in blockers:
+        status_value = "blocked_gitea_bundle_dev_prod_repo_truth_unavailable"
+        safe_next_step = "restore_committed_dev_prod_repo_truth_snapshot_then_rerun_readback"
     else:
         status_value = "blocked_gitea_bundle_integrity_gap"
         safe_next_step = "inspect_awoooi_gitea_bundle_metrics_before_any_restore_or_repo_change"
@@ -120,8 +146,21 @@ def build_gitea_repo_bundle_backup_readback(
             "all_expected_ok": all_expected_ok,
             "sample_restore_dry_run_present": sample_restore_present,
             "sample_restore_dry_run_ok": sample_restore_ok,
+            "dev_prod_repo_truth_expected_count": dev_prod_coverage[
+                "expected_repo_count"
+            ],
+            "dev_prod_repo_truth_backup_covered_count": dev_prod_coverage[
+                "backup_covered_repo_count"
+            ],
+            "missing_dev_prod_repo_backup_count": dev_prod_coverage[
+                "missing_backup_repo_count"
+            ],
+            "all_dev_prod_repos_have_bundle_backup": dev_prod_coverage[
+                "all_dev_prod_repos_have_bundle_backup"
+            ],
         },
         "repos": repo_rows,
+        "dev_prod_repo_truth_coverage": dev_prod_coverage,
         "metrics_error": metrics_error,
         "restore_dry_run": {
             "metric_name": "awoooi_gitea_bundle_sample_restore_dry_run_ok",
@@ -140,6 +179,7 @@ def load_latest_gitea_repo_bundle_backup_readback(
     *,
     host: str = DEFAULT_HOST,
     prometheus_url: str | None = None,
+    operations_dir: Path | None = None,
 ) -> dict[str, Any]:
     """Load live Prometheus readback, failing closed when metrics are unavailable."""
     url = (prometheus_url or settings.PROMETHEUS_URL).rstrip("/")
@@ -161,6 +201,16 @@ def load_latest_gitea_repo_bundle_backup_readback(
     ]
     samples: list[dict[str, Any]] = []
     errors: list[str] = []
+    dev_prod_refs: list[str] = []
+    dev_prod_ref = f"docs/operations/{_DEV_PROD_REPO_READBACK_FILE}"
+    dev_prod_error = ""
+    try:
+        dev_prod_refs = _load_dev_prod_repo_refs(
+            (operations_dir or _DEFAULT_OPERATIONS_DIR)
+            / _DEV_PROD_REPO_READBACK_FILE
+        )
+    except Exception as exc:  # noqa: BLE001 - fail closed with redacted error class
+        dev_prod_error = exc.__class__.__name__
     for query in queries:
         try:
             samples.extend(_query_prometheus_vector(url, query))
@@ -170,6 +220,9 @@ def load_latest_gitea_repo_bundle_backup_readback(
         metric_samples=samples,
         host=host,
         metrics_error=",".join(errors),
+        dev_prod_repo_refs=dev_prod_refs,
+        dev_prod_repo_truth_ref=dev_prod_ref,
+        dev_prod_repo_truth_error=dev_prod_error,
     )
 
 
@@ -247,3 +300,58 @@ def _repo_rows(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
         elif name == "awoooi_gitea_bundle_checksum_digest_present":
             row["checksum_digest_present"] = value == 1
     return [repos[key] for key in sorted(repos)]
+
+
+def _load_dev_prod_repo_refs(path: Path) -> list[str]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    products = payload.get("products") if isinstance(payload, dict) else None
+    if not isinstance(products, list):
+        raise ValueError("dev_prod_products_not_list")
+    refs = [
+        str(product.get("gitea_repo") or "")
+        for product in products
+        if isinstance(product, dict) and product.get("gitea_repo")
+    ]
+    if not refs:
+        raise ValueError("dev_prod_repo_refs_empty")
+    return sorted(set(refs))
+
+
+def _dev_prod_repo_truth_coverage(
+    *,
+    repo_rows: list[dict[str, Any]],
+    expected_repo_refs: list[str] | None,
+    source_ref: str,
+    source_error: str,
+) -> dict[str, Any]:
+    expected = sorted(set(expected_repo_refs or []))
+    covered = sorted(
+        str(row.get("repo"))
+        for row in repo_rows
+        if row.get("repo")
+        and row.get("present") is True
+        and row.get("ok") is True
+        and row.get("checksum_digest_present") is True
+    )
+    covered_set = set(covered)
+    expected_set = set(expected)
+    missing = sorted(expected_set - covered_set)
+    unexpected = sorted(covered_set - expected_set) if expected else []
+    evaluated = bool(expected) or bool(source_error)
+    return {
+        "schema_version": "gitea_bundle_dev_prod_repo_truth_coverage_v1",
+        "source_ref": source_ref,
+        "source_error": source_error,
+        "evaluated": evaluated,
+        "expected_repo_count": len(expected),
+        "backup_covered_repo_count": len(expected_set & covered_set),
+        "missing_backup_repo_count": len(missing),
+        "unexpected_backup_repo_count": len(unexpected),
+        "all_dev_prod_repos_have_bundle_backup": evaluated
+        and not source_error
+        and bool(expected)
+        and not missing,
+        "expected_repos": expected,
+        "missing_backup_repos": missing,
+        "unexpected_backup_repos": unexpected,
+    }

apps/api/src/services/reboot_auto_recovery_slo_scorecard.py

@@ -1368,6 +1368,18 @@ def apply_gitea_repo_bundle_backup_readback(
         "sample_restore_dry_run_ok": (
             summary.get("sample_restore_dry_run_ok") is True
         ),
+        "dev_prod_repo_truth_expected_count": _int(
+            summary.get("dev_prod_repo_truth_expected_count")
+        ),
+        "dev_prod_repo_truth_backup_covered_count": _int(
+            summary.get("dev_prod_repo_truth_backup_covered_count")
+        ),
+        "missing_dev_prod_repo_backup_count": _int(
+            summary.get("missing_dev_prod_repo_backup_count")
+        ),
+        "all_dev_prod_repos_have_bundle_backup": (
+            summary.get("all_dev_prod_repos_have_bundle_backup") is True
+        ),
         "production_restore_performed": (
             _dict(readback.get("restore_dry_run")).get("production_restore_performed")
             is True
@@ -1391,6 +1403,18 @@ def apply_gitea_repo_bundle_backup_readback(
     ]
     service_backup["gitea_repo_bundle_active_blockers"] = active_blockers
     service_backup["gitea_repo_bundle_active_blocker_count"] = len(active_blockers)
+    service_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = (
+        bundle_summary["dev_prod_repo_truth_expected_count"]
+    )
+    service_backup["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = (
+        bundle_summary["dev_prod_repo_truth_backup_covered_count"]
+    )
+    service_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = (
+        bundle_summary["missing_dev_prod_repo_backup_count"]
+    )
+    service_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
+        bundle_summary["all_dev_prod_repos_have_bundle_backup"]
+    )
     service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
         service_backup.get("backup_core_green") is False and ready
     )
@@ -1415,6 +1439,18 @@ def apply_gitea_repo_bundle_backup_readback(
     ]
     rollups["gitea_repo_bundle_repo_row_count"] = bundle_summary["repo_row_count"]
     rollups["gitea_repo_bundle_active_blocker_count"] = len(active_blockers)
+    rollups["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = (
+        bundle_summary["dev_prod_repo_truth_expected_count"]
+    )
+    rollups["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = (
+        bundle_summary["dev_prod_repo_truth_backup_covered_count"]
+    )
+    rollups["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = (
+        bundle_summary["missing_dev_prod_repo_backup_count"]
+    )
+    rollups["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
+        bundle_summary["all_dev_prod_repos_have_bundle_backup"]
+    )
     rollups["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
         service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"]
     )