From 7085a94eb6060e9da29e8c4ee2b5ac603714d927 Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 3 Jul 2026 09:01:47 +0800 Subject: [PATCH] fix(backup): cross-check bundle coverage with dev prod repos --- .../awoooi_priority_work_order_readback.py | 63 ++++++++++ .../gitea_repo_bundle_backup_readback.py | 108 ++++++++++++++++++ .../reboot_auto_recovery_slo_scorecard.py | 36 ++++++ ...awoooi_priority_work_order_readback_api.py | 12 ++ ...t_gitea_repo_bundle_backup_readback_api.py | 97 ++++++++++++++++ ..._reboot_auto_recovery_slo_scorecard_api.py | 28 +++++ docs/LOGBOOK.md | 10 ++ 7 files changed, 354 insertions(+) diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index ee0f63bba..efa680811 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -3107,6 +3107,31 @@ def _enrich_from_current_readbacks( reboot_service_data_backup.get("gitea_repo_bundle_active_blocker_count") or gitea_bundle_backup.get("active_blocker_count") ) + state["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int( + reboot_service_data_backup.get( + "gitea_repo_bundle_dev_prod_repo_truth_expected_count" + ) + or gitea_bundle_backup.get("dev_prod_repo_truth_expected_count") + ) + state["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = _int( + reboot_service_data_backup.get( + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count" + ) + or gitea_bundle_backup.get("dev_prod_repo_truth_backup_covered_count") + ) + state["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int( + reboot_service_data_backup.get( + "gitea_repo_bundle_missing_dev_prod_repo_backup_count" + ) + or gitea_bundle_backup.get("missing_dev_prod_repo_backup_count") + ) + state["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = bool( + reboot_service_data_backup.get( + "gitea_repo_bundle_all_dev_prod_repos_have_backup" + ) + is True + or gitea_bundle_backup.get("all_dev_prod_repos_have_bundle_backup") is True + ) state["backup_core_green_false_but_gitea_repo_bundle_ready"] = bool( reboot_service_data_backup.get( "backup_core_green_false_but_gitea_repo_bundle_ready" @@ -3352,6 +3377,20 @@ def _enrich_from_current_readbacks( evidence["gitea_repo_bundle_active_blocker_count"] = _int( state.get("gitea_repo_bundle_active_blocker_count") ) + evidence["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count") + ) + evidence[ + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count" + ] = _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count") + ) + evidence["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int( + state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count") + ) + evidence["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = ( + state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True + ) evidence["backup_core_green_false_but_gitea_repo_bundle_ready"] = ( state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True ) @@ -3965,6 +4004,18 @@ def _set_rollups_and_summary( "gitea_repo_bundle_active_blocker_count": _int( state.get("gitea_repo_bundle_active_blocker_count") ), + "gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count") + ), + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count") + ), + "gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int( + state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count") + ), + "gitea_repo_bundle_all_dev_prod_repos_have_backup": ( + state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True + ), "backup_core_green_false_but_gitea_repo_bundle_ready": ( state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True ), @@ -4073,6 +4124,18 @@ def _set_rollups_and_summary( "gitea_repo_bundle_active_blocker_count": _int( state.get("gitea_repo_bundle_active_blocker_count") ), + "gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count") + ), + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int( + state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count") + ), + "gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int( + state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count") + ), + "gitea_repo_bundle_all_dev_prod_repos_have_backup": ( + state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True + ), "backup_core_green_false_but_gitea_repo_bundle_ready": ( state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True ), diff --git a/apps/api/src/services/gitea_repo_bundle_backup_readback.py b/apps/api/src/services/gitea_repo_bundle_backup_readback.py index 2c8890898..7efa6a4c1 100644 --- a/apps/api/src/services/gitea_repo_bundle_backup_readback.py +++ b/apps/api/src/services/gitea_repo_bundle_backup_readback.py @@ -9,17 +9,21 @@ from __future__ import annotations import json from datetime import datetime +from pathlib import Path from typing import Any from urllib.parse import urlencode from urllib.request import Request, urlopen from zoneinfo import ZoneInfo from src.core.config import settings +from src.services.snapshot_paths import default_operations_dir SCHEMA_VERSION = "gitea_repo_bundle_backup_readback_v1" TZ_TAIPEI = ZoneInfo("Asia/Taipei") DEFAULT_HOST = "188" PROMETHEUS_TIMEOUT_SECONDS = 4 +_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__)) +_DEV_PROD_REPO_READBACK_FILE = "gitea-all-product-dev-prod-repo-readback.snapshot.json" def build_gitea_repo_bundle_backup_readback( @@ -28,6 +32,9 @@ def build_gitea_repo_bundle_backup_readback( generated_at: str | None = None, host: str = DEFAULT_HOST, metrics_error: str = "", + dev_prod_repo_refs: list[str] | None = None, + dev_prod_repo_truth_ref: str = "", + dev_prod_repo_truth_error: str = "", ) -> dict[str, Any]: """Build the readback payload from Prometheus vector samples.""" now = generated_at or datetime.now(TZ_TAIPEI).isoformat(timespec="seconds") @@ -39,6 +46,12 @@ def build_gitea_repo_bundle_backup_readback( metrics_present = bool(filtered) and not metrics_error scalar = _scalar_index(filtered) repo_rows = _repo_rows(filtered) + dev_prod_coverage = _dev_prod_repo_truth_coverage( + repo_rows=repo_rows, + expected_repo_refs=dev_prod_repo_refs, + source_ref=dev_prod_repo_truth_ref, + source_error=dev_prod_repo_truth_error, + ) expected_repo_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_count", 0)) missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_missing_count", 0)) @@ -72,6 +85,13 @@ def build_gitea_repo_bundle_backup_readback( blockers.append("gitea_bundle_sample_restore_dry_run_missing") elif metrics_present and not sample_restore_ok: blockers.append("gitea_bundle_sample_restore_dry_run_failed") + if dev_prod_coverage["source_error"]: + blockers.append("gitea_bundle_dev_prod_repo_truth_unavailable") + elif ( + dev_prod_coverage["evaluated"] + and not dev_prod_coverage["all_dev_prod_repos_have_bundle_backup"] + ): + blockers.append("gitea_bundle_dev_prod_repo_truth_missing") ready = not blockers if ready: @@ -86,6 +106,12 @@ def build_gitea_repo_bundle_backup_readback( elif "gitea_bundle_sample_restore_dry_run_missing" in blockers: status_value = "blocked_gitea_bundle_restore_dry_run_missing" safe_next_step = "run_gitea_bundle_sample_restore_dry_run_no_secret_no_prod_restore" + elif "gitea_bundle_dev_prod_repo_truth_missing" in blockers: + status_value = "blocked_gitea_bundle_dev_prod_repo_truth_gap" + safe_next_step = "run_gitea_bundle_backup_until_all_dev_prod_repos_are_covered" + elif "gitea_bundle_dev_prod_repo_truth_unavailable" in blockers: + status_value = "blocked_gitea_bundle_dev_prod_repo_truth_unavailable" + safe_next_step = "restore_committed_dev_prod_repo_truth_snapshot_then_rerun_readback" else: status_value = "blocked_gitea_bundle_integrity_gap" safe_next_step = "inspect_awoooi_gitea_bundle_metrics_before_any_restore_or_repo_change" @@ -120,8 +146,21 @@ def build_gitea_repo_bundle_backup_readback( "all_expected_ok": all_expected_ok, "sample_restore_dry_run_present": sample_restore_present, "sample_restore_dry_run_ok": sample_restore_ok, + "dev_prod_repo_truth_expected_count": dev_prod_coverage[ + "expected_repo_count" + ], + "dev_prod_repo_truth_backup_covered_count": dev_prod_coverage[ + "backup_covered_repo_count" + ], + "missing_dev_prod_repo_backup_count": dev_prod_coverage[ + "missing_backup_repo_count" + ], + "all_dev_prod_repos_have_bundle_backup": dev_prod_coverage[ + "all_dev_prod_repos_have_bundle_backup" + ], }, "repos": repo_rows, + "dev_prod_repo_truth_coverage": dev_prod_coverage, "metrics_error": metrics_error, "restore_dry_run": { "metric_name": "awoooi_gitea_bundle_sample_restore_dry_run_ok", @@ -140,6 +179,7 @@ def load_latest_gitea_repo_bundle_backup_readback( *, host: str = DEFAULT_HOST, prometheus_url: str | None = None, + operations_dir: Path | None = None, ) -> dict[str, Any]: """Load live Prometheus readback, failing closed when metrics are unavailable.""" url = (prometheus_url or settings.PROMETHEUS_URL).rstrip("/") @@ -161,6 +201,16 @@ def load_latest_gitea_repo_bundle_backup_readback( ] samples: list[dict[str, Any]] = [] errors: list[str] = [] + dev_prod_refs: list[str] = [] + dev_prod_ref = f"docs/operations/{_DEV_PROD_REPO_READBACK_FILE}" + dev_prod_error = "" + try: + dev_prod_refs = _load_dev_prod_repo_refs( + (operations_dir or _DEFAULT_OPERATIONS_DIR) + / _DEV_PROD_REPO_READBACK_FILE + ) + except Exception as exc: # noqa: BLE001 - fail closed with redacted error class + dev_prod_error = exc.__class__.__name__ for query in queries: try: samples.extend(_query_prometheus_vector(url, query)) @@ -170,6 +220,9 @@ def load_latest_gitea_repo_bundle_backup_readback( metric_samples=samples, host=host, metrics_error=",".join(errors), + dev_prod_repo_refs=dev_prod_refs, + dev_prod_repo_truth_ref=dev_prod_ref, + dev_prod_repo_truth_error=dev_prod_error, ) @@ -247,3 +300,58 @@ def _repo_rows(samples: list[dict[str, Any]]) -> list[dict[str, Any]]: elif name == "awoooi_gitea_bundle_checksum_digest_present": row["checksum_digest_present"] = value == 1 return [repos[key] for key in sorted(repos)] + + +def _load_dev_prod_repo_refs(path: Path) -> list[str]: + payload = json.loads(path.read_text(encoding="utf-8")) + products = payload.get("products") if isinstance(payload, dict) else None + if not isinstance(products, list): + raise ValueError("dev_prod_products_not_list") + refs = [ + str(product.get("gitea_repo") or "") + for product in products + if isinstance(product, dict) and product.get("gitea_repo") + ] + if not refs: + raise ValueError("dev_prod_repo_refs_empty") + return sorted(set(refs)) + + +def _dev_prod_repo_truth_coverage( + *, + repo_rows: list[dict[str, Any]], + expected_repo_refs: list[str] | None, + source_ref: str, + source_error: str, +) -> dict[str, Any]: + expected = sorted(set(expected_repo_refs or [])) + covered = sorted( + str(row.get("repo")) + for row in repo_rows + if row.get("repo") + and row.get("present") is True + and row.get("ok") is True + and row.get("checksum_digest_present") is True + ) + covered_set = set(covered) + expected_set = set(expected) + missing = sorted(expected_set - covered_set) + unexpected = sorted(covered_set - expected_set) if expected else [] + evaluated = bool(expected) or bool(source_error) + return { + "schema_version": "gitea_bundle_dev_prod_repo_truth_coverage_v1", + "source_ref": source_ref, + "source_error": source_error, + "evaluated": evaluated, + "expected_repo_count": len(expected), + "backup_covered_repo_count": len(expected_set & covered_set), + "missing_backup_repo_count": len(missing), + "unexpected_backup_repo_count": len(unexpected), + "all_dev_prod_repos_have_bundle_backup": evaluated + and not source_error + and bool(expected) + and not missing, + "expected_repos": expected, + "missing_backup_repos": missing, + "unexpected_backup_repos": unexpected, + } diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index c146cf2f2..0a663cf3a 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -1368,6 +1368,18 @@ def apply_gitea_repo_bundle_backup_readback( "sample_restore_dry_run_ok": ( summary.get("sample_restore_dry_run_ok") is True ), + "dev_prod_repo_truth_expected_count": _int( + summary.get("dev_prod_repo_truth_expected_count") + ), + "dev_prod_repo_truth_backup_covered_count": _int( + summary.get("dev_prod_repo_truth_backup_covered_count") + ), + "missing_dev_prod_repo_backup_count": _int( + summary.get("missing_dev_prod_repo_backup_count") + ), + "all_dev_prod_repos_have_bundle_backup": ( + summary.get("all_dev_prod_repos_have_bundle_backup") is True + ), "production_restore_performed": ( _dict(readback.get("restore_dry_run")).get("production_restore_performed") is True @@ -1391,6 +1403,18 @@ def apply_gitea_repo_bundle_backup_readback( ] service_backup["gitea_repo_bundle_active_blockers"] = active_blockers service_backup["gitea_repo_bundle_active_blocker_count"] = len(active_blockers) + service_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = ( + bundle_summary["dev_prod_repo_truth_expected_count"] + ) + service_backup["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = ( + bundle_summary["dev_prod_repo_truth_backup_covered_count"] + ) + service_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = ( + bundle_summary["missing_dev_prod_repo_backup_count"] + ) + service_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = ( + bundle_summary["all_dev_prod_repos_have_bundle_backup"] + ) service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] = ( service_backup.get("backup_core_green") is False and ready ) @@ -1415,6 +1439,18 @@ def apply_gitea_repo_bundle_backup_readback( ] rollups["gitea_repo_bundle_repo_row_count"] = bundle_summary["repo_row_count"] rollups["gitea_repo_bundle_active_blocker_count"] = len(active_blockers) + rollups["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = ( + bundle_summary["dev_prod_repo_truth_expected_count"] + ) + rollups["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = ( + bundle_summary["dev_prod_repo_truth_backup_covered_count"] + ) + rollups["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = ( + bundle_summary["missing_dev_prod_repo_backup_count"] + ) + rollups["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = ( + bundle_summary["all_dev_prod_repos_have_bundle_backup"] + ) rollups["backup_core_green_false_but_gitea_repo_bundle_ready"] = ( service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] ) diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 8864bc42d..8a2f24099 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -68,6 +68,10 @@ def _gitea_bundle_ready() -> dict: "all_expected_ok": True, "sample_restore_dry_run_present": True, "sample_restore_dry_run_ok": True, + "dev_prod_repo_truth_expected_count": 12, + "dev_prod_repo_truth_backup_covered_count": 12, + "missing_dev_prod_repo_backup_count": 0, + "all_dev_prod_repos_have_bundle_backup": True, }, "restore_dry_run": {"production_restore_performed": False}, } @@ -151,12 +155,20 @@ def test_awoooi_priority_work_order_readback_surfaces_gitea_bundle_truth(): assert state["gitea_repo_bundle_expected_repo_count"] == 12 assert state["gitea_repo_bundle_repo_row_count"] == 12 assert state["gitea_repo_bundle_active_blocker_count"] == 0 + assert state["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] == 12 + assert ( + state["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] + == 12 + ) + assert state["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] == 0 + assert state["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True assert state["backup_core_green_false_but_gitea_repo_bundle_ready"] is True assert payload["summary"]["gitea_repo_bundle_backup_ready"] is True assert payload["rollups"]["gitea_repo_bundle_backup_ready"] is True evidence = payload["in_progress_or_blocked_in_priority_order"][0]["evidence"] assert evidence["gitea_repo_bundle_backup_ready"] is True assert evidence["gitea_repo_bundle_expected_repo_count"] == 12 + assert evidence["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True assert ( payload["mainline_execution_state"][ "windows99_verify_collection_can_collect_no_secret" diff --git a/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py b/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py index c42f63895..f2147eddd 100644 --- a/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py +++ b/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py @@ -39,6 +39,47 @@ def _fresh_repo_samples() -> list[dict]: ] +def _fresh_all_product_repo_samples() -> list[dict]: + repos = [ + "wooo/awoooi", + "wooo/ewoooc", + "wooo/2026FIFAWorldCup", + "wooo/agent-bounty-protocol", + "wooo/AwoooGo", + "wooo/stockplatform-v2", + "wooo/vibework", + "wooo/momo-pro-system", + "wooo/tsenyang-website", + "wooo/clawbot-v5", + "wooo/bitan-pharmacy", + "wooo/vtuber", + ] + samples = [ + _sample("awoooi_gitea_bundle_expected_repo_count", len(repos)), + _sample("awoooi_gitea_bundle_expected_repo_missing_count", 0), + _sample("awoooi_gitea_bundle_failed_repo_count", 0), + _sample("awoooi_gitea_bundle_checksum_missing_count", 0), + _sample("awoooi_gitea_bundle_age_seconds", 600), + _sample("awoooi_gitea_bundle_fresh", 1), + _sample("awoooi_gitea_bundle_all_expected_ok", 1), + _sample("awoooi_gitea_bundle_sample_restore_dry_run_ok", 1), + ] + for repo in repos: + samples.extend( + [ + _sample("awoooi_gitea_bundle_repo_present", 1, repo=repo), + _sample("awoooi_gitea_bundle_repo_ok", 1, repo=repo), + _sample("awoooi_gitea_bundle_repo_head_count", 2, repo=repo), + _sample( + "awoooi_gitea_bundle_checksum_digest_present", + 1, + repo=repo, + ), + ] + ) + return samples + + def test_gitea_repo_bundle_readback_separates_stale_from_missing_repos() -> None: samples = _fresh_repo_samples() for sample in samples: @@ -73,6 +114,62 @@ def test_gitea_repo_bundle_readback_ready_requires_restore_dry_run_metric() -> N assert payload["restore_dry_run"]["production_restore_performed"] is False +def test_gitea_repo_bundle_readback_cross_checks_dev_prod_repo_truth() -> None: + samples = _fresh_all_product_repo_samples() + expected_repos = sorted( + { + sample["labels"]["repo"] + for sample in samples + if sample["name"] == "awoooi_gitea_bundle_repo_present" + } + ) + + payload = build_gitea_repo_bundle_backup_readback( + metric_samples=samples, + dev_prod_repo_refs=expected_repos, + dev_prod_repo_truth_ref=( + "docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json" + ), + ) + + coverage = payload["dev_prod_repo_truth_coverage"] + assert payload["status"] == "gitea_repo_bundle_backup_readback_ready" + assert payload["summary"]["dev_prod_repo_truth_expected_count"] == 12 + assert payload["summary"]["dev_prod_repo_truth_backup_covered_count"] == 12 + assert payload["summary"]["missing_dev_prod_repo_backup_count"] == 0 + assert payload["summary"]["all_dev_prod_repos_have_bundle_backup"] is True + assert coverage["all_dev_prod_repos_have_bundle_backup"] is True + assert coverage["missing_backup_repos"] == [] + assert coverage["unexpected_backup_repos"] == [] + + +def test_gitea_repo_bundle_readback_blocks_when_dev_prod_repo_bundle_missing() -> None: + samples = _fresh_all_product_repo_samples() + samples = [ + sample + for sample in samples + if sample.get("labels", {}).get("repo") != "wooo/tsenyang-website" + ] + expected_repos = [ + "wooo/awoooi", + "wooo/tsenyang-website", + ] + + payload = build_gitea_repo_bundle_backup_readback( + metric_samples=samples, + dev_prod_repo_refs=expected_repos, + dev_prod_repo_truth_ref=( + "docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json" + ), + ) + + coverage = payload["dev_prod_repo_truth_coverage"] + assert payload["status"] == "blocked_gitea_bundle_dev_prod_repo_truth_gap" + assert "gitea_bundle_dev_prod_repo_truth_missing" in payload["active_blockers"] + assert payload["summary"]["missing_dev_prod_repo_backup_count"] == 1 + assert coverage["missing_backup_repos"] == ["wooo/tsenyang-website"] + + def test_gitea_repo_bundle_readback_endpoint_returns_payload(monkeypatch) -> None: def fake_loader() -> dict: return build_gitea_repo_bundle_backup_readback( diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 974d8266e..fe01cb291 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -844,6 +844,21 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatc service_data_backup = payload["controlled_service_data_backup_readback"] assert service_data_backup["gitea_repo_bundle_backup_ready"] is True assert service_data_backup["gitea_repo_bundle_expected_repo_count"] == 12 + assert ( + service_data_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] + == 12 + ) + assert ( + service_data_backup[ + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count" + ] + == 12 + ) + assert ( + service_data_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] + == 0 + ) + assert service_data_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True assert ( service_data_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] is True @@ -1397,6 +1412,15 @@ def test_reboot_auto_recovery_slo_scorecard_surfaces_gitea_bundle_subreadback(): assert service_data_backup["gitea_repo_bundle_expected_repo_count"] == 12 assert service_data_backup["gitea_repo_bundle_repo_row_count"] == 12 assert service_data_backup["gitea_repo_bundle_active_blockers"] == [] + assert service_data_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] == 12 + assert ( + service_data_backup[ + "gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count" + ] + == 12 + ) + assert service_data_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] == 0 + assert service_data_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True assert ( service_data_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] is True @@ -1447,6 +1471,10 @@ def _gitea_bundle_ready() -> dict: "all_expected_ok": True, "sample_restore_dry_run_present": True, "sample_restore_dry_run_ok": True, + "dev_prod_repo_truth_expected_count": 12, + "dev_prod_repo_truth_backup_covered_count": 12, + "missing_dev_prod_repo_backup_count": 0, + "all_dev_prod_repos_have_bundle_backup": True, }, "restore_dry_run": {"production_restore_performed": False}, } diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index eee0a3280..dda1805e8 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,13 @@ +## 2026-07-03 — 09:05 Gitea bundle backup / dev-prod repo truth 交叉驗證 + +**完成內容**： +- `gitea-repo-bundle-backup-readback` 新增 `dev_prod_repo_truth_coverage`，會讀取 committed `docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json`，把 12 個 dev/prod Gitea repo truth 與 188 bundle backup repo rows 做集合比對。 +- 若 dev/prod truth 內任何 repo 缺 bundle / `repo_ok` / checksum，readback 會 fail-closed 成 `blocked_gitea_bundle_dev_prod_repo_truth_gap`，不再只靠 bundle metric 的 expected count 宣稱完整。 +- `reboot-auto-recovery-slo-scorecard` 與 `awoooi-priority-work-order-readback` 同步上卷：`gitea_repo_bundle_dev_prod_repo_truth_expected_count`、`gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count`、`gitea_repo_bundle_missing_dev_prod_repo_backup_count`、`gitea_repo_bundle_all_dev_prod_repos_have_backup`。 + +**仍維持**： +- 本輪只做 readback contract / API projection；未執行 backup、restore、offsite sync；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未建立 repo、未改 visibility、未 sync refs、未 workflow_dispatch；未重啟 host / VM / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 10:38 Gitea 全專案 dev/prod repo truth 上卷 **完成內容**：