fix(backup): cross-check bundle coverage with dev prod repos
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Failing after 31s
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 59s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Failing after 31s
This commit is contained in:
@@ -3107,6 +3107,31 @@ def _enrich_from_current_readbacks(
|
||||
reboot_service_data_backup.get("gitea_repo_bundle_active_blocker_count")
|
||||
or gitea_bundle_backup.get("active_blocker_count")
|
||||
)
|
||||
state["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int(
|
||||
reboot_service_data_backup.get(
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_expected_count"
|
||||
)
|
||||
or gitea_bundle_backup.get("dev_prod_repo_truth_expected_count")
|
||||
)
|
||||
state["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = _int(
|
||||
reboot_service_data_backup.get(
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
|
||||
)
|
||||
or gitea_bundle_backup.get("dev_prod_repo_truth_backup_covered_count")
|
||||
)
|
||||
state["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int(
|
||||
reboot_service_data_backup.get(
|
||||
"gitea_repo_bundle_missing_dev_prod_repo_backup_count"
|
||||
)
|
||||
or gitea_bundle_backup.get("missing_dev_prod_repo_backup_count")
|
||||
)
|
||||
state["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = bool(
|
||||
reboot_service_data_backup.get(
|
||||
"gitea_repo_bundle_all_dev_prod_repos_have_backup"
|
||||
)
|
||||
is True
|
||||
or gitea_bundle_backup.get("all_dev_prod_repos_have_bundle_backup") is True
|
||||
)
|
||||
state["backup_core_green_false_but_gitea_repo_bundle_ready"] = bool(
|
||||
reboot_service_data_backup.get(
|
||||
"backup_core_green_false_but_gitea_repo_bundle_ready"
|
||||
@@ -3352,6 +3377,20 @@ def _enrich_from_current_readbacks(
|
||||
evidence["gitea_repo_bundle_active_blocker_count"] = _int(
|
||||
state.get("gitea_repo_bundle_active_blocker_count")
|
||||
)
|
||||
evidence["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
|
||||
)
|
||||
evidence[
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
|
||||
] = _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
|
||||
)
|
||||
evidence["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = _int(
|
||||
state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
|
||||
)
|
||||
evidence["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
|
||||
state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
|
||||
)
|
||||
evidence["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
|
||||
state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
|
||||
)
|
||||
@@ -3965,6 +4004,18 @@ def _set_rollups_and_summary(
|
||||
"gitea_repo_bundle_active_blocker_count": _int(
|
||||
state.get("gitea_repo_bundle_active_blocker_count")
|
||||
),
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
|
||||
),
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
|
||||
),
|
||||
"gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int(
|
||||
state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
|
||||
),
|
||||
"gitea_repo_bundle_all_dev_prod_repos_have_backup": (
|
||||
state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
|
||||
),
|
||||
"backup_core_green_false_but_gitea_repo_bundle_ready": (
|
||||
state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
|
||||
),
|
||||
@@ -4073,6 +4124,18 @@ def _set_rollups_and_summary(
|
||||
"gitea_repo_bundle_active_blocker_count": _int(
|
||||
state.get("gitea_repo_bundle_active_blocker_count")
|
||||
),
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_expected_count": _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_expected_count")
|
||||
),
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count": _int(
|
||||
state.get("gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count")
|
||||
),
|
||||
"gitea_repo_bundle_missing_dev_prod_repo_backup_count": _int(
|
||||
state.get("gitea_repo_bundle_missing_dev_prod_repo_backup_count")
|
||||
),
|
||||
"gitea_repo_bundle_all_dev_prod_repos_have_backup": (
|
||||
state.get("gitea_repo_bundle_all_dev_prod_repos_have_backup") is True
|
||||
),
|
||||
"backup_core_green_false_but_gitea_repo_bundle_ready": (
|
||||
state.get("backup_core_green_false_but_gitea_repo_bundle_ready") is True
|
||||
),
|
||||
|
||||
@@ -9,17 +9,21 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.parse import urlencode
|
||||
from urllib.request import Request, urlopen
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from src.core.config import settings
|
||||
from src.services.snapshot_paths import default_operations_dir
|
||||
|
||||
SCHEMA_VERSION = "gitea_repo_bundle_backup_readback_v1"
|
||||
TZ_TAIPEI = ZoneInfo("Asia/Taipei")
|
||||
DEFAULT_HOST = "188"
|
||||
PROMETHEUS_TIMEOUT_SECONDS = 4
|
||||
_DEFAULT_OPERATIONS_DIR = default_operations_dir(Path(__file__))
|
||||
_DEV_PROD_REPO_READBACK_FILE = "gitea-all-product-dev-prod-repo-readback.snapshot.json"
|
||||
|
||||
|
||||
def build_gitea_repo_bundle_backup_readback(
|
||||
@@ -28,6 +32,9 @@ def build_gitea_repo_bundle_backup_readback(
|
||||
generated_at: str | None = None,
|
||||
host: str = DEFAULT_HOST,
|
||||
metrics_error: str = "",
|
||||
dev_prod_repo_refs: list[str] | None = None,
|
||||
dev_prod_repo_truth_ref: str = "",
|
||||
dev_prod_repo_truth_error: str = "",
|
||||
) -> dict[str, Any]:
|
||||
"""Build the readback payload from Prometheus vector samples."""
|
||||
now = generated_at or datetime.now(TZ_TAIPEI).isoformat(timespec="seconds")
|
||||
@@ -39,6 +46,12 @@ def build_gitea_repo_bundle_backup_readback(
|
||||
metrics_present = bool(filtered) and not metrics_error
|
||||
scalar = _scalar_index(filtered)
|
||||
repo_rows = _repo_rows(filtered)
|
||||
dev_prod_coverage = _dev_prod_repo_truth_coverage(
|
||||
repo_rows=repo_rows,
|
||||
expected_repo_refs=dev_prod_repo_refs,
|
||||
source_ref=dev_prod_repo_truth_ref,
|
||||
source_error=dev_prod_repo_truth_error,
|
||||
)
|
||||
|
||||
expected_repo_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_count", 0))
|
||||
missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_missing_count", 0))
|
||||
@@ -72,6 +85,13 @@ def build_gitea_repo_bundle_backup_readback(
|
||||
blockers.append("gitea_bundle_sample_restore_dry_run_missing")
|
||||
elif metrics_present and not sample_restore_ok:
|
||||
blockers.append("gitea_bundle_sample_restore_dry_run_failed")
|
||||
if dev_prod_coverage["source_error"]:
|
||||
blockers.append("gitea_bundle_dev_prod_repo_truth_unavailable")
|
||||
elif (
|
||||
dev_prod_coverage["evaluated"]
|
||||
and not dev_prod_coverage["all_dev_prod_repos_have_bundle_backup"]
|
||||
):
|
||||
blockers.append("gitea_bundle_dev_prod_repo_truth_missing")
|
||||
|
||||
ready = not blockers
|
||||
if ready:
|
||||
@@ -86,6 +106,12 @@ def build_gitea_repo_bundle_backup_readback(
|
||||
elif "gitea_bundle_sample_restore_dry_run_missing" in blockers:
|
||||
status_value = "blocked_gitea_bundle_restore_dry_run_missing"
|
||||
safe_next_step = "run_gitea_bundle_sample_restore_dry_run_no_secret_no_prod_restore"
|
||||
elif "gitea_bundle_dev_prod_repo_truth_missing" in blockers:
|
||||
status_value = "blocked_gitea_bundle_dev_prod_repo_truth_gap"
|
||||
safe_next_step = "run_gitea_bundle_backup_until_all_dev_prod_repos_are_covered"
|
||||
elif "gitea_bundle_dev_prod_repo_truth_unavailable" in blockers:
|
||||
status_value = "blocked_gitea_bundle_dev_prod_repo_truth_unavailable"
|
||||
safe_next_step = "restore_committed_dev_prod_repo_truth_snapshot_then_rerun_readback"
|
||||
else:
|
||||
status_value = "blocked_gitea_bundle_integrity_gap"
|
||||
safe_next_step = "inspect_awoooi_gitea_bundle_metrics_before_any_restore_or_repo_change"
|
||||
@@ -120,8 +146,21 @@ def build_gitea_repo_bundle_backup_readback(
|
||||
"all_expected_ok": all_expected_ok,
|
||||
"sample_restore_dry_run_present": sample_restore_present,
|
||||
"sample_restore_dry_run_ok": sample_restore_ok,
|
||||
"dev_prod_repo_truth_expected_count": dev_prod_coverage[
|
||||
"expected_repo_count"
|
||||
],
|
||||
"dev_prod_repo_truth_backup_covered_count": dev_prod_coverage[
|
||||
"backup_covered_repo_count"
|
||||
],
|
||||
"missing_dev_prod_repo_backup_count": dev_prod_coverage[
|
||||
"missing_backup_repo_count"
|
||||
],
|
||||
"all_dev_prod_repos_have_bundle_backup": dev_prod_coverage[
|
||||
"all_dev_prod_repos_have_bundle_backup"
|
||||
],
|
||||
},
|
||||
"repos": repo_rows,
|
||||
"dev_prod_repo_truth_coverage": dev_prod_coverage,
|
||||
"metrics_error": metrics_error,
|
||||
"restore_dry_run": {
|
||||
"metric_name": "awoooi_gitea_bundle_sample_restore_dry_run_ok",
|
||||
@@ -140,6 +179,7 @@ def load_latest_gitea_repo_bundle_backup_readback(
|
||||
*,
|
||||
host: str = DEFAULT_HOST,
|
||||
prometheus_url: str | None = None,
|
||||
operations_dir: Path | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Load live Prometheus readback, failing closed when metrics are unavailable."""
|
||||
url = (prometheus_url or settings.PROMETHEUS_URL).rstrip("/")
|
||||
@@ -161,6 +201,16 @@ def load_latest_gitea_repo_bundle_backup_readback(
|
||||
]
|
||||
samples: list[dict[str, Any]] = []
|
||||
errors: list[str] = []
|
||||
dev_prod_refs: list[str] = []
|
||||
dev_prod_ref = f"docs/operations/{_DEV_PROD_REPO_READBACK_FILE}"
|
||||
dev_prod_error = ""
|
||||
try:
|
||||
dev_prod_refs = _load_dev_prod_repo_refs(
|
||||
(operations_dir or _DEFAULT_OPERATIONS_DIR)
|
||||
/ _DEV_PROD_REPO_READBACK_FILE
|
||||
)
|
||||
except Exception as exc: # noqa: BLE001 - fail closed with redacted error class
|
||||
dev_prod_error = exc.__class__.__name__
|
||||
for query in queries:
|
||||
try:
|
||||
samples.extend(_query_prometheus_vector(url, query))
|
||||
@@ -170,6 +220,9 @@ def load_latest_gitea_repo_bundle_backup_readback(
|
||||
metric_samples=samples,
|
||||
host=host,
|
||||
metrics_error=",".join(errors),
|
||||
dev_prod_repo_refs=dev_prod_refs,
|
||||
dev_prod_repo_truth_ref=dev_prod_ref,
|
||||
dev_prod_repo_truth_error=dev_prod_error,
|
||||
)
|
||||
|
||||
|
||||
@@ -247,3 +300,58 @@ def _repo_rows(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
elif name == "awoooi_gitea_bundle_checksum_digest_present":
|
||||
row["checksum_digest_present"] = value == 1
|
||||
return [repos[key] for key in sorted(repos)]
|
||||
|
||||
|
||||
def _load_dev_prod_repo_refs(path: Path) -> list[str]:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
products = payload.get("products") if isinstance(payload, dict) else None
|
||||
if not isinstance(products, list):
|
||||
raise ValueError("dev_prod_products_not_list")
|
||||
refs = [
|
||||
str(product.get("gitea_repo") or "")
|
||||
for product in products
|
||||
if isinstance(product, dict) and product.get("gitea_repo")
|
||||
]
|
||||
if not refs:
|
||||
raise ValueError("dev_prod_repo_refs_empty")
|
||||
return sorted(set(refs))
|
||||
|
||||
|
||||
def _dev_prod_repo_truth_coverage(
|
||||
*,
|
||||
repo_rows: list[dict[str, Any]],
|
||||
expected_repo_refs: list[str] | None,
|
||||
source_ref: str,
|
||||
source_error: str,
|
||||
) -> dict[str, Any]:
|
||||
expected = sorted(set(expected_repo_refs or []))
|
||||
covered = sorted(
|
||||
str(row.get("repo"))
|
||||
for row in repo_rows
|
||||
if row.get("repo")
|
||||
and row.get("present") is True
|
||||
and row.get("ok") is True
|
||||
and row.get("checksum_digest_present") is True
|
||||
)
|
||||
covered_set = set(covered)
|
||||
expected_set = set(expected)
|
||||
missing = sorted(expected_set - covered_set)
|
||||
unexpected = sorted(covered_set - expected_set) if expected else []
|
||||
evaluated = bool(expected) or bool(source_error)
|
||||
return {
|
||||
"schema_version": "gitea_bundle_dev_prod_repo_truth_coverage_v1",
|
||||
"source_ref": source_ref,
|
||||
"source_error": source_error,
|
||||
"evaluated": evaluated,
|
||||
"expected_repo_count": len(expected),
|
||||
"backup_covered_repo_count": len(expected_set & covered_set),
|
||||
"missing_backup_repo_count": len(missing),
|
||||
"unexpected_backup_repo_count": len(unexpected),
|
||||
"all_dev_prod_repos_have_bundle_backup": evaluated
|
||||
and not source_error
|
||||
and bool(expected)
|
||||
and not missing,
|
||||
"expected_repos": expected,
|
||||
"missing_backup_repos": missing,
|
||||
"unexpected_backup_repos": unexpected,
|
||||
}
|
||||
|
||||
@@ -1368,6 +1368,18 @@ def apply_gitea_repo_bundle_backup_readback(
|
||||
"sample_restore_dry_run_ok": (
|
||||
summary.get("sample_restore_dry_run_ok") is True
|
||||
),
|
||||
"dev_prod_repo_truth_expected_count": _int(
|
||||
summary.get("dev_prod_repo_truth_expected_count")
|
||||
),
|
||||
"dev_prod_repo_truth_backup_covered_count": _int(
|
||||
summary.get("dev_prod_repo_truth_backup_covered_count")
|
||||
),
|
||||
"missing_dev_prod_repo_backup_count": _int(
|
||||
summary.get("missing_dev_prod_repo_backup_count")
|
||||
),
|
||||
"all_dev_prod_repos_have_bundle_backup": (
|
||||
summary.get("all_dev_prod_repos_have_bundle_backup") is True
|
||||
),
|
||||
"production_restore_performed": (
|
||||
_dict(readback.get("restore_dry_run")).get("production_restore_performed")
|
||||
is True
|
||||
@@ -1391,6 +1403,18 @@ def apply_gitea_repo_bundle_backup_readback(
|
||||
]
|
||||
service_backup["gitea_repo_bundle_active_blockers"] = active_blockers
|
||||
service_backup["gitea_repo_bundle_active_blocker_count"] = len(active_blockers)
|
||||
service_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = (
|
||||
bundle_summary["dev_prod_repo_truth_expected_count"]
|
||||
)
|
||||
service_backup["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = (
|
||||
bundle_summary["dev_prod_repo_truth_backup_covered_count"]
|
||||
)
|
||||
service_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = (
|
||||
bundle_summary["missing_dev_prod_repo_backup_count"]
|
||||
)
|
||||
service_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
|
||||
bundle_summary["all_dev_prod_repos_have_bundle_backup"]
|
||||
)
|
||||
service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
|
||||
service_backup.get("backup_core_green") is False and ready
|
||||
)
|
||||
@@ -1415,6 +1439,18 @@ def apply_gitea_repo_bundle_backup_readback(
|
||||
]
|
||||
rollups["gitea_repo_bundle_repo_row_count"] = bundle_summary["repo_row_count"]
|
||||
rollups["gitea_repo_bundle_active_blocker_count"] = len(active_blockers)
|
||||
rollups["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] = (
|
||||
bundle_summary["dev_prod_repo_truth_expected_count"]
|
||||
)
|
||||
rollups["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"] = (
|
||||
bundle_summary["dev_prod_repo_truth_backup_covered_count"]
|
||||
)
|
||||
rollups["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] = (
|
||||
bundle_summary["missing_dev_prod_repo_backup_count"]
|
||||
)
|
||||
rollups["gitea_repo_bundle_all_dev_prod_repos_have_backup"] = (
|
||||
bundle_summary["all_dev_prod_repos_have_bundle_backup"]
|
||||
)
|
||||
rollups["backup_core_green_false_but_gitea_repo_bundle_ready"] = (
|
||||
service_backup["backup_core_green_false_but_gitea_repo_bundle_ready"]
|
||||
)
|
||||
|
||||
@@ -68,6 +68,10 @@ def _gitea_bundle_ready() -> dict:
|
||||
"all_expected_ok": True,
|
||||
"sample_restore_dry_run_present": True,
|
||||
"sample_restore_dry_run_ok": True,
|
||||
"dev_prod_repo_truth_expected_count": 12,
|
||||
"dev_prod_repo_truth_backup_covered_count": 12,
|
||||
"missing_dev_prod_repo_backup_count": 0,
|
||||
"all_dev_prod_repos_have_bundle_backup": True,
|
||||
},
|
||||
"restore_dry_run": {"production_restore_performed": False},
|
||||
}
|
||||
@@ -151,12 +155,20 @@ def test_awoooi_priority_work_order_readback_surfaces_gitea_bundle_truth():
|
||||
assert state["gitea_repo_bundle_expected_repo_count"] == 12
|
||||
assert state["gitea_repo_bundle_repo_row_count"] == 12
|
||||
assert state["gitea_repo_bundle_active_blocker_count"] == 0
|
||||
assert state["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] == 12
|
||||
assert (
|
||||
state["gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"]
|
||||
== 12
|
||||
)
|
||||
assert state["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] == 0
|
||||
assert state["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True
|
||||
assert state["backup_core_green_false_but_gitea_repo_bundle_ready"] is True
|
||||
assert payload["summary"]["gitea_repo_bundle_backup_ready"] is True
|
||||
assert payload["rollups"]["gitea_repo_bundle_backup_ready"] is True
|
||||
evidence = payload["in_progress_or_blocked_in_priority_order"][0]["evidence"]
|
||||
assert evidence["gitea_repo_bundle_backup_ready"] is True
|
||||
assert evidence["gitea_repo_bundle_expected_repo_count"] == 12
|
||||
assert evidence["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True
|
||||
assert (
|
||||
payload["mainline_execution_state"][
|
||||
"windows99_verify_collection_can_collect_no_secret"
|
||||
|
||||
@@ -39,6 +39,47 @@ def _fresh_repo_samples() -> list[dict]:
|
||||
]
|
||||
|
||||
|
||||
def _fresh_all_product_repo_samples() -> list[dict]:
|
||||
repos = [
|
||||
"wooo/awoooi",
|
||||
"wooo/ewoooc",
|
||||
"wooo/2026FIFAWorldCup",
|
||||
"wooo/agent-bounty-protocol",
|
||||
"wooo/AwoooGo",
|
||||
"wooo/stockplatform-v2",
|
||||
"wooo/vibework",
|
||||
"wooo/momo-pro-system",
|
||||
"wooo/tsenyang-website",
|
||||
"wooo/clawbot-v5",
|
||||
"wooo/bitan-pharmacy",
|
||||
"wooo/vtuber",
|
||||
]
|
||||
samples = [
|
||||
_sample("awoooi_gitea_bundle_expected_repo_count", len(repos)),
|
||||
_sample("awoooi_gitea_bundle_expected_repo_missing_count", 0),
|
||||
_sample("awoooi_gitea_bundle_failed_repo_count", 0),
|
||||
_sample("awoooi_gitea_bundle_checksum_missing_count", 0),
|
||||
_sample("awoooi_gitea_bundle_age_seconds", 600),
|
||||
_sample("awoooi_gitea_bundle_fresh", 1),
|
||||
_sample("awoooi_gitea_bundle_all_expected_ok", 1),
|
||||
_sample("awoooi_gitea_bundle_sample_restore_dry_run_ok", 1),
|
||||
]
|
||||
for repo in repos:
|
||||
samples.extend(
|
||||
[
|
||||
_sample("awoooi_gitea_bundle_repo_present", 1, repo=repo),
|
||||
_sample("awoooi_gitea_bundle_repo_ok", 1, repo=repo),
|
||||
_sample("awoooi_gitea_bundle_repo_head_count", 2, repo=repo),
|
||||
_sample(
|
||||
"awoooi_gitea_bundle_checksum_digest_present",
|
||||
1,
|
||||
repo=repo,
|
||||
),
|
||||
]
|
||||
)
|
||||
return samples
|
||||
|
||||
|
||||
def test_gitea_repo_bundle_readback_separates_stale_from_missing_repos() -> None:
|
||||
samples = _fresh_repo_samples()
|
||||
for sample in samples:
|
||||
@@ -73,6 +114,62 @@ def test_gitea_repo_bundle_readback_ready_requires_restore_dry_run_metric() -> N
|
||||
assert payload["restore_dry_run"]["production_restore_performed"] is False
|
||||
|
||||
|
||||
def test_gitea_repo_bundle_readback_cross_checks_dev_prod_repo_truth() -> None:
|
||||
samples = _fresh_all_product_repo_samples()
|
||||
expected_repos = sorted(
|
||||
{
|
||||
sample["labels"]["repo"]
|
||||
for sample in samples
|
||||
if sample["name"] == "awoooi_gitea_bundle_repo_present"
|
||||
}
|
||||
)
|
||||
|
||||
payload = build_gitea_repo_bundle_backup_readback(
|
||||
metric_samples=samples,
|
||||
dev_prod_repo_refs=expected_repos,
|
||||
dev_prod_repo_truth_ref=(
|
||||
"docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json"
|
||||
),
|
||||
)
|
||||
|
||||
coverage = payload["dev_prod_repo_truth_coverage"]
|
||||
assert payload["status"] == "gitea_repo_bundle_backup_readback_ready"
|
||||
assert payload["summary"]["dev_prod_repo_truth_expected_count"] == 12
|
||||
assert payload["summary"]["dev_prod_repo_truth_backup_covered_count"] == 12
|
||||
assert payload["summary"]["missing_dev_prod_repo_backup_count"] == 0
|
||||
assert payload["summary"]["all_dev_prod_repos_have_bundle_backup"] is True
|
||||
assert coverage["all_dev_prod_repos_have_bundle_backup"] is True
|
||||
assert coverage["missing_backup_repos"] == []
|
||||
assert coverage["unexpected_backup_repos"] == []
|
||||
|
||||
|
||||
def test_gitea_repo_bundle_readback_blocks_when_dev_prod_repo_bundle_missing() -> None:
|
||||
samples = _fresh_all_product_repo_samples()
|
||||
samples = [
|
||||
sample
|
||||
for sample in samples
|
||||
if sample.get("labels", {}).get("repo") != "wooo/tsenyang-website"
|
||||
]
|
||||
expected_repos = [
|
||||
"wooo/awoooi",
|
||||
"wooo/tsenyang-website",
|
||||
]
|
||||
|
||||
payload = build_gitea_repo_bundle_backup_readback(
|
||||
metric_samples=samples,
|
||||
dev_prod_repo_refs=expected_repos,
|
||||
dev_prod_repo_truth_ref=(
|
||||
"docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json"
|
||||
),
|
||||
)
|
||||
|
||||
coverage = payload["dev_prod_repo_truth_coverage"]
|
||||
assert payload["status"] == "blocked_gitea_bundle_dev_prod_repo_truth_gap"
|
||||
assert "gitea_bundle_dev_prod_repo_truth_missing" in payload["active_blockers"]
|
||||
assert payload["summary"]["missing_dev_prod_repo_backup_count"] == 1
|
||||
assert coverage["missing_backup_repos"] == ["wooo/tsenyang-website"]
|
||||
|
||||
|
||||
def test_gitea_repo_bundle_readback_endpoint_returns_payload(monkeypatch) -> None:
|
||||
def fake_loader() -> dict:
|
||||
return build_gitea_repo_bundle_backup_readback(
|
||||
|
||||
@@ -844,6 +844,21 @@ def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatc
|
||||
service_data_backup = payload["controlled_service_data_backup_readback"]
|
||||
assert service_data_backup["gitea_repo_bundle_backup_ready"] is True
|
||||
assert service_data_backup["gitea_repo_bundle_expected_repo_count"] == 12
|
||||
assert (
|
||||
service_data_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"]
|
||||
== 12
|
||||
)
|
||||
assert (
|
||||
service_data_backup[
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
|
||||
]
|
||||
== 12
|
||||
)
|
||||
assert (
|
||||
service_data_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"]
|
||||
== 0
|
||||
)
|
||||
assert service_data_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True
|
||||
assert (
|
||||
service_data_backup["backup_core_green_false_but_gitea_repo_bundle_ready"]
|
||||
is True
|
||||
@@ -1397,6 +1412,15 @@ def test_reboot_auto_recovery_slo_scorecard_surfaces_gitea_bundle_subreadback():
|
||||
assert service_data_backup["gitea_repo_bundle_expected_repo_count"] == 12
|
||||
assert service_data_backup["gitea_repo_bundle_repo_row_count"] == 12
|
||||
assert service_data_backup["gitea_repo_bundle_active_blockers"] == []
|
||||
assert service_data_backup["gitea_repo_bundle_dev_prod_repo_truth_expected_count"] == 12
|
||||
assert (
|
||||
service_data_backup[
|
||||
"gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count"
|
||||
]
|
||||
== 12
|
||||
)
|
||||
assert service_data_backup["gitea_repo_bundle_missing_dev_prod_repo_backup_count"] == 0
|
||||
assert service_data_backup["gitea_repo_bundle_all_dev_prod_repos_have_backup"] is True
|
||||
assert (
|
||||
service_data_backup["backup_core_green_false_but_gitea_repo_bundle_ready"]
|
||||
is True
|
||||
@@ -1447,6 +1471,10 @@ def _gitea_bundle_ready() -> dict:
|
||||
"all_expected_ok": True,
|
||||
"sample_restore_dry_run_present": True,
|
||||
"sample_restore_dry_run_ok": True,
|
||||
"dev_prod_repo_truth_expected_count": 12,
|
||||
"dev_prod_repo_truth_backup_covered_count": 12,
|
||||
"missing_dev_prod_repo_backup_count": 0,
|
||||
"all_dev_prod_repos_have_bundle_backup": True,
|
||||
},
|
||||
"restore_dry_run": {"production_restore_performed": False},
|
||||
}
|
||||
|
||||
@@ -1,3 +1,13 @@
|
||||
## 2026-07-03 — 09:05 Gitea bundle backup / dev-prod repo truth 交叉驗證
|
||||
|
||||
**完成內容**:
|
||||
- `gitea-repo-bundle-backup-readback` 新增 `dev_prod_repo_truth_coverage`,會讀取 committed `docs/operations/gitea-all-product-dev-prod-repo-readback.snapshot.json`,把 12 個 dev/prod Gitea repo truth 與 188 bundle backup repo rows 做集合比對。
|
||||
- 若 dev/prod truth 內任何 repo 缺 bundle / `repo_ok` / checksum,readback 會 fail-closed 成 `blocked_gitea_bundle_dev_prod_repo_truth_gap`,不再只靠 bundle metric 的 expected count 宣稱完整。
|
||||
- `reboot-auto-recovery-slo-scorecard` 與 `awoooi-priority-work-order-readback` 同步上卷:`gitea_repo_bundle_dev_prod_repo_truth_expected_count`、`gitea_repo_bundle_dev_prod_repo_truth_backup_covered_count`、`gitea_repo_bundle_missing_dev_prod_repo_backup_count`、`gitea_repo_bundle_all_dev_prod_repos_have_backup`。
|
||||
|
||||
**仍維持**:
|
||||
- 本輪只做 readback contract / API projection;未執行 backup、restore、offsite sync;未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / gh;未建立 repo、未改 visibility、未 sync refs、未 workflow_dispatch;未重啟 host / VM / service;未 Docker / Nginx / K3s / DB / firewall restart;未 DROP / TRUNCATE / restore / prune / delete / force push。
|
||||
|
||||
## 2026-07-03 — 10:38 Gitea 全專案 dev/prod repo truth 上卷
|
||||
|
||||
**完成內容**:
|
||||
|
||||
Reference in New Issue
Block a user