fix(gitea): expose bundle backup restore readback

apps/api/src/services/awoooi_priority_work_order_readback.py

@@ -668,9 +668,17 @@ _COMMANDER_INSERTED_REQUIREMENT_WORK_ITEMS: list[dict[str, Any]] = [
         "lane": "gitea_backup_restore",
         "request": "Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？",
         "normalized_work_item": "Gitea repository identity、backup proof、restore drill 需比對 SSH heads、repo path、bundle backup、restore sample。",
-        "current_state": "已有 9 expected repos OK / backup health missing=0；仍需 restore drill 證明。",
+        "current_state": (
+            "已有 Gitea bundle readback API / sample restore dry-run verifier；"
+            "live 188 metrics 顯示 9 expected repos present/ok、missing=0、"
+            "failed=0、checksum_missing=0；當前 blocker 是 bundle freshness "
+            "stale 與 restore dry-run metric missing，不是 repo missing。"
+        ),
         "acceptance": "Gitea repo bundle backup readback 與 sample restore dry-run verifier 可讀回；禁止刪 repo / 改 visibility。",
-        "next_action": "補 Gitea repo bundle backup readback 與 sample restore dry-run verifier。",
+        "next_action": (
+            "補跑 bundle backup / 188 sync / sample restore dry-run textfile metric，"
+            "再讀 production gitea-repo-bundle-backup-readback。"
+        ),
         "mapped_workplan_id": "P0-003",
     },
     {

apps/api/src/services/gitea_repo_bundle_backup_readback.py

@@ -0,0 +1,249 @@
+"""Read-only Gitea repository bundle backup readback.
+
+This service turns backup textfile metrics into an operator-facing evidence
+payload. It never executes a backup, restore, offsite sync, credential read, or
+repository mutation.
+"""
+
+from __future__ import annotations
+
+import json
+from datetime import datetime
+from typing import Any
+from urllib.parse import urlencode
+from urllib.request import Request, urlopen
+from zoneinfo import ZoneInfo
+
+from src.core.config import settings
+
+SCHEMA_VERSION = "gitea_repo_bundle_backup_readback_v1"
+TZ_TAIPEI = ZoneInfo("Asia/Taipei")
+DEFAULT_HOST = "188"
+PROMETHEUS_TIMEOUT_SECONDS = 4
+
+
+def build_gitea_repo_bundle_backup_readback(
+    *,
+    metric_samples: list[dict[str, Any]],
+    generated_at: str | None = None,
+    host: str = DEFAULT_HOST,
+    metrics_error: str = "",
+) -> dict[str, Any]:
+    """Build the readback payload from Prometheus vector samples."""
+    now = generated_at or datetime.now(TZ_TAIPEI).isoformat(timespec="seconds")
+    filtered = [
+        sample
+        for sample in metric_samples
+        if str(sample.get("labels", {}).get("host", host)) == host
+    ]
+    metrics_present = bool(filtered) and not metrics_error
+    scalar = _scalar_index(filtered)
+    repo_rows = _repo_rows(filtered)
+
+    expected_repo_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_count", 0))
+    missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_missing_count", 0))
+    failed_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_failed_repo_count", 0))
+    checksum_missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_checksum_missing_count", 0))
+    age_seconds = int(_first_scalar(scalar, "awoooi_gitea_bundle_age_seconds", 0))
+    fresh = _first_scalar(scalar, "awoooi_gitea_bundle_fresh", 0) == 1
+    all_expected_ok = _first_scalar(scalar, "awoooi_gitea_bundle_all_expected_ok", 0) == 1
+    sample_restore_value = _first_scalar(
+        scalar,
+        "awoooi_gitea_bundle_sample_restore_dry_run_ok",
+        None,
+    )
+    sample_restore_present = sample_restore_value is not None
+    sample_restore_ok = sample_restore_value == 1
+
+    blockers: list[str] = []
+    if not metrics_present:
+        blockers.append("gitea_bundle_metrics_readback_unavailable")
+    if missing_count > 0:
+        blockers.append("gitea_bundle_expected_repo_missing")
+    if failed_count > 0:
+        blockers.append("gitea_bundle_repo_failed")
+    if checksum_missing_count > 0:
+        blockers.append("gitea_bundle_checksum_missing")
+    if metrics_present and not fresh:
+        blockers.append("gitea_bundle_backup_stale")
+    if metrics_present and not all_expected_ok:
+        blockers.append("gitea_bundle_all_expected_not_ok")
+    if metrics_present and not sample_restore_present:
+        blockers.append("gitea_bundle_sample_restore_dry_run_missing")
+    elif metrics_present and not sample_restore_ok:
+        blockers.append("gitea_bundle_sample_restore_dry_run_failed")
+
+    ready = not blockers
+    if ready:
+        status_value = "gitea_repo_bundle_backup_readback_ready"
+        safe_next_step = "keep_daily_bundle_backup_and_restore_dry_run_monitoring"
+    elif "gitea_bundle_metrics_readback_unavailable" in blockers:
+        status_value = "blocked_gitea_bundle_metrics_readback_unavailable"
+        safe_next_step = "restore_188_backup_metrics_readback_then_rerun_gitea_bundle_readback"
+    elif "gitea_bundle_backup_stale" in blockers:
+        status_value = "blocked_gitea_bundle_backup_stale"
+        safe_next_step = "run_gitea_repo_bundle_backup_and_sample_restore_dry_run_no_secret_no_prod_restore"
+    elif "gitea_bundle_sample_restore_dry_run_missing" in blockers:
+        status_value = "blocked_gitea_bundle_restore_dry_run_missing"
+        safe_next_step = "run_gitea_bundle_sample_restore_dry_run_no_secret_no_prod_restore"
+    else:
+        status_value = "blocked_gitea_bundle_integrity_gap"
+        safe_next_step = "inspect_awoooi_gitea_bundle_metrics_before_any_restore_or_repo_change"
+
+    return {
+        "schema_version": SCHEMA_VERSION,
+        "generated_at": now,
+        "status": status_value,
+        "host": host,
+        "readback_present": metrics_present,
+        "ready": ready,
+        "active_blockers": blockers,
+        "active_blocker_count": len(blockers),
+        "safe_next_step": safe_next_step,
+        "operation_boundaries": {
+            "read_only_api_allowed": True,
+            "backup_execution_allowed": False,
+            "restore_to_production_allowed": False,
+            "offsite_sync_execution_allowed": False,
+            "credential_read_allowed": False,
+            "repo_visibility_change_allowed": False,
+            "repo_delete_allowed": False,
+        },
+        "summary": {
+            "expected_repo_count": expected_repo_count,
+            "repo_row_count": len(repo_rows),
+            "expected_repo_missing_count": missing_count,
+            "failed_repo_count": failed_count,
+            "checksum_missing_count": checksum_missing_count,
+            "bundle_age_seconds": age_seconds,
+            "bundle_fresh": fresh,
+            "all_expected_ok": all_expected_ok,
+            "sample_restore_dry_run_present": sample_restore_present,
+            "sample_restore_dry_run_ok": sample_restore_ok,
+        },
+        "repos": repo_rows,
+        "metrics_error": metrics_error,
+        "restore_dry_run": {
+            "metric_name": "awoooi_gitea_bundle_sample_restore_dry_run_ok",
+            "present": sample_restore_present,
+            "ok": sample_restore_ok,
+            "verifier_command": (
+                "scripts/backup/gitea-bundle-sample-restore-dry-run.sh "
+                "--repo wooo/awoooi --write-textfile"
+            ),
+            "production_restore_performed": False,
+        },
+    }
+
+
+def load_latest_gitea_repo_bundle_backup_readback(
+    *,
+    host: str = DEFAULT_HOST,
+    prometheus_url: str | None = None,
+) -> dict[str, Any]:
+    """Load live Prometheus readback, failing closed when metrics are unavailable."""
+    url = (prometheus_url or settings.PROMETHEUS_URL).rstrip("/")
+    queries = [
+        f"awoooi_gitea_bundle_root_exists{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_manifest_present{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_age_seconds{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_fresh{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_expected_repo_count{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_expected_repo_missing_count{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_failed_repo_count{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_checksum_missing_count{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_all_expected_ok{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_repo_present{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_repo_ok{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_repo_head_count{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_checksum_digest_present{{host=\"{host}\"}}",
+        f"awoooi_gitea_bundle_sample_restore_dry_run_ok{{host=\"{host}\"}}",
+    ]
+    samples: list[dict[str, Any]] = []
+    errors: list[str] = []
+    for query in queries:
+        try:
+            samples.extend(_query_prometheus_vector(url, query))
+        except Exception as exc:  # noqa: BLE001 - fail closed with redacted error class
+            errors.append(f"{query.split('{', 1)[0]}:{exc.__class__.__name__}")
+    return build_gitea_repo_bundle_backup_readback(
+        metric_samples=samples,
+        host=host,
+        metrics_error=",".join(errors),
+    )
+
+
+def _query_prometheus_vector(prometheus_url: str, query: str) -> list[dict[str, Any]]:
+    params = urlencode({"query": query})
+    request = Request(
+        f"{prometheus_url}/api/v1/query?{params}",
+        headers={"User-Agent": "awoooi-gitea-bundle-readback"},
+    )
+    with urlopen(request, timeout=PROMETHEUS_TIMEOUT_SECONDS) as response:  # noqa: S310
+        payload = json.loads(response.read().decode("utf-8", errors="replace"))
+    if payload.get("status") != "success":
+        raise ValueError("prometheus_status_not_success")
+    rows = payload.get("data", {}).get("result") or []
+    results: list[dict[str, Any]] = []
+    for row in rows:
+        metric = row.get("metric") or {}
+        value = row.get("value") or []
+        try:
+            numeric = float(value[1])
+        except (IndexError, TypeError, ValueError):
+            continue
+        labels = {str(key): str(val) for key, val in metric.items() if key != "__name__"}
+        results.append(
+            {
+                "name": str(metric.get("__name__") or query.split("{", 1)[0]),
+                "labels": labels,
+                "value": numeric,
+            }
+        )
+    return results
+
+
+def _scalar_index(samples: list[dict[str, Any]]) -> dict[str, list[float]]:
+    indexed: dict[str, list[float]] = {}
+    for sample in samples:
+        indexed.setdefault(str(sample.get("name")), []).append(float(sample.get("value") or 0))
+    return indexed
+
+
+def _first_scalar(
+    indexed: dict[str, list[float]],
+    name: str,
+    default: float | None,
+) -> float | None:
+    values = indexed.get(name) or []
+    return values[0] if values else default
+
+
+def _repo_rows(samples: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    repos: dict[str, dict[str, Any]] = {}
+    for sample in samples:
+        labels = sample.get("labels") or {}
+        repo = str(labels.get("repo") or "")
+        if not repo:
+            continue
+        row = repos.setdefault(
+            repo,
+            {
+                "repo": repo,
+                "present": False,
+                "ok": False,
+                "head_count": 0,
+                "checksum_digest_present": False,
+            },
+        )
+        name = str(sample.get("name"))
+        value = float(sample.get("value") or 0)
+        if name == "awoooi_gitea_bundle_repo_present":
+            row["present"] = value == 1
+        elif name == "awoooi_gitea_bundle_repo_ok":
+            row["ok"] = value == 1
+        elif name == "awoooi_gitea_bundle_repo_head_count":
+            row["head_count"] = int(value)
+        elif name == "awoooi_gitea_bundle_checksum_digest_present":
+            row["checksum_digest_present"] = value == 1
+    return [repos[key] for key in sorted(repos)]