From 2f3db555e4bb8d1952d1e6f51b20a7c55a4595cb Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 23:13:13 +0800 Subject: [PATCH] fix(gitea): expose bundle backup restore readback --- .gitea/workflows/cd.yaml | 12 + apps/api/src/api/v1/agents.py | 26 ++ .../awoooi_priority_work_order_readback.py | 12 +- .../gitea_repo_bundle_backup_readback.py | 249 ++++++++++++++++++ ...t_gitea_repo_bundle_backup_readback_api.py | 98 +++++++ docs/LOGBOOK.md | 24 ++ docs/runbooks/REBOOT-RECOVERY-SOP.md | 19 ++ ...oot-cold-start-backup-recovery-workplan.md | 2 +- ...r-inserted-requirements-priority-ledger.md | 2 +- ops/monitoring/alerts-unified.yml | 17 ++ .../test_cd_controlled_runtime_profile.py | 8 + .../gitea-bundle-sample-restore-dry-run.sh | 149 +++++++++++ ...est_gitea_bundle_sample_restore_dry_run.py | 58 ++++ .../test_reboot_p0_operational_contract.py | 7 + 14 files changed, 679 insertions(+), 4 deletions(-) create mode 100644 apps/api/src/services/gitea_repo_bundle_backup_readback.py create mode 100644 apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py create mode 100755 scripts/backup/gitea-bundle-sample-restore-dry-run.sh create mode 100644 scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 881d55372..278268091 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -464,6 +464,8 @@ jobs: ;; apps/api/src/services/backup_dr_readiness_matrix.py) ;; + apps/api/src/services/gitea_repo_bundle_backup_readback.py) + ;; apps/api/src/services/channel_hub.py) ;; apps/api/src/services/decision_manager.py) @@ -528,6 +530,8 @@ jobs: ;; apps/api/tests/test_backup_dr_readiness_matrix_api.py) ;; + apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py) + ;; apps/api/tests/test_credential_escrow_evidence_intake_readiness_api.py) ;; apps/api/tests/test_gitea_private_inventory_p0_scorecard_api.py) @@ -626,8 +630,12 @@ jobs: ;; scripts/backup/gitea-repo-bundle-backup.sh) ;; + scripts/backup/gitea-bundle-sample-restore-dry-run.sh) + ;; scripts/backup/tests/test_backup_status_contract.py) ;; + scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py) + ;; scripts/ops/backup-alert-label-contract-check.py) ;; scripts/ops/backup-from-110.sh) @@ -875,6 +883,7 @@ jobs: src/services/auto_approve.py \ src/services/backup_dr_target_inventory.py \ src/services/backup_dr_readiness_matrix.py \ + src/services/gitea_repo_bundle_backup_readback.py \ src/services/decision_fusion.py \ src/services/delivery_closure_workbench.py \ src/services/heartbeat_report_service.py \ @@ -963,6 +972,7 @@ jobs: ../../scripts/backup/backup-awoooi-frequent.sh \ ../../scripts/backup/backup-status.sh \ ../../scripts/backup/gitea-repo-bundle-backup.sh \ + ../../scripts/backup/gitea-bundle-sample-restore-dry-run.sh \ ../../scripts/ops/backup-from-110.sh \ ../../scripts/ops/deploy-alertmanager-config.sh \ ../../scripts/ops/docker-health-monitor.sh \ @@ -997,6 +1007,7 @@ jobs: tests/test_backup_dr_target_inventory_api.py \ tests/test_backup_dr_readiness_matrix.py \ tests/test_backup_dr_readiness_matrix_api.py \ + tests/test_gitea_repo_bundle_backup_readback_api.py \ tests/test_credential_escrow_evidence_intake_readiness_api.py \ tests/test_gitea_private_inventory_p0_scorecard_api.py \ tests/test_gitea_workflow_runner_owner_attestation_request_api.py \ @@ -1023,6 +1034,7 @@ jobs: ../../ops/runner/test_check_awoooi_110_controlled_cd_lane_readiness.py \ ../../ops/runner/test_verify_awoooi_non110_cd_closure.py \ ../../scripts/backup/tests/test_backup_status_contract.py \ + ../../scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py \ ../../scripts/ops/tests/test_backup_health_textfile_exporter.py \ ../../scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py \ ../../scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py \ diff --git a/apps/api/src/api/v1/agents.py b/apps/api/src/api/v1/agents.py index 7f810ea05..8def315e5 100644 --- a/apps/api/src/api/v1/agents.py +++ b/apps/api/src/api/v1/agents.py @@ -398,6 +398,9 @@ from src.services.gitea_private_inventory_closeout_validation import ( from src.services.gitea_private_inventory_p0_scorecard import ( load_latest_gitea_private_inventory_p0_scorecard, ) +from src.services.gitea_repo_bundle_backup_readback import ( + load_latest_gitea_repo_bundle_backup_readback, +) from src.services.gitea_workflow_runner_health import ( load_latest_gitea_workflow_runner_health, ) @@ -4698,6 +4701,29 @@ async def get_backup_dr_readiness_matrix() -> dict[str, Any]: ) from exc +@router.get( + "/gitea-repo-bundle-backup-readback", + response_model=dict[str, Any], + summary="取得 Gitea repository bundle backup readback", + description=( + "讀取 Prometheus / node-exporter 上的 Gitea bundle backup 指標，" + "分層回傳 repo present、repo ok、checksum、freshness 與 sample restore dry-run。" + "此端點只做 read-only metrics readback，不執行 backup、restore、offsite sync、" + "不讀 credential、不改 repo visibility、不刪 repo、不寫 Gitea。" + ), +) +async def get_gitea_repo_bundle_backup_readback() -> dict[str, Any]: + """Return live read-only Gitea repository bundle backup evidence.""" + try: + return await asyncio.to_thread(load_latest_gitea_repo_bundle_backup_readback) + except (json.JSONDecodeError, ValueError) as exc: + logger.error("gitea_repo_bundle_backup_readback_invalid", error=str(exc)) + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Gitea repository bundle backup readback 無效", + ) from exc + + @router.get( "/credential-escrow-evidence-intake-readiness", response_model=dict[str, Any], diff --git a/apps/api/src/services/awoooi_priority_work_order_readback.py b/apps/api/src/services/awoooi_priority_work_order_readback.py index 4606326e2..8695ef765 100644 --- a/apps/api/src/services/awoooi_priority_work_order_readback.py +++ b/apps/api/src/services/awoooi_priority_work_order_readback.py @@ -668,9 +668,17 @@ _COMMANDER_INSERTED_REQUIREMENT_WORK_ITEMS: list[dict[str, Any]] = [ "lane": "gitea_backup_restore", "request": "Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？", "normalized_work_item": "Gitea repository identity、backup proof、restore drill 需比對 SSH heads、repo path、bundle backup、restore sample。", - "current_state": "已有 9 expected repos OK / backup health missing=0；仍需 restore drill 證明。", + "current_state": ( + "已有 Gitea bundle readback API / sample restore dry-run verifier；" + "live 188 metrics 顯示 9 expected repos present/ok、missing=0、" + "failed=0、checksum_missing=0；當前 blocker 是 bundle freshness " + "stale 與 restore dry-run metric missing，不是 repo missing。" + ), "acceptance": "Gitea repo bundle backup readback 與 sample restore dry-run verifier 可讀回；禁止刪 repo / 改 visibility。", - "next_action": "補 Gitea repo bundle backup readback 與 sample restore dry-run verifier。", + "next_action": ( + "補跑 bundle backup / 188 sync / sample restore dry-run textfile metric，" + "再讀 production gitea-repo-bundle-backup-readback。" + ), "mapped_workplan_id": "P0-003", }, { diff --git a/apps/api/src/services/gitea_repo_bundle_backup_readback.py b/apps/api/src/services/gitea_repo_bundle_backup_readback.py new file mode 100644 index 000000000..2c8890898 --- /dev/null +++ b/apps/api/src/services/gitea_repo_bundle_backup_readback.py @@ -0,0 +1,249 @@ +"""Read-only Gitea repository bundle backup readback. + +This service turns backup textfile metrics into an operator-facing evidence +payload. It never executes a backup, restore, offsite sync, credential read, or +repository mutation. +""" + +from __future__ import annotations + +import json +from datetime import datetime +from typing import Any +from urllib.parse import urlencode +from urllib.request import Request, urlopen +from zoneinfo import ZoneInfo + +from src.core.config import settings + +SCHEMA_VERSION = "gitea_repo_bundle_backup_readback_v1" +TZ_TAIPEI = ZoneInfo("Asia/Taipei") +DEFAULT_HOST = "188" +PROMETHEUS_TIMEOUT_SECONDS = 4 + + +def build_gitea_repo_bundle_backup_readback( + *, + metric_samples: list[dict[str, Any]], + generated_at: str | None = None, + host: str = DEFAULT_HOST, + metrics_error: str = "", +) -> dict[str, Any]: + """Build the readback payload from Prometheus vector samples.""" + now = generated_at or datetime.now(TZ_TAIPEI).isoformat(timespec="seconds") + filtered = [ + sample + for sample in metric_samples + if str(sample.get("labels", {}).get("host", host)) == host + ] + metrics_present = bool(filtered) and not metrics_error + scalar = _scalar_index(filtered) + repo_rows = _repo_rows(filtered) + + expected_repo_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_count", 0)) + missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_expected_repo_missing_count", 0)) + failed_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_failed_repo_count", 0)) + checksum_missing_count = int(_first_scalar(scalar, "awoooi_gitea_bundle_checksum_missing_count", 0)) + age_seconds = int(_first_scalar(scalar, "awoooi_gitea_bundle_age_seconds", 0)) + fresh = _first_scalar(scalar, "awoooi_gitea_bundle_fresh", 0) == 1 + all_expected_ok = _first_scalar(scalar, "awoooi_gitea_bundle_all_expected_ok", 0) == 1 + sample_restore_value = _first_scalar( + scalar, + "awoooi_gitea_bundle_sample_restore_dry_run_ok", + None, + ) + sample_restore_present = sample_restore_value is not None + sample_restore_ok = sample_restore_value == 1 + + blockers: list[str] = [] + if not metrics_present: + blockers.append("gitea_bundle_metrics_readback_unavailable") + if missing_count > 0: + blockers.append("gitea_bundle_expected_repo_missing") + if failed_count > 0: + blockers.append("gitea_bundle_repo_failed") + if checksum_missing_count > 0: + blockers.append("gitea_bundle_checksum_missing") + if metrics_present and not fresh: + blockers.append("gitea_bundle_backup_stale") + if metrics_present and not all_expected_ok: + blockers.append("gitea_bundle_all_expected_not_ok") + if metrics_present and not sample_restore_present: + blockers.append("gitea_bundle_sample_restore_dry_run_missing") + elif metrics_present and not sample_restore_ok: + blockers.append("gitea_bundle_sample_restore_dry_run_failed") + + ready = not blockers + if ready: + status_value = "gitea_repo_bundle_backup_readback_ready" + safe_next_step = "keep_daily_bundle_backup_and_restore_dry_run_monitoring" + elif "gitea_bundle_metrics_readback_unavailable" in blockers: + status_value = "blocked_gitea_bundle_metrics_readback_unavailable" + safe_next_step = "restore_188_backup_metrics_readback_then_rerun_gitea_bundle_readback" + elif "gitea_bundle_backup_stale" in blockers: + status_value = "blocked_gitea_bundle_backup_stale" + safe_next_step = "run_gitea_repo_bundle_backup_and_sample_restore_dry_run_no_secret_no_prod_restore" + elif "gitea_bundle_sample_restore_dry_run_missing" in blockers: + status_value = "blocked_gitea_bundle_restore_dry_run_missing" + safe_next_step = "run_gitea_bundle_sample_restore_dry_run_no_secret_no_prod_restore" + else: + status_value = "blocked_gitea_bundle_integrity_gap" + safe_next_step = "inspect_awoooi_gitea_bundle_metrics_before_any_restore_or_repo_change" + + return { + "schema_version": SCHEMA_VERSION, + "generated_at": now, + "status": status_value, + "host": host, + "readback_present": metrics_present, + "ready": ready, + "active_blockers": blockers, + "active_blocker_count": len(blockers), + "safe_next_step": safe_next_step, + "operation_boundaries": { + "read_only_api_allowed": True, + "backup_execution_allowed": False, + "restore_to_production_allowed": False, + "offsite_sync_execution_allowed": False, + "credential_read_allowed": False, + "repo_visibility_change_allowed": False, + "repo_delete_allowed": False, + }, + "summary": { + "expected_repo_count": expected_repo_count, + "repo_row_count": len(repo_rows), + "expected_repo_missing_count": missing_count, + "failed_repo_count": failed_count, + "checksum_missing_count": checksum_missing_count, + "bundle_age_seconds": age_seconds, + "bundle_fresh": fresh, + "all_expected_ok": all_expected_ok, + "sample_restore_dry_run_present": sample_restore_present, + "sample_restore_dry_run_ok": sample_restore_ok, + }, + "repos": repo_rows, + "metrics_error": metrics_error, + "restore_dry_run": { + "metric_name": "awoooi_gitea_bundle_sample_restore_dry_run_ok", + "present": sample_restore_present, + "ok": sample_restore_ok, + "verifier_command": ( + "scripts/backup/gitea-bundle-sample-restore-dry-run.sh " + "--repo wooo/awoooi --write-textfile" + ), + "production_restore_performed": False, + }, + } + + +def load_latest_gitea_repo_bundle_backup_readback( + *, + host: str = DEFAULT_HOST, + prometheus_url: str | None = None, +) -> dict[str, Any]: + """Load live Prometheus readback, failing closed when metrics are unavailable.""" + url = (prometheus_url or settings.PROMETHEUS_URL).rstrip("/") + queries = [ + f"awoooi_gitea_bundle_root_exists{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_manifest_present{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_age_seconds{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_fresh{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_expected_repo_count{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_expected_repo_missing_count{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_failed_repo_count{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_checksum_missing_count{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_all_expected_ok{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_repo_present{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_repo_ok{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_repo_head_count{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_checksum_digest_present{{host=\"{host}\"}}", + f"awoooi_gitea_bundle_sample_restore_dry_run_ok{{host=\"{host}\"}}", + ] + samples: list[dict[str, Any]] = [] + errors: list[str] = [] + for query in queries: + try: + samples.extend(_query_prometheus_vector(url, query)) + except Exception as exc: # noqa: BLE001 - fail closed with redacted error class + errors.append(f"{query.split('{', 1)[0]}:{exc.__class__.__name__}") + return build_gitea_repo_bundle_backup_readback( + metric_samples=samples, + host=host, + metrics_error=",".join(errors), + ) + + +def _query_prometheus_vector(prometheus_url: str, query: str) -> list[dict[str, Any]]: + params = urlencode({"query": query}) + request = Request( + f"{prometheus_url}/api/v1/query?{params}", + headers={"User-Agent": "awoooi-gitea-bundle-readback"}, + ) + with urlopen(request, timeout=PROMETHEUS_TIMEOUT_SECONDS) as response: # noqa: S310 + payload = json.loads(response.read().decode("utf-8", errors="replace")) + if payload.get("status") != "success": + raise ValueError("prometheus_status_not_success") + rows = payload.get("data", {}).get("result") or [] + results: list[dict[str, Any]] = [] + for row in rows: + metric = row.get("metric") or {} + value = row.get("value") or [] + try: + numeric = float(value[1]) + except (IndexError, TypeError, ValueError): + continue + labels = {str(key): str(val) for key, val in metric.items() if key != "__name__"} + results.append( + { + "name": str(metric.get("__name__") or query.split("{", 1)[0]), + "labels": labels, + "value": numeric, + } + ) + return results + + +def _scalar_index(samples: list[dict[str, Any]]) -> dict[str, list[float]]: + indexed: dict[str, list[float]] = {} + for sample in samples: + indexed.setdefault(str(sample.get("name")), []).append(float(sample.get("value") or 0)) + return indexed + + +def _first_scalar( + indexed: dict[str, list[float]], + name: str, + default: float | None, +) -> float | None: + values = indexed.get(name) or [] + return values[0] if values else default + + +def _repo_rows(samples: list[dict[str, Any]]) -> list[dict[str, Any]]: + repos: dict[str, dict[str, Any]] = {} + for sample in samples: + labels = sample.get("labels") or {} + repo = str(labels.get("repo") or "") + if not repo: + continue + row = repos.setdefault( + repo, + { + "repo": repo, + "present": False, + "ok": False, + "head_count": 0, + "checksum_digest_present": False, + }, + ) + name = str(sample.get("name")) + value = float(sample.get("value") or 0) + if name == "awoooi_gitea_bundle_repo_present": + row["present"] = value == 1 + elif name == "awoooi_gitea_bundle_repo_ok": + row["ok"] = value == 1 + elif name == "awoooi_gitea_bundle_repo_head_count": + row["head_count"] = int(value) + elif name == "awoooi_gitea_bundle_checksum_digest_present": + row["checksum_digest_present"] = value == 1 + return [repos[key] for key in sorted(repos)] diff --git a/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py b/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py new file mode 100644 index 000000000..c42f63895 --- /dev/null +++ b/apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py @@ -0,0 +1,98 @@ +from __future__ import annotations + +from fastapi import FastAPI +from fastapi.testclient import TestClient + +from src.api.v1 import agents +from src.api.v1.agents import router +from src.services.gitea_repo_bundle_backup_readback import ( + build_gitea_repo_bundle_backup_readback, +) + + +def _sample(name: str, value: float, **labels: str) -> dict: + merged = {"host": "188", **labels} + return {"name": name, "labels": merged, "value": value} + + +def _fresh_repo_samples() -> list[dict]: + return [ + _sample("awoooi_gitea_bundle_expected_repo_count", 2), + _sample("awoooi_gitea_bundle_expected_repo_missing_count", 0), + _sample("awoooi_gitea_bundle_failed_repo_count", 0), + _sample("awoooi_gitea_bundle_checksum_missing_count", 0), + _sample("awoooi_gitea_bundle_age_seconds", 600), + _sample("awoooi_gitea_bundle_fresh", 1), + _sample("awoooi_gitea_bundle_all_expected_ok", 1), + _sample("awoooi_gitea_bundle_repo_present", 1, repo="wooo/awoooi"), + _sample("awoooi_gitea_bundle_repo_ok", 1, repo="wooo/awoooi"), + _sample("awoooi_gitea_bundle_repo_head_count", 196, repo="wooo/awoooi"), + _sample("awoooi_gitea_bundle_checksum_digest_present", 1, repo="wooo/awoooi"), + _sample("awoooi_gitea_bundle_repo_present", 1, repo="wooo/tsenyang-website"), + _sample("awoooi_gitea_bundle_repo_ok", 1, repo="wooo/tsenyang-website"), + _sample("awoooi_gitea_bundle_repo_head_count", 2, repo="wooo/tsenyang-website"), + _sample( + "awoooi_gitea_bundle_checksum_digest_present", + 1, + repo="wooo/tsenyang-website", + ), + ] + + +def test_gitea_repo_bundle_readback_separates_stale_from_missing_repos() -> None: + samples = _fresh_repo_samples() + for sample in samples: + if sample["name"] == "awoooi_gitea_bundle_age_seconds": + sample["value"] = 116_157 + if sample["name"] == "awoooi_gitea_bundle_fresh": + sample["value"] = 0 + if sample["name"] == "awoooi_gitea_bundle_all_expected_ok": + sample["value"] = 0 + + payload = build_gitea_repo_bundle_backup_readback(metric_samples=samples) + + assert payload["status"] == "blocked_gitea_bundle_backup_stale" + assert payload["summary"]["expected_repo_missing_count"] == 0 + assert payload["summary"]["failed_repo_count"] == 0 + assert payload["summary"]["checksum_missing_count"] == 0 + assert "gitea_bundle_backup_stale" in payload["active_blockers"] + assert "gitea_bundle_expected_repo_missing" not in payload["active_blockers"] + assert "gitea_bundle_sample_restore_dry_run_missing" in payload["active_blockers"] + assert payload["repos"][0]["repo"] == "wooo/awoooi" + + +def test_gitea_repo_bundle_readback_ready_requires_restore_dry_run_metric() -> None: + samples = _fresh_repo_samples() + samples.append(_sample("awoooi_gitea_bundle_sample_restore_dry_run_ok", 1)) + + payload = build_gitea_repo_bundle_backup_readback(metric_samples=samples) + + assert payload["status"] == "gitea_repo_bundle_backup_readback_ready" + assert payload["ready"] is True + assert payload["active_blockers"] == [] + assert payload["restore_dry_run"]["production_restore_performed"] is False + + +def test_gitea_repo_bundle_readback_endpoint_returns_payload(monkeypatch) -> None: + def fake_loader() -> dict: + return build_gitea_repo_bundle_backup_readback( + metric_samples=_fresh_repo_samples(), + generated_at="2026-07-02T20:00:00+08:00", + ) + + monkeypatch.setattr( + agents, + "load_latest_gitea_repo_bundle_backup_readback", + fake_loader, + ) + app = FastAPI() + app.include_router(router, prefix="/api/v1") + client = TestClient(app) + + response = client.get("/api/v1/agents/gitea-repo-bundle-backup-readback") + + assert response.status_code == 200 + payload = response.json() + assert payload["schema_version"] == "gitea_repo_bundle_backup_readback_v1" + assert payload["status"] == "blocked_gitea_bundle_restore_dry_run_missing" + assert payload["operation_boundaries"]["restore_to_production_allowed"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 38e6588e6..b4bfc5873 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53424,3 +53424,27 @@ production browser smoke: **下一步**： - commit / push 到 Gitea main；等 deploy marker 後重新讀 production matrix / coverage endpoint，再跑 AwoooP Runs / Work Items / Alerts smoke。 + +## 2026-07-02 — P0 Gitea bundle backup readback / restore dry-run verifier + +**完成內容**： +- 新增 `gitea-repo-bundle-backup-readback` service / API，從 Prometheus / node-exporter 的 `awoooi_gitea_bundle_*` 指標分層回傳 repo present、repo ok、checksum、freshness 與 sample restore dry-run；endpoint 為 `/api/v1/agents/gitea-repo-bundle-backup-readback`。 +- 新增 `scripts/backup/gitea-bundle-sample-restore-dry-run.sh`，只把 bundle clone 到暫存 mirror 並執行 `git fsck --connectivity-only`，可寫入 `awoooi_gitea_bundle_sample_restore_dry_run_ok` textfile metric；不 restore 到 production、不連 Gitea、不讀 token。 +- `ops/monitoring/alerts-unified.yml` 新增 `GiteaPrivateBundleRestoreDryRunMissing`，避免「bundle 存在但沒有還原 dry-run」被誤判成備份可信。 +- `docs/runbooks/REBOOT-RECOVERY-SOP.md`、reboot workplan、commander inserted priority ledger 與 `awoooi_priority_work_order_readback` 已同步：Gitea UI/API 可見性、SSH refs、bundle backup、sample restore dry-run 必須分四層判讀。 + +**live 188 readback**： +- 目前 188 metrics 顯示 `expected_repo_count=9`、`expected_repo_missing_count=0`、`failed_repo_count=0`、`checksum_missing_count=0`、9 個 repo `repo_ok=1`。 +- 新 readback builder 對 live metrics 判定為 `blocked_gitea_bundle_backup_stale`；active blockers 是 `gitea_bundle_backup_stale`、`gitea_bundle_all_expected_not_ok`、`gitea_bundle_sample_restore_dry_run_missing`。這代表當前問題是 freshness / restore confidence，不是 repo missing。 + +**本地驗證結果**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py ops/runner/test_cd_controlled_runtime_profile.py -q -p no:cacheprovider`：`59 passed`。 +- `python3.11 -m py_compile apps/api/src/services/gitea_repo_bundle_backup_readback.py apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py`、`bash -n scripts/backup/gitea-bundle-sample-restore-dry-run.sh .gitea/workflows/cd.yaml`、YAML parse、`python3 ops/runner/guard-gitea-runner-pressure.py --root .`、`git diff --check`：通過。 + +**仍維持**： +- 沒有讀 secret / runner token / `.runner` 內容 / `.env` / raw sessions / SQLite / auth。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune，沒有 restore 到 production。 + +**下一步**： +- commit / push 到 Gitea main 並讀回 CD；deploy 後讀 production `/api/v1/agents/gitea-repo-bundle-backup-readback`。接續 P0 action 是補跑 bundle backup / 188 sync / sample restore dry-run textfile metric，讓 `bundle_fresh=true`、`all_expected_ok=true`、`sample_restore_dry_run_ok=true`。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 258f144cf..677255db8 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -380,6 +380,25 @@ curl -s -o /dev/null -w '%{http_code}' http://192.168.0.110:5000/v2/ combined local-console output 只能依 marker 判讀：`AWOOOI_110_CONTROLLED_CD_LANE_READY` 只屬於 110 controlled lane；沒有 `AWOOOI_NON110_RUNNER_READY` 時，不得從同一份 `BLOCKER` 行推導 non110 runner blocker。 +**Gitea repository / bundle backup 判讀 (2026-07-02)**: + +Gitea UI/API 可見性、SSH refs、bundle backup、sample restore dry-run 是四層不同證據，不得混成「repo 不見」或「backup 完成」一句話。 + +固定順序： + +```bash +curl -s https://awoooi.wooo.work/api/v1/agents/gitea-repo-bundle-backup-readback | python3 -m json.tool +curl -s http://192.168.0.188:9100/metrics | grep '^awoooi_gitea_bundle_' +scripts/backup/gitea-bundle-sample-restore-dry-run.sh --repo wooo/awoooi --write-textfile +``` + +判讀規則： + +- `expected_repo_missing_count=0`、`failed_repo_count=0`、`checksum_missing_count=0` 代表 bundle repo / checksum 層沒有缺，不代表 freshness 或 restore drill 已完成。 +- `awoooi_gitea_bundle_fresh=0` 是 freshness blocker；下一步是補跑 `gitea-repo-bundle-backup.sh` 與同步到 188，不得刪 repo、改 visibility、或 restore 到 production。 +- `awoooi_gitea_bundle_sample_restore_dry_run_ok` absent / `0` 是 restore confidence blocker；只允許在暫存目錄 clone bundle + `git fsck --connectivity-only`，不得把 bundle restore 回 Gitea production。 +- public repo search 只代表 public visibility；private/internal repo 要看 authenticated inventory、SSH heads 或 bundle metrics，不能用 404 直接推論 history 消失。 + **Harbor Exited 128 修復**: ```bash # 等 harbor-log healthy diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 13570bd8b..30eccf4b8 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,7 +20,7 @@ | P0-3 | BLOCKED | StockPlatform data freshness | 23:27 public route / health 為 200，但 `/api/v1/system/freshness` 與 `/api/v1/system/ingestion` 回 `status=not_configured`、`blockers=["postgres_not_ready"]`、`latest_trading_date=null`；網站可達不代表資料最新。 | 恢復 110 control path 後，read-only 查 `/home/wooo/stockplatform-v2` compose / DB schema / migration status；禁止 fake freshness、manual DB rows、restore/prune。 | | P0-4 | BLOCKED | AWOOOI production 版本最新性 | Gitea SSH `main` 最新已到 `68084470 feat(agent): expose deploy marker receipt input`，且包含 `3de828f97 fix(agent): surface harbor cd retry receipt blocker`。Public Gitea queue 23:27 讀到 latest CD `#4117 Running` for `68084470`；production 尚不能證明已部署最新 source。 | 補 deploy marker / runtime SHA / endpoint readback 一致；Harbor `/v2` 恢復前 CD 無法把最新 source 發到 production，未一致前不可宣稱 AWOOOI 最新。 | | P0-5 | BLOCKED_SOURCE_ADVANCED | 110 control path / Harbor registry `/v2` | 23:27 queue readback 回 `status=blocked_harbor_110_repair_no_matching_runner`；Harbor repair workflow `#4115` 仍 `Waiting`，no-matching label `awoooi-host`，jobs API 仍 stale/mismatched。Live route probe 同步回 public registry `/v2=502`、internal `192.168.0.110:5000/v2=502`、Harbor health `502`、SignOz `502`。Harbor receipt validator 對 live queue 回 `status=blocked_waiting_harbor_controlled_recovery_receipt`，active blockers 含 `gitea_queue_harbor_110_repair_no_matching_runner`、`gitea_queue_harbor_110_repair_jobs_stale_or_mismatched`、`public_registry_v2_verifier_not_green`、`internal_registry_v2_verifier_not_green`。23:27 cold-start 同步證明 110 registry `/v2` blocked、110 SSH read-only check blocked、K3s pull refused by `110:5000`。23:30 source 修正讓 `awoooi-startup-110.sh` 預設 guard-on 啟動 controlled drain lane；legacy runner / generic label 仍 fail-closed。 | 部署 / 讀回 110 startup source 後，確認 `awoooi-cd-lane-drain.service` 只在 guardrails 通過時 active，讓 110-local repair workflow 或 110 console/local script 真正執行 `recover-110-control-path-and-harbor-local.sh --check` / `--apply-all`，並讀回 public/internal `/v2` 為 `200/401`。恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | -| P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 起初為空。已建立 verified emergency bundle `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`：4 個 public/internal repo bundle verify + checksum 成功，`AwoooGo`、`stockplatform-v2`、`vibework` 因 private auth fail-closed。20:18 summary 因 110 `backup-status` 不可讀回，`BACKUP_CORE_GREEN=0`、`DR_ESCROW_BLOCKED=1`、`DR_ESCROW_EVIDENCE_UNKNOWN=1`。 | 188 `gitea_repo_mirror_from_110` subtree metric / alert 已補；下一步仍是恢復 110 SSH command path 後跑正式 `gitea dump`、private repo 非互動備份、repo count、backup-status 與 restore drill readback。unknown 不得當作 backup / DR green。 | +| P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 public repo，不代表 private/internal repo 消失；internal refs / bundle / restore confidence 必須分層讀回。2026-07-02 live 188 metrics 顯示 `expected_repo_count=9`、`expected_repo_missing_count=0`、`failed_repo_count=0`、`checksum_missing_count=0`、9 個 repo `repo_ok=1`；目前 blocker 不是 repo missing，而是 `awoooi_gitea_bundle_fresh=0`、`all_expected_ok=0`、缺 `awoooi_gitea_bundle_sample_restore_dry_run_ok`。已新增 `/api/v1/agents/gitea-repo-bundle-backup-readback` 與 `scripts/backup/gitea-bundle-sample-restore-dry-run.sh`，讓 repo present / checksum / freshness / restore dry-run 分開判讀。 | 補跑 Gitea bundle backup 並同步到 188 後，執行 sample restore dry-run 寫入 textfile metric；再讀 production API readback 到 `bundle_fresh=true`、`all_expected_ok=true`、`sample_restore_dry_run_ok=true`。不得刪 repo、改 visibility、讀 token、restore 到 production 或用 public search 當完整備份證據。 | | P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`；22:05 host probe 讀到 99 ping reachable 但 `boot_id=reachable_unknown_boot` / uptime unknown，111 不可達，112/120/121/188 可讀，188 startup unit failed/degraded。先前只讀 readback 顯示 99 RDP 3389 / SSH 22 可達、WinRM 5985 fail，`administrator@192.168.0.99` SSH publickey denied。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence，要求 all-host required observed/reachable 且 99 不再是 unknown uptime。 | | P0-8 | SOURCE_READY_RUNTIME_BLOCKED | 502 maintenance fallback / Telegram / backup alert | L0/L1 fallback runbook、Nginx snippet、reboot / backup alert rules 已在 source；runtime 尚需部署與外部 L1 provider readback。 | L0 以測試 vhost 驗證 `X-AWOOOI-Fallback`；L1 需外部雲端/CDN probe；Telegram 以脫敏 alert receipt 驗證。 | diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 3c4045928..05b3facc3 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -66,7 +66,7 @@ | 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | | 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `47%`、active blockers `11`；本輪再補 per-blocker Prometheus metric / Telegram alert 投影，避免只剩總 blocker_count；`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | -| 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 已有 9 expected repos OK / backup health missing=0 的 handoff；仍需 restore drill 證明 | 補 Gitea repo bundle backup readback + sample restore dry-run verifier；禁止刪 repo / 改 visibility | +| 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 2026-07-02 live 188 metrics 顯示 9 expected repos present/ok、missing=0、failed=0、checksum_missing=0；新增 `/api/v1/agents/gitea-repo-bundle-backup-readback`、`gitea-bundle-sample-restore-dry-run.sh` 與 `GiteaPrivateBundleRestoreDryRunMissing` alert。當前 blocker 是 bundle freshness stale + restore dry-run metric missing，不是 repo missing | 補跑 Gitea repo bundle backup / 188 sync / sample restore dry-run metric；禁止刪 repo / 改 visibility / 讀 token / restore 到 production | | 11 | CIR-P0-CPU-001 | P0 | 「110 / 188 CPU 負載持續過高，為什麼沒監控告警、沒主動修復」 | Sustained CPU pressure automation：Alertmanager → controller → evidence → service playbook → verifier → KM writeback | 110 已有 `Host110SustainedModeratePressure`、Gitea playbook、Stock/Postgres evidence；188 仍需同級 controller/alerts readback | 下一步接 `postgres_hot_query_or_backup_export_playbook`；並補 188 equivalent readback，不以單次下降結案 | | 12 | CIR-P0-CPU-002 | P0 | 「噪音會影響真問題，要整合一起做」 | Alert noise / real issue correlation：backup aggregate noise、CPU pressure、Gitea queue、Stock freshness 要分清主因與次因 | 部分已在 SOP 註記；仍需統一 correlation scorecard | 建立 incident correlation readback：primary_blocker、secondary_noise、ignored_noise_reason、evidence_ref | | 13 | CIR-P0-CD-001 | P0 | 「所有專案都不能推版 / 要看到實作結果」 | Gitea-only CD baseline：每次 main push 要有 visible run、deploy marker、production readback；GitHub 不作解法 | AWOOOI 最新 main 可推，CD success/deploy marker 已多次證明；全產品未全綠 | 將 product governance matrix 接入各產品 Gitea CD readiness，不再只報 AWOOOI | diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index 3c63bd337..c508f2c70 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1581,6 +1581,23 @@ groups: description: "Gitea private bundle manifest、bundle、checksum 或 freshness 有缺口；public repo search 不能代表完整備份。" runbook: "先讀 awoooi_gitea_bundle_* 指標找缺少的 repo，執行 scripts/backup/gitea-repo-bundle-backup.sh 產生 bundle 並同步到 188；不得刪 repo、不得 restore 到 production、不得讀 token。" + - alert: GiteaPrivateBundleRestoreDryRunMissing + expr: absent(awoooi_gitea_bundle_sample_restore_dry_run_ok{host="188"}) or awoooi_gitea_bundle_sample_restore_dry_run_ok{host="188"} == 0 + for: 30m + labels: + severity: critical + layer: host-backup + component: gitea-private-bundle + host: "188" + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "188 Gitea private repo bundle 還原 dry-run 未通過" + description: "Gitea bundle 備份需要 sample restore dry-run 證明可還原；bundle 存在不等於 restore confidence。" + runbook: "在 188 執行 scripts/backup/gitea-bundle-sample-restore-dry-run.sh --repo wooo/awoooi --write-textfile；只允許暫存目錄 dry-run，不得 restore 到 production，不得刪 repo，不得讀 token。" + - alert: BackupCoverageDomainStale expr: awoooi_backup_coverage_domain_fresh == 0 for: 15m diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 4c5dbe029..c07c24a87 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -157,7 +157,9 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "scripts/backup/backup-awoooi-frequent.sh)", "scripts/backup/backup-status.sh)", "scripts/backup/gitea-repo-bundle-backup.sh)", + "scripts/backup/gitea-bundle-sample-restore-dry-run.sh)", "scripts/backup/tests/test_backup_status_contract.py)", + "scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py)", "scripts/ops/backup-alert-label-contract-check.py)", "scripts/ops/backup-health-textfile-exporter.py)", "scripts/ops/tests/test_backup_health_textfile_exporter.py)", @@ -172,7 +174,9 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "scripts/ops/tests/test_alertmanager_webhook_config.py)", "../../scripts/backup/backup-awoooi-frequent.sh", "../../scripts/backup/backup-status.sh", + "../../scripts/backup/gitea-bundle-sample-restore-dry-run.sh", "../../scripts/backup/tests/test_backup_status_contract.py", + "../../scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py", "../../scripts/ops/backup-alert-label-contract-check.py", "../../scripts/ops/gitea-queue-hook-backlog-playbook.py", "../../scripts/ops/host-runaway-process-exporter.py", @@ -774,8 +778,10 @@ def test_gitea_private_inventory_scorecard_stays_on_controlled_runtime_profile() "apps/api/src/services/gitea_owner_coverage_attestation_validation.py)", "apps/api/src/services/gitea_private_inventory_closeout_validation.py)", "apps/api/src/services/gitea_private_inventory_p0_scorecard.py)", + "apps/api/src/services/gitea_repo_bundle_backup_readback.py)", "apps/api/src/services/gitea_workflow_runner_owner_attestation_request.py)", "apps/api/tests/test_gitea_private_inventory_p0_scorecard_api.py)", + "apps/api/tests/test_gitea_repo_bundle_backup_readback_api.py)", "apps/api/tests/test_gitea_workflow_runner_owner_attestation_request_api.py)", "docs/operations/awoooi-gitea-authenticated-inventory-payload-validation.snapshot.json)", "docs/security/GITEA-REPO-INVENTORY-SNAPSHOT.md)", @@ -788,8 +794,10 @@ def test_gitea_private_inventory_scorecard_stays_on_controlled_runtime_profile() "src/services/gitea_owner_coverage_attestation_validation.py", "src/services/gitea_private_inventory_closeout_validation.py", "src/services/gitea_private_inventory_p0_scorecard.py", + "src/services/gitea_repo_bundle_backup_readback.py", "src/services/gitea_workflow_runner_owner_attestation_request.py", "tests/test_gitea_private_inventory_p0_scorecard_api.py", + "tests/test_gitea_repo_bundle_backup_readback_api.py", "tests/test_gitea_workflow_runner_owner_attestation_request_api.py", "scripts/security/tests/test_gitea_authenticated_inventory_payload_validator.py)", "../../scripts/security/gitea-private-inventory-p0-scorecard.py", diff --git a/scripts/backup/gitea-bundle-sample-restore-dry-run.sh b/scripts/backup/gitea-bundle-sample-restore-dry-run.sh new file mode 100755 index 000000000..f6f7f440d --- /dev/null +++ b/scripts/backup/gitea-bundle-sample-restore-dry-run.sh @@ -0,0 +1,149 @@ +#!/usr/bin/env bash +# Verify one Gitea bundle can be restored into a temporary mirror clone. +# This is a dry-run verifier only. It does not restore into production, mutate +# Gitea, read credentials, delete repositories, or change repo visibility. + +set -euo pipefail + +ROOT="${AIOPS_GITEA_BUNDLE_ROOT:-/home/ollama/backup/110/gitea/git-bundles/latest-private-complete}" +REPO="${AIOPS_GITEA_BUNDLE_SAMPLE_REPO:-wooo/awoooi}" +HOST_LABEL="${AIOPS_HOST_LABEL:-188}" +TEXTFILE_PATH="${AIOPS_GITEA_BUNDLE_RESTORE_TEXTFILE:-/home/ollama/node_exporter_textfiles/gitea_bundle_restore_dry_run.prom}" +WRITE_TEXTFILE=0 +TIMEOUT_SECONDS="${TIMEOUT_SECONDS:-120}" + +usage() { + cat <<'USAGE' +Usage: scripts/backup/gitea-bundle-sample-restore-dry-run.sh [options] + +Options: + --root PATH Bundle root directory. + --repo OWNER/NAME Repository sample to verify. Default: wooo/awoooi. + --write-textfile Write node-exporter textfile metric after verification. + --textfile PATH Textfile metric path. + -h, --help Show this help. + +This verifier clones the bundle into a temporary mirror and runs git fsck. +It never restores to production, never contacts Gitea, and never reads tokens. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --root) + shift + ROOT="${1:?--root requires PATH}" + ;; + --repo) + shift + REPO="${1:?--repo requires OWNER/NAME}" + ;; + --write-textfile) + WRITE_TEXTFILE=1 + ;; + --textfile) + shift + TEXTFILE_PATH="${1:?--textfile requires PATH}" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +slug="${REPO//\//__}" +repo_name="${REPO##*/}" +bundle_path="" +checksum_path="" + +if [ -f "${ROOT}/${slug}.bundle" ]; then + bundle_path="${ROOT}/${slug}.bundle" +elif [ -f "${ROOT}/manifest.remote.tsv" ] || [ -f "${ROOT}/manifest.tsv" ]; then + manifest="${ROOT}/manifest.remote.tsv" + [ -f "$manifest" ] || manifest="${ROOT}/manifest.tsv" + bundle_path="$(awk -F '\t' -v repo="$REPO" 'NR > 1 && $1 == repo {print $4; exit}' "$manifest")" + if [ -n "$bundle_path" ] && [ ! -f "$bundle_path" ] && [ -f "${ROOT}/$(basename "$bundle_path")" ]; then + bundle_path="${ROOT}/$(basename "$bundle_path")" + fi +fi + +if [ -z "$bundle_path" ] || [ ! -f "$bundle_path" ]; then + first_match="$(find "$ROOT" -maxdepth 1 -type f -name "${repo_name}-local-*.bundle" | sort | tail -n 1)" + bundle_path="$first_match" +fi + +ok=0 +ref_count=0 +error_reason="" +tmp_root="" + +if [ -z "$bundle_path" ] || [ ! -f "$bundle_path" ]; then + error_reason="bundle_not_found" +else + checksum_path="${bundle_path}.sha256" + if [ -f "$checksum_path" ]; then + expected="$(awk 'NR == 1 {print $1}' "$checksum_path")" + if command -v sha256sum >/dev/null 2>&1; then + actual="$(sha256sum "$bundle_path" | awk '{print $1}')" + else + actual="$(shasum -a 256 "$bundle_path" | awk '{print $1}')" + fi + if [ "$expected" != "$actual" ]; then + error_reason="checksum_mismatch" + fi + fi +fi + +if [ -z "$error_reason" ]; then + tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/gitea-bundle-restore.XXXXXX")" + trap 'rm -rf "$tmp_root"' EXIT + if ! timeout "$TIMEOUT_SECONDS" git clone --mirror --quiet "$bundle_path" "${tmp_root}/restore.git"; then + error_reason="clone_failed" + elif ! timeout "$TIMEOUT_SECONDS" git -C "${tmp_root}/restore.git" fsck --connectivity-only >/dev/null; then + error_reason="fsck_failed" + else + ref_count="$(git -C "${tmp_root}/restore.git" for-each-ref --format='%(refname)' | wc -l | tr -d ' ')" + ok=1 + fi +fi + +timestamp="$(date +%s)" +bundle_name="$(basename "${bundle_path:-missing}")" +root_label="${ROOT//\\/\\\\}" +root_label="${root_label//\"/\\\"}" +repo_label="${REPO//\\/\\\\}" +repo_label="${repo_label//\"/\\\"}" +bundle_label="${bundle_name//\\/\\\\}" +bundle_label="${bundle_label//\"/\\\"}" +error_label="${error_reason//\\/\\\\}" +error_label="${error_label//\"/\\\"}" + +if [ "$WRITE_TEXTFILE" -eq 1 ]; then + mkdir -p "$(dirname "$TEXTFILE_PATH")" + tmp_textfile="${TEXTFILE_PATH}.$$" + { + echo '# HELP awoooi_gitea_bundle_sample_restore_dry_run_ok Whether sample Gitea bundle restore dry-run succeeded.' + echo '# TYPE awoooi_gitea_bundle_sample_restore_dry_run_ok gauge' + printf 'awoooi_gitea_bundle_sample_restore_dry_run_ok{host="%s",repo="%s",root="%s",bundle="%s",error="%s"} %s\n' "$HOST_LABEL" "$repo_label" "$root_label" "$bundle_label" "$error_label" "$ok" + printf 'awoooi_gitea_bundle_sample_restore_dry_run_timestamp{host="%s",repo="%s"} %s\n' "$HOST_LABEL" "$repo_label" "$timestamp" + printf 'awoooi_gitea_bundle_sample_restore_dry_run_ref_count{host="%s",repo="%s"} %s\n' "$HOST_LABEL" "$repo_label" "$ref_count" + } > "$tmp_textfile" + mv "$tmp_textfile" "$TEXTFILE_PATH" +fi + +printf 'GITEA_BUNDLE_SAMPLE_RESTORE_DRY_RUN_OK=%s\n' "$ok" +printf 'REPO=%s\n' "$REPO" +printf 'BUNDLE=%s\n' "${bundle_path:-missing}" +printf 'REF_COUNT=%s\n' "$ref_count" +printf 'TEXTFILE_WRITTEN=%s\n' "$WRITE_TEXTFILE" +if [ -n "$error_reason" ]; then + printf 'ERROR=%s\n' "$error_reason" + exit 1 +fi diff --git a/scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py b/scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py new file mode 100644 index 000000000..5f78dd5eb --- /dev/null +++ b/scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import hashlib +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[1] +SCRIPT = ROOT / "gitea-bundle-sample-restore-dry-run.sh" + + +def run(command: list[str], cwd: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run(command, cwd=cwd, text=True, capture_output=True, check=True) + + +def test_gitea_bundle_sample_restore_dry_run_writes_success_metric(tmp_path: Path) -> None: + source = tmp_path / "source" + bundle_root = tmp_path / "bundles" + textfile = tmp_path / "gitea_bundle_restore_dry_run.prom" + source.mkdir() + bundle_root.mkdir() + + run(["git", "init", "-q"], source) + run(["git", "config", "user.email", "codex@example.invalid"], source) + run(["git", "config", "user.name", "Codex"], source) + (source / "README.md").write_text("restore dry-run\n", encoding="utf-8") + run(["git", "add", "README.md"], source) + run(["git", "commit", "-q", "-m", "seed"], source) + bundle = bundle_root / "wooo__awoooi.bundle" + run(["git", "bundle", "create", str(bundle), "--all"], source) + digest = hashlib.sha256(bundle.read_bytes()).hexdigest() + (bundle_root / "wooo__awoooi.bundle.sha256").write_text( + f"{digest} wooo__awoooi.bundle\n", + encoding="utf-8", + ) + + result = subprocess.run( + [ + "bash", + str(SCRIPT), + "--root", + str(bundle_root), + "--repo", + "wooo/awoooi", + "--write-textfile", + "--textfile", + str(textfile), + ], + text=True, + capture_output=True, + check=True, + ) + + assert "GITEA_BUNDLE_SAMPLE_RESTORE_DRY_RUN_OK=1" in result.stdout + rendered = textfile.read_text(encoding="utf-8") + assert 'awoooi_gitea_bundle_sample_restore_dry_run_ok{host="188",repo="wooo/awoooi"' in rendered + assert 'bundle="wooo__awoooi.bundle",error=""} 1' in rendered + assert "awoooi_gitea_bundle_sample_restore_dry_run_ref_count" in rendered diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 738c825d6..7a4be0312 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -218,10 +218,13 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert 'awoooi_backup_coverage_domain_expected_info{host="188"}' in alerts assert "BackupCoverageDomainMetricMissing188" in alerts assert "awoooi_backup_coverage_domain_fresh == 0" in alerts + assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts + assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: script = read("scripts/backup/gitea-repo-bundle-backup.sh") + restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh") assert "GIT_TERMINAL_PROMPT=0" in script assert "bundle create" in script @@ -231,3 +234,7 @@ def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: assert "gitea dump" in script assert "does not replace `gitea dump`" in script assert "tokens or passwords" in script + assert "git clone --mirror --quiet" in restore_script + assert "fsck --connectivity-only" in restore_script + assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script + assert "never reads tokens" in restore_script