From 3b65f876246d4b4e3672088e6bc7246a9e588279 Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 30 Jun 2026 19:11:40 +0800 Subject: [PATCH] fix(backup): add verified gitea bundle fallback --- docs/LOGBOOK.md | 17 +++ .../runbooks/REBOOT-POST-START-QUICK-CHECK.md | 14 +- ...oot-cold-start-backup-recovery-workplan.md | 4 +- scripts/backup/gitea-repo-bundle-backup.sh | 143 ++++++++++++++++++ .../test_reboot_p0_operational_contract.py | 13 ++ 5 files changed, 188 insertions(+), 3 deletions(-) create mode 100755 scripts/backup/gitea-repo-bundle-backup.sh diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 39afc709..ebd13f23 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,20 @@ +## 2026-06-30 — 19:47 Gitea verified emergency bundle backup fallback integrated + +**照主線修正的問題**： +- Gitea repo「看不到」已拆成 public visibility / private auth / internal Git remote / server-side storage 四層；目前沒有證據可宣稱所有 repo 都消失，但有明確證據顯示完整備份不足。 +- 188 `gitea_repo_mirror_from_110` metric 已能把 Gitea 子樹 / bundle 變成獨立 service coverage，不再讓 `backup_from_110 fresh=1` 掩蓋 Gitea 空目錄。 +- 新增 `scripts/backup/gitea-repo-bundle-backup.sh` 作為 110 SSH command path 未恢復時的 emergency repo-history fallback；腳本使用 `GIT_TERMINAL_PROMPT=0`、輸出 `manifest.tsv`、建立 `.bundle`、跑 `git bundle verify`、產生 `.sha256`，且明確聲明不能取代 `gitea dump`。 +- 本次 runtime evidence：188 已建立 `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`，`latest` symlink 指向該 snapshot；`awoooi`、`ewoooc`、`2026FIFAWorldCup`、`agent-bounty-protocol` bundle verify + `.sha256` 成功；`AwoooGo`、`stockplatform-v2`、`vibework` 因 private repo 需要非互動 credential 而 fail-closed。這是部分 Git history 備援，不是 Gitea DB / settings / issues / packages / LFS / private repo 全量備份。 +- `REBOOT-POST-START-QUICK-CHECK.md` 升到 v1.20，將 emergency bundle、manifest 判讀、private repo fail-closed、正式 `gitea dump` / restore drill 仍為 P0 blocker 寫入 SOP；reboot workplan 的 P0-6 同步更新。 + +**live readback / 驗證**： +- Gitea version API：`1.25.5`；unauth public repo search：4 個 public repos；`/Users/ogt/stockplatform-v2` 的 `gitea` remote 可 `ls-remote --heads`，因此不能宣稱所有 Git objects 消失。 +- 188 backup metric：`gitea_repo_mirror_from_110` fresh `1`、snapshot_count `19`、`awoooi_backup_coverage_domain_fresh{host="188",domain="service"} 1`；這代表 emergency bundle subtree 已被 service coverage metric 看到，但不代表 full Gitea dump 完成。 +- 110 SSH command path 仍 timeout；正式 `gitea dump`、private repo non-interactive backup、repo count 與 restore drill 仍是 P0 blocker。 +- 本地驗證：`bash -n scripts/backup/gitea-repo-bundle-backup.sh`、bundle script dry-run、focused reboot contract pytest `5 passed`、`YAML_OK`、runner pressure guard、Gitea secret step guard、`py_compile`、`git diff --check` 全部通過。 + +**邊界**：未 workflow_dispatch，未 SSH 寫 110，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 restore / prune / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 + ## 2026-06-30 — 19:33 AI Agent autonomous runtime public redaction CD fix **照 CD 紅燈修正的問題**： diff --git a/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md b/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md index b37d536e..40374d98 100644 --- a/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md +++ b/docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md @@ -1,6 +1,6 @@ # 主機重啟後一頁式總檢查 -> Version: v1.19 +> Version: v1.20 > Last updated: 2026-06-30 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 post-reboot service recovery. 112 Kali / Wazuh / active scan 只做可達性與託管狀態 evidence，不做 active scan。 @@ -33,6 +33,7 @@ | 110 control path | ping / SSH port 可達，但 `wooo@192.168.0.110` SSH command timeout。 | 110 control path 是 P0 blocker；未恢復前不可宣稱 Stock DB/schema、Gitea dump、110 backup completeness 已現場驗證。 | | Gitea visibility | `https://gitea.wooo.work/api/v1/version` 回 200 / `1.25.5`；public repo search 只列 4 個 public repos；`stockplatform-v2` public page/API 404，但 direct internal Git remote `http://192.168.0.110:3001/wooo/stockplatform-v2.git` 可 `ls-remote`。 | 不能直接說 repo 全消失；要區分 public visibility、private/auth visibility、內網 Git remote、server-side storage 四層。 | | Gitea backup | 188 `/home/ollama/backup/110/gitea` 目前為空；舊 `backup_from_110` freshness 只證明 rsync 心跳，不證明 Gitea 子樹或 dump 完整。 | 備份監控必須新增 Gitea subtree freshness / sample count；空目錄是 P0 backup completeness blocker。 | +| Gitea emergency bundle | 188 已建立 verified emergency Git bundle snapshot `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`，`awoooi`、`ewoooc`、`2026FIFAWorldCup`、`agent-bounty-protocol` bundle verify + `.sha256` 成功；`AwoooGo`、`stockplatform-v2`、`vibework` 因 private repo 需要非互動 credential 而 fail-closed。 | 這只保住部分 repo Git history，不等於 `gitea dump`；Gitea DB、settings、issues、packages、secrets、LFS、private repo completeness 與 restore drill 仍是 P0 blocker。 | 本次事故後新增固定判讀： @@ -277,10 +278,21 @@ Gitea backup completeness 必須同時看： ```bash ssh ollama@192.168.0.188 'find /home/ollama/backup/110/gitea -maxdepth 3 -type f | head' ssh ollama@192.168.0.188 'grep -E "gitea_repo_mirror_from_110|awoooi_backup_coverage_domain" /home/ollama/node_exporter_textfiles/backup_health.prom' +scripts/backup/gitea-repo-bundle-backup.sh --dry-run ``` `backup_from_110` fresh 但 `/home/ollama/backup/110/gitea` 空，必須視為 `gitea_backup_completeness_blocked`；只能先修 backup coverage / dump / rsync include，再做 restore drill。不得用「每日備份心跳正常」覆蓋 Gitea 子樹空目錄。 +若 110 SSH command path 尚未恢復、無法立刻執行正式 `gitea dump`，可先跑 emergency repo-history fallback： + +```bash +scripts/backup/gitea-repo-bundle-backup.sh \ + --output-root /home/ollama/backup/110/gitea/git-bundles \ + --gitea-base http://192.168.0.110:3001 +``` + +合格輸出必須有 `manifest.tsv`、每個成功 repo 的 `.bundle`、`.sha256`，且 bundle 經 `git bundle verify` 通過。manifest 中的 `ls_remote_failed`、`clone_failed`、`bundle_failed`、`verify_failed` 都是 backup completeness blocker；不得因部分 public repo 成功就宣稱 Gitea 備份完整。這支腳本使用 `GIT_TERMINAL_PROMPT=0`，不要求、不讀取、不印出 token 或 password；private repo 缺非互動 credential 時必須 fail-closed。 + ### Step 6 - Public routes 只作輔助證據 ```bash diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 7cbddd8e..67045730 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -20,14 +20,14 @@ | P0-3 | BLOCKED | StockPlatform data freshness | public `/healthz`、`/api/healthz` 回 200；freshness / ingestion 回 `not_configured`、`postgres_not_ready`。 | 恢復 110 control path 後，read-only 查 `/home/wooo/stockplatform-v2` compose / DB schema / migration status；禁止 fake freshness、manual DB rows、restore/prune。 | | P0-4 | BLOCKED | AWOOOI production 版本最新性 | source/Gitea `main=adc0d8816`；production readback 仍為 `7890778b83`；新 Stock preflight endpoint production 404。 | 補 deploy marker / runtime SHA / endpoint readback 一致；未一致前不可宣稱 AWOOOI 最新。 | | P0-5 | BLOCKED | 110 control path | ping / SSH port 可達，但 SSH command timeout。 | 恢復 SSH read-only command path；完成後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | -| P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 為空。 | 新增 188 `gitea_repo_mirror_from_110` subtree freshness metric 與 188 coverage alert；再補實際 Gitea dump / mirror / restore drill readback。 | +| P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 起初為空。已建立 verified emergency bundle `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`：4 個 public/internal repo bundle verify + checksum 成功，`AwoooGo`、`stockplatform-v2`、`vibework` 因 private auth fail-closed。 | 188 `gitea_repo_mirror_from_110` subtree metric / alert 已補；下一步仍是恢復 110 SSH command path 後跑正式 `gitea dump`、private repo 非互動備份、repo count 與 restore drill readback。 | | P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`，但 99 SSH/WinRM 尚未可用；VM host 111 仍不可達。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence。 | | P0-8 | SOURCE_READY_RUNTIME_BLOCKED | 502 maintenance fallback / Telegram / backup alert | L0/L1 fallback runbook、Nginx snippet、reboot / backup alert rules 已在 source；runtime 尚需部署與外部 L1 provider readback。 | L0 以測試 vhost 驗證 `X-AWOOOI-Fallback`；L1 需外部雲端/CDN probe；Telegram 以脫敏 alert receipt 驗證。 | 本次核心經驗： - Gitea public UI/API 只代表 public visibility；private/internal repo 必須用 authenticated inventory 或 internal `git ls-remote` 分層驗證。 -- 備份心跳 fresh 不等於完整備份；Gitea 子樹 / dump / repo count / restore drill 必須單獨告警。 +- 備份心跳 fresh 不等於完整備份；Gitea 子樹 / emergency bundle / dump / repo count / restore drill 必須單獨告警。bundle 只保 Git history，不等於 Gitea 全量 dump。 - 使用者可見 502 優先於資料 freshness；先恢復靜態/容器服務，再回到資料層與版本一致性。 - 版本最新性要同時看 source SHA、deploy marker、runtime SHA 與 public endpoint；不能只看 Gitea main。 diff --git a/scripts/backup/gitea-repo-bundle-backup.sh b/scripts/backup/gitea-repo-bundle-backup.sh new file mode 100755 index 00000000..f4616b9c --- /dev/null +++ b/scripts/backup/gitea-repo-bundle-backup.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# Create emergency Git bundle backups from Gitea over Git HTTP. +# This is a repository-history fallback only. It does not replace `gitea dump`, +# does not back up Gitea DB rows, settings, issues, packages, secrets, or LFS. + +set -euo pipefail + +GITEA_BASE_URL="${GITEA_BASE_URL:-http://192.168.0.110:3001}" +OUTPUT_ROOT="${OUTPUT_ROOT:-/home/ollama/backup/110/gitea/git-bundles}" +TIMEOUT_LS_REMOTE_SECONDS="${TIMEOUT_LS_REMOTE_SECONDS:-60}" +TIMEOUT_CLONE_SECONDS="${TIMEOUT_CLONE_SECONDS:-180}" +TIMEOUT_BUNDLE_SECONDS="${TIMEOUT_BUNDLE_SECONDS:-180}" +TIMEOUT_VERIFY_SECONDS="${TIMEOUT_VERIFY_SECONDS:-60}" +STAMP="${STAMP:-$(date +%Y%m%d-%H%M%S)}" +DRY_RUN=0 +REPOS=() + +usage() { + cat <<'USAGE' +Usage: scripts/backup/gitea-repo-bundle-backup.sh [options] + +Options: + --repo OWNER/NAME Repository to bundle. May be passed more than once. + --output-root PATH Bundle root. Default: /home/ollama/backup/110/gitea/git-bundles + --gitea-base URL Gitea base URL. Default: http://192.168.0.110:3001 + --stamp STAMP Output directory stamp. Default: current local timestamp. + --dry-run Print target URLs without cloning or writing bundles. + -h, --help Show this help. + +This script uses GIT_TERMINAL_PROMPT=0 and never asks for, reads, or prints +tokens or passwords. Private repositories without an existing credential fail +closed in the manifest. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --repo) + shift + REPOS+=("${1:?--repo requires OWNER/NAME}") + ;; + --output-root) + shift + OUTPUT_ROOT="${1:?--output-root requires PATH}" + ;; + --gitea-base) + shift + GITEA_BASE_URL="${1:?--gitea-base requires URL}" + ;; + --stamp) + shift + STAMP="${1:?--stamp requires value}" + ;; + --dry-run) + DRY_RUN=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +if [ "${#REPOS[@]}" -eq 0 ]; then + REPOS=( + "wooo/awoooi" + "wooo/ewoooc" + "wooo/2026FIFAWorldCup" + "wooo/agent-bounty-protocol" + "wooo/AwoooGo" + "wooo/stockplatform-v2" + "wooo/vibework" + ) +fi + +if [ "$DRY_RUN" -eq 1 ]; then + for repo in "${REPOS[@]}"; do + printf 'DRY_RUN repo=%s url=%s/%s.git\n' "$repo" "$GITEA_BASE_URL" "$repo" + done + exit 0 +fi + +backup_root="${OUTPUT_ROOT%/}/${STAMP}" +tmp_root="$(mktemp -d "${TMPDIR:-/tmp}/gitea-bundle.XXXXXX")" +manifest="${backup_root}/manifest.tsv" +mkdir -p "$backup_root" +trap 'rm -rf "$tmp_root"' EXIT + +printf 'repo\tstatus\thead_count\tbundle\tchecksum\n' > "$manifest" + +for repo in "${REPOS[@]}"; do + slug="${repo//\//__}" + url="${GITEA_BASE_URL%/}/${repo}.git" + refs_file="${backup_root}/${slug}.refs" + err_file="${backup_root}/${slug}.err" + bundle_file="${backup_root}/${slug}.bundle" + checksum_file="${bundle_file}.sha256" + repo_tmp="${tmp_root}/${slug}.git" + + if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_LS_REMOTE_SECONDS" \ + git ls-remote --heads "$url" >"$refs_file" 2>"$err_file"; then + printf '%s\tls_remote_failed\t0\t\t\n' "$repo" >> "$manifest" + continue + fi + + head_count="$(wc -l < "$refs_file" | tr -d ' ')" + rm -rf "$repo_tmp" + if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_CLONE_SECONDS" \ + git clone --mirror --quiet "$url" "$repo_tmp" 2>>"$err_file"; then + printf '%s\tclone_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + continue + fi + + if ! timeout "$TIMEOUT_BUNDLE_SECONDS" \ + git -C "$repo_tmp" bundle create "$bundle_file" --all >/dev/null 2>>"$err_file"; then + printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + continue + fi + + if ! timeout "$TIMEOUT_VERIFY_SECONDS" \ + git -C "$repo_tmp" bundle verify "$bundle_file" >/dev/null 2>>"$err_file"; then + printf '%s\tverify_failed\t%s\t%s\t\n' "$repo" "$head_count" "$bundle_file" >> "$manifest" + continue + fi + + (sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file" + rm -rf "$repo_tmp" + printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest" +done + +latest_link="${OUTPUT_ROOT%/}/latest" +if [ ! -e "$latest_link" ] || [ -L "$latest_link" ]; then + ln -sfn "$backup_root" "$latest_link" +else + echo "WARN latest path exists and is not a symlink: $latest_link" >&2 +fi +cat "$manifest" diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 510bf6f9..f70d895e 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -61,3 +61,16 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert 'awoooi_backup_coverage_domain_expected_info{host="188"}' in alerts assert "BackupCoverageDomainMetricMissing188" in alerts assert "awoooi_backup_coverage_domain_fresh == 0" in alerts + + +def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: + script = read("scripts/backup/gitea-repo-bundle-backup.sh") + + assert "GIT_TERMINAL_PROMPT=0" in script + assert "bundle create" in script + assert "bundle verify" in script + assert ".sha256" in script + assert "manifest.tsv" in script + assert "gitea dump" in script + assert "does not replace `gitea dump`" in script + assert "tokens or passwords" in script