version: 2026-05-19.v7
scope: "110/120/121/188 全服務、資料、設定與還原驗證備份基準"

principles:
  - "資料備份與設定備份分層：DB/PV/物件資料負責資料，configs 負責可啟動狀態。"
  - "Secrets、TLS private keys、SSH host keys 可進加密 restic/Velero 備份，但不得印到 log、repo、Telegram。"
  - "備份系統本身也要備份：restic repository health、password/key escrow、offsite copy、restore drill evidence 缺一不可。"
  - "每個備份都必須有三個證據：排程存在、最近成功時間、還原或 dry-run 驗證。"
  - "AI 自動修復在備份/還原領域預設 observe-only；禁止未經新成功備份證據與 baseline gate 的刪除、DROP DB、覆蓋 production namespace。"
  - "2026-05-19 起備份保留策略為 latest-only：每個本地 restic repo、188 MOMO 檔案備份與 Google Drive/rclone 離機鏡像都只保留最新一份。"

backup_domains:
  - id: host_configs
    owner_host: "110"
    script: "/backup/scripts/backup-configs.sh"
    repository: "/backup/configs"
    schedule: "daily via /backup/scripts/backup-all.sh"
    max_age_hours: 48
    includes:
      - "110/188/120/121: /etc/nginx, /etc/systemd/system, /etc/cron.d, /etc/crontab"
      - "110/188/120/121: /etc/letsencrypt, /etc/ssh, /etc/fstab, /etc/hosts, /etc/netplan"
      - "110: /opt/harbor, /opt/sentry, /home/wooo/monitoring, /home/wooo/scripts, /backup/scripts"
      - "188: /opt/n8n, /opt/open-webui, /opt/litellm, /opt/signoz, /home/ollama/momo-pro, /home/ollama/bin"
      - "120/121: /etc/rancher/k3s, K3s manifests, containerd/keepalived host config"
      - "K8s: workloads, services, ingress, configmaps, secrets, RBAC, PV/PVC, CRDs, Velero schedules/backups"
    restore_test: "抽樣 restic restore 到隔離目錄，確認 nginx/systemd/K8s YAML 可讀；不得直接覆蓋 production。"

  - id: awoooi_databases
    owner_host: "110"
    scripts:
      - "/backup/scripts/backup-awoooi.sh"
      - "/backup/scripts/backup-awoooi-frequent.sh"
    repository: "/backup/awoooi"
    schedule: "daily 02:00 + high-frequency 08:00/14:00/20:00"
    max_age_hours: 7
    includes:
      - "awoooi_prod"
      - "awoooi_dev"
      - "k3s_datastore if present"
    restore_test: "pg_restore/psql 到隔離 DB，驗證 schema 與核心表筆數；不可覆蓋 production DB。"

  - id: gitea_and_ci
    owner_host: "110"
    repository: "/backup/gitea"
    schedule: "daily via backup-all"
    max_age_hours: 48
    includes:
      - "Gitea DB"
      - "Git repositories"
      - "Gitea app.ini 與 runner registration/config evidence"
      - "workflow definitions from repos"
    restore_test: "抽樣 git fsck / git clone；Gitea DB dump 可讀。"

  - id: harbor_registry
    owner_host: "110"
    repository: "/backup/harbor"
    schedule: "daily via backup-all"
    max_age_hours: 48
    includes:
      - "Harbor DB/config"
      - "registry storage"
      - "TLS/config state from configs backup"
    restore_test: "抽樣 registry manifest/blobs 可讀；Harbor compose/config 可重建。"

  - id: observability
    owner_host: "110"
    repositories:
      - "/backup/monitoring"
      - "/backup/signoz"
    schedule: "daily via backup-all"
    max_age_hours: 48
    includes:
      - "Prometheus TSDB"
      - "Grafana dashboards/datasources"
      - "Alertmanager config/state"
      - "SignOz ClickHouse/SQLite/config"
      - "blackbox/node-exporter textfile config"
    restore_test: "Prometheus/Grafana/Alertmanager 設定 lint；SignOz dump 可列出表。"

  - id: sentry
    owner_host: "110"
    coverage_status: "covered_by_backup_sentry_script"
    script: "/backup/scripts/backup-sentry.sh"
    repository: "/backup/sentry"
    schedule: "daily via backup-all; config also covered by /backup/configs"
    max_age_hours: 48
    includes:
      - "Sentry compose/.env/config"
      - "Sentry Postgres logical dump"
      - "Sentry ClickHouse volume snapshot and table inventory"
      - "Sentry Kafka queue volume snapshot"
      - "Sentry Redis / SeaweedFS / Taskbroker / Vroom / Symbolicator state"
    restore_test: "先在隔離 compose stack 驗證 Postgres dump 可讀、ClickHouse volume 可掛載、web/symbolicator/snuba 可啟動。"

  - id: credential_escrow
    owner_host: "human-controlled"
    coverage_status: "gap_p0_out_of_band_escrow_required"
    repository: "不可放在同一個 restic repo；需放在密碼管理器或離線加密金庫"
    schedule: "每次新增/輪替 Secret 後立即更新 escrow；每月人工抽查"
    max_age_hours: 744
    includes:
      - "restic password files / repository keys / Google Drive rclone.conf / offsite provider credentials"
      - "Cloud DNS / registrar / CDN / tunnel 管理帳號與 recovery codes"
      - "Gitea/Harbor/Sentry/admin break-glass credentials"
      - "Git deploy keys、runner registration tokens、K8s bootstrap/admin kubeconfig 的復原路徑"
      - "Google Drive / OAuth / Telegram / AI provider tokens 的輪替與復原流程，不包含明文輸出"
    restore_test: "用人工雙人覆核方式確認 key escrow 可找到、可解密、可用於列出 snapshots；不得把 Secret 值寫進 repo 或監控 label。"

  - id: external_dns_and_public_routes
    owner_host: "110"
    coverage_status: "covered_by_public_route_evidence_backup; provider_zone_export_still_requires_credentials"
    script: "/backup/scripts/backup-public-routes.sh"
    repository: "/backup/public-routes"
    schedule: "daily via backup-all; DNS/CDN provider zone export after every routing change when credentials are available"
    max_age_hours: 168
    includes:
      - "wooo.work DNS answers；CDN/Cloudflare/registrar 設定匯出仍需 provider token"
      - "public nginx route map、TLS renewal config、ACME account evidence"
      - "blackbox public endpoint inventory 與 expected status codes"
      - "VPN/tunnel/port-forward/HA VIP 對外路由設定"
    restore_test: "從匯出檔重建 public route checklist，確認 awoooi/mo/registry/harbor/gitea 等 endpoint 對應正確；不得在測試中改正式 DNS。"

  - id: backup_repositories_and_integrity
    owner_host: "110/188/121/offsite"
    coverage_status: "covered_locally_by_check_backup_integrity_script; offsite copy still depends on credentials"
    scripts:
      - "/backup/scripts/check-backup-integrity.sh"
      - "/backup/scripts/configure-offsite-rclone.sh"
      - "/backup/scripts/configure-offsite-b2.sh"
      - "/backup/scripts/sync-offsite-backups.sh"
      - "/backup/scripts/backup-offsite-readiness-gate.sh"
      - "/backup/scripts/offsite-escrow-evidence-report.sh"
      - "/backup/scripts/mark-credential-escrow-verified.sh"
    repositories:
      - "/backup/* restic repos"
      - "/home/ollama/backup/110"
      - "Google Drive/rclone/offsite remote when credentials are configured"
    schedule: "daily freshness; daily 06:10 offsite status; daily 06:15 offsite escrow evidence report; weekly restic check; monthly sample restore drill"
    max_age_hours: 168
    includes:
      - "restic snapshots metadata、repo config、locks/prune policy"
      - "188 backup-from-110 rsync copy"
      - "offsite copy status and retention policy"
      - "restore drill logs with snapshot id and restored object counts"
    restore_test: "每週 `restic check --read-data-subset=1%`；每月 `restic dump latest <sample>` 到 0700 暫存目錄驗證可讀。"
    retention_policy: "latest-only；本地 restic repo 新 snapshot 成功後 --group-by \"\" --keep-last=1 + prune；188 MOMO 檔案備份只留最新一份；離機 Google Drive/rclone 以本地 repo 為準鏡像刪舊。"
    offsite_sync_policy: "offsite-escrow-evidence-report.sh 先產出紅acted 證據與 NEXT_STEP；backup-offsite-readiness-gate.sh 再做 status / dry-run-small / pre-full-sync；sync-offsite-backups.sh 預設 status；dry-run 可隨時執行；Google Drive/rclone full sync 需選低峰窗口，成功後才寫 /backup/offsite/rclone-last-success，且 OFFSITE_SYNC_DELETE_OLD=1 時會刪除遠端舊檔。full sync 不得與本地備份程序重疊，且必須距離下一次備份排程至少 270 分鐘。"

  - id: momo_web_and_data
    owner_host: "188"
    scripts:
      - "/backup/scripts/backup-momo.sh on 110"
      - "/home/ollama/bin/momo-pg-backup.sh on 188"
    repositories:
      - "/backup/momo"
      - "/home/ollama/momo_backups"
    schedule: "110 daily + 188 daily 02:00"
    max_age_hours: 30
    includes:
      - "mo.wooo.work app DB"
      - "momo uploads/files/config"
      - "scheduler config and cron"
    restore_test: "隔離 DB restore 後跑 app health check；確認 mo.wooo.work 需要的資料表與資料筆數。"

  - id: ai_and_tooling
    owner_host: "188"
    coverage_status: "covered_by_backup_ai_artifacts_for_manifest_and_metadata; model_blobs_require_manual_classification"
    script: "/backup/scripts/backup-ai-artifacts.sh"
    repositories:
      - "/backup/langfuse"
      - "/backup/open-webui"
      - "/backup/clawbot"
      - "/backup/configs"
      - "/backup/ai-artifacts"
    schedule: "daily via backup-all"
    max_age_hours: 48
    includes:
      - "Langfuse traces/evaluations"
      - "Open-WebUI conversations/config"
      - "LiteLLM config, model routing, provider state"
      - "OpenClaw/ClawBot Redis or persistent state"
      - "n8n workflows/credentials through encrypted config backup"
      - "Ollama model manifest/tag list/Modelfile；自製或不可重新下載的 model/adapters 才備份 blobs"
      - "KM/RAG/vector 狀態；若存在於 AWOOOI DB，隨 DB dump 還原；若是外部 vector store 必須有獨立 dump"
    restore_test: "抽樣匯出 workflow/config；Redis dump 可讀；Langfuse/Open-WebUI DB dump 可讀；Ollama manifest tar 可列出模型 tags。"

  - id: source_of_truth_and_ops_memory
    owner_host: "110/Gitea"
    coverage_status: "gap_p1_sanitized_operational_context"
    repositories:
      - "/backup/gitea"
      - "/backup/configs"
    schedule: "Gitea daily; configs daily; 每次事故後更新 docs/LOGBOOK.md 與 runbooks"
    max_age_hours: 48
    includes:
      - "所有 Git repositories、Ansible roles/playbooks/inventory、K8s manifests、monitoring rules"
      - "AGENTS/HARD_RULES/runbooks/LOGBOOK/ADR 等決策與啟動順序文件"
      - "AI agent handoff summaries and operational memory exports after sanitization"
      - "CI/CD workflow definitions、runner labels、deployment marker policy"
    restore_test: "從 Gitea backup 抽樣 clone repo，跑 ansible/k8s/alerts YAML validation；不得備份含明文 token 的聊天或 shell transcript。"

  - id: k3s_and_velero
    owner_host: "120"
    schedule: "Velero daily-awoooi-prod + weekly restore dry-run"
    max_age_hours: 25
    includes:
      - "K8s manifests and CRDs"
      - "Secrets/ConfigMaps/RBAC"
      - "PVC/PV snapshots via Velero provider"
      - "backup-restore-test CronJob and result metrics"
    restore_test: "backup-restore-test CronJob 每週 dry-run 到 restore-test-dry namespace mapping。"

  - id: offsite_and_dr
    owner_host: "188/121"
    schedule: "188 backup-from-110 daily 01:00; 121 DR drill monthly"
    max_age_hours: 25
    includes:
      - "110 Harbor/Gitea/bitan rsync copy on 188"
      - "DR drill evidence on 121"
      - "Google Drive/rclone remote when credentials are configured"
    restore_test: "121 DR drill dry-run finds latest Completed Velero backup; 188 backup-from-110 textfile fresh。"

monitoring_contract:
  textfile_metrics:
    "110": "/home/wooo/node_exporter_textfiles/backup_health.prom"
    "188": "/home/ollama/node_exporter_textfiles/backup_health.prom"
    "120": "由 110 backup_health.prom 透過 120 kubectl 查詢 Velero/CronJob/Job 狀態"
  offsite_and_escrow_metrics:
    - "awoooi_backup_offsite_configured：只回報 Google Drive/rclone 或相容 provider 是否看起來已配置，不輸出 credential 值。"
    - "awoooi_backup_offsite_fresh：由 /backup/offsite/*last_success 類 marker 判斷離機同步是否新鮮。"
    - "awoooi_backup_offsite_partial_fresh：由小範圍 partial sync marker 判斷 Google Drive/rclone 寫入路徑是否已被證明。"
    - "awoooi_backup_credential_escrow_fresh：由 /backup/escrow-evidence/*.last_verified 類 marker 判斷人工金庫覆核是否在 31 天內完成。"
    - "awoooi_backup_dr_next_step_info：用 next_step label 告訴 AI 巡檢與 operator 下一個安全人工作業，不包含 secret。"
    - "awoooi_backup_dr_credential_escrow_missing_count：金庫覆核尚缺的項目數。"
    - "awoooi_backup_cron_active_duplicate_count：110 active crontab 中 exact duplicate entry 的數量。"
    - "awoooi_backup_cron_singular_entry_ok：offsite/status/verifier/exporter 等單一入口排程是否剛好只有一條 active cron。"
    - "awoooi_backup_config_capture_ok：最新 configs snapshot 是否實際捕捉 110/120/121/188 host config 與 K8s workloads/secrets，不輸出 secret。"
    - "awoooi_backup_config_capture_critical_failed_count：最新設定檔備份缺少的 critical capture target 數量。"
  prometheus_alerts:
    - BackupHealthMonitorMissing110
    - BackupHealthMonitorMissing188
    - BackupHealthMonitorStale
    - BackupExpectedJobMissing
    - BackupScheduleDuplicateActiveEntries
    - BackupScheduleSingletonMismatch
    - BackupScriptMissing
    - BackupJobStale
    - BackupAggregateRunFailed
    - BackupConfigCapturePartial
    - BackupConfigCaptureStatusStale
    - BackupIntegrityCheckMissingOrFailed
    - BackupRestoreDrillMissingOrFailed
    - BackupRestoreTestMissing
    - BackupRestoreTestCronMissing
    - BackupRestoreTestFailed
    - BackupRestoreTestStale
    - BackupOffsiteCopyNotConfigured
    - BackupOffsiteCopyStale
    - BackupCredentialEscrowEvidenceMissing
    - BackupRetentionPolicyNotLatestOnly
    - BackupSnapshotRetentionExceeded
    - BackupOffsiteFullVerifyFailed
    - BackupOffsiteRemoteSnapshotRetentionExceeded
  live_visibility_checks:
    - "如果 awoooi_backup_offsite_configured{host=\"110\"} 為 0，Prometheus 必須有 BackupOffsiteCopyNotConfigured firing，Alertmanager 必須有 active alert。"
    - "如果 offsite provider 已配置、full marker 尚未 fresh，且 full sync enable marker 缺失或已超過 30 小時，Prometheus 與 Alertmanager 必須看得到 BackupOffsiteCopyStale。"
    - "如果 awoooi_backup_credential_escrow_fresh{host=\"110\"} == 0，Prometheus 與 Alertmanager 必須依 item 看得到 BackupCredentialEscrowEvidenceMissing。"
    - "如果 awoooi_backup_retention_latest_only{host=\"110\"} 或 awoooi_backup_retention_offsite_delete_old_enabled{host=\"110\",provider=\"rclone\"} 缺失/不為 1，Prometheus 與 Alertmanager 必須看得到 BackupRetentionPolicyNotLatestOnly。"
    - "如果任一 awoooi_backup_job_snapshot_count{host=\"110\",type=\"restic\"} > 1，Prometheus 與 Alertmanager 必須看得到 BackupSnapshotRetentionExceeded。"
    - "如果 full offsite marker fresh 但 awoooi_backup_offsite_remote_verify_ok{host=\"110\",provider=\"rclone\"} 不為 1 或缺失，Prometheus 必須看得到 BackupOffsiteFullVerifyFailed。"
    - "如果 full offsite marker fresh 且任一 awoooi_backup_offsite_remote_snapshot_count{host=\"110\",provider=\"rclone\"} > 1，Prometheus 必須看得到 BackupOffsiteRemoteSnapshotRetentionExceeded。"
    - "如果 awoooi_backup_cron_active_duplicate_count{host=\"110\"} > 0，Prometheus 與 Alertmanager 必須看得到 BackupScheduleDuplicateActiveEntries。"
    - "如果任一 awoooi_backup_cron_singular_entry_ok{host=\"110\"} == 0，Prometheus 與 Alertmanager 必須看得到 BackupScheduleSingletonMismatch。"
    - "如果任一 awoooi_backup_config_capture_ok{host=\"110\",critical=\"true\"} == 0，Prometheus 與 Alertmanager 必須看得到 BackupConfigCapturePartial，且 target label 必須指出缺哪個設定來源。"
    - "如果 awoooi_backup_config_capture_status_timestamp 缺失或超過 48 小時，Prometheus 與 Alertmanager 必須看得到 BackupConfigCaptureStatusStale。"
    - "live visibility check 只讀 Prometheus / Alertmanager API，不送測試告警、不改 silence、不改 route、不觸發修復。"
  prometheus_recording_rules:
    - awoooi_recovery_core_ready
    - awoooi_recovery_dr_offsite_ready

release_gate:
  cold_start_script: "bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color"
  p3_script: "bash scripts/reboot-recovery/p3-controlled-release-gate.sh"
  recovery_core_scorecard: "bash scripts/reboot-recovery/full-stack-recovery-scorecard.sh --require-core"
  dr_offsite_operator_checklist: "bash scripts/reboot-recovery/dr-offsite-operator-checklist.sh"
  dr_offsite_scorecard: "bash scripts/reboot-recovery/full-stack-recovery-scorecard.sh --require-dr"
  dr_offsite_final_gate: "bash scripts/reboot-recovery/dr-offsite-operator-checklist.sh --require-dr"
  dr_offsite_post_marker_wait: "bash scripts/reboot-recovery/wait-dr-offsite-ready.sh --timeout-seconds 900 --interval-seconds 30 --no-color"
  required_green:
    - "backup_health.prom fresh on 110/188"
    - "awoooi_backup_job_fresh == 1 for every expected job"
    - "Velero latest Completed backup < 25h"
    - "backup-restore-test CronJob present and lastSuccessfulTime not stale"
    - "weekly restic check successful"
    - "monthly sample restore drill successful"
  warning_until_human_escrow_ready:
    - "offsite provider configured and latest offsite copy marker fresh"
    - "credential escrow marker files refreshed after human verification; marker files must contain only timestamp/evidence id, never secret values"
  strict_dr_exit_conditions:
    - "Google Drive/rclone provider configured on 110 host-local rclone.conf; /backup/scripts/offsite.env keeps only non-secret remote/path with mode 0600"
    - "credential escrow markers fresh for restic_repository_password, offsite_provider_credentials, break_glass_admin_credentials, dns_registrar_recovery, oauth_ai_provider_recovery"
    - "full offsite marker /backup/offsite/rclone-last-success fresh after full 13 repo sync"
    - "full-stack-recovery-scorecard.sh --require-dr exits 0"
    - "recovery-scorecard-contract-check.py --expect-dr-ready exits 0 against 110 Prometheus"
    - "dr-offsite-operator-checklist.sh --require-dr exits 0 after scorecard, Prometheus recording rule, and backup alert visibility contract agree"
    - "wait-dr-offsite-ready.sh exits 0 after post-marker textfile, Prometheus, Alertmanager, and final checklist convergence"
