diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index 61e7e3822..3d96766e2 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -1,6 +1,6 @@ { "schema_version": "agent99_enterprise_ai_automation_work_items_v1", - "generated_at": "2026-07-10T16:22:00+08:00", + "generated_at": "2026-07-10T19:40:00+08:00", "status": "in_progress_p0_alert_routing_grouping_dedupe", "scope_complete": false, "north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.", @@ -20,8 +20,8 @@ "id": "AG99-P0-002", "title": "Alert taxonomy, routing, grouping and dedupe", "reason": "Explicit event kind and suggested mode are not consistently authoritative, and forwarding before grouping can create wrong-domain or duplicate Agent99 work.", - "implementation_status": "in_progress_replay_corpus_and_pre_bridge_grouping", - "next_action": "Add routing precedence and a replay corpus for backup, provider, reboot, PostgreSQL, host and K3s events; move grouping and single-flight ahead of Agent99 dispatch." + "implementation_status": "local_package_and_contract_green_runtime_deploy_pending", + "next_action": "Finish the current Gitea source receipt, atomically deploy the tested Agent99 package to Windows 99 when an authenticated no-secret management channel is available, then replay backup, provider, reboot, PostgreSQL, host and K3s events against the runtime manifest." }, "current_runtime_truth": { "observed_at": "2026-07-10T15:04:27+08:00", @@ -316,7 +316,10 @@ "verifier": "Replay routing corpus and assert one active incident and expected mode per fingerprint.", "callback": "Publish normalized route decision and dedupe reason to AWOOOI and AwoooP.", "acceptance": "Known backup, provider, reboot, PostgreSQL, host and K3s samples route to the correct lane with no duplicate same-state dispatch.", - "next_action": "Add routing precedence and pre-bridge grouping regression corpus." + "completion_percent": 80, + "implementation_progress": "Local Agent99 contract and synthetic replay are green; Windows 99 runtime deployment and runtime replay remain pending.", + "evidence_refs": ["local Agent99 PowerShell contract 19/19", "local Agent99 synthetic replay 8/8", "isolated atomic deploy test passed"], + "next_action": "Deploy the tested package atomically to Windows 99, verify the runtime manifest, and rerun the routing/dedupe corpus through the live ingress." }, { "id": "AG99-P0-003", @@ -536,7 +539,10 @@ "verifier": "Each baseline domain has schedule, freshness, integrity and isolated restore evidence.", "callback": "Report one lifecycle incident per failed backup domain and update DR scorecard.", "acceptance": "No unknown backup target; failures name the domain, age, SLA, owner, evidence and safe next action.", - "next_action": "Reconcile failed readbacks and add missing full Gitea, DB, packages, LFS and tool/log restore receipts." + "completion_percent": 78, + "implementation_progress": "The 110/188 backup heartbeat, Gitea full-dump isolated restore, private bundle off-host copy, sample restore, Velero missed-run backfill and Prometheus alert slice are live green.", + "evidence_refs": ["110 backup 13/13", "188 backup 4/4", "Gitea dump CRC and inventory 15/15", "Gitea product bundles and remote SHA-256 12/12 on 188", "Velero backfill job succeeded", "Prometheus source/canonical/active/runtime SHA matched", "Backup/Gitea non-inactive alerts 0"], + "next_action": "Merge and deploy the source branch through Gitea, then complete the remaining DB, registry, package, tool, log and AI-artifact restore-receipt matrix." }, { "id": "AG99-P0-013", @@ -580,7 +586,10 @@ "verifier": "All expected repos and refs, full dump components, registry artifacts, CD run and runtime markers read back successfully.", "callback": "Update source-control, release and DR scorecards with one trace.", "acceptance": "All 12 products have source and release proof and the platform can be reconstructed in isolation from verified backups.", - "next_action": "Add full Gitea state and registry restore receipts to the existing repo-bundle proof." + "completion_percent": 72, + "implementation_progress": "Gitea full dump and off-host private bundle restore-confidence are live verified; Harbor/Registry artifacts, runner/CD and complete platform reconstruction remain open.", + "evidence_refs": ["Gitea full dump 6928031514 bytes", "regular repository inventory 15/15", "product bundle manifest 12/12", "188 sample clone and fsck passed"], + "next_action": "Merge/deploy the Gitea DR source, then add Harbor/Registry artifact restore receipts and runner/CD/runtime-marker reconstruction proof." }, { "id": "AG99-P1-001", diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 24b9a1716..7a6b90937 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.131 -> Last updated: 2026-07-09 Asia/Taipei +> Version: v1.132 +> Last updated: 2026-07-10 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,14 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.132 2026-07-10 備份與 Gitea 可還原性閉環 + +重啟恢復不得再以「備份檔存在」或「Gitea HTTP 200」宣告備份正常。固定證據鏈為：backup exporter/status freshness -> Gitea full dump isolated restore -> private bundle complete-set sync -> sample clone/fsck -> Prometheus rule/alert readback。任一層缺失都只能標示 degraded，不能標示 restore-ready。 + +2026-07-10 19:40 live receipt：110 backup heartbeat 為 `13/13`、188 為 `4/4`、`core_blockers=0`、`dr_warnings=0`；分母由 live `awoooi_backup_job_fresh` series 動態計算，不再使用硬編碼 `13/2`，缺 metric family 會 fail closed。Gitea isolated full-dump drill 驗證 snapshot `29c8b7a8`、dump `6928031514` bytes、ZIP CRC、DB/config required artifacts、backup/live regular repositories `15/15`、wiki `0/0` 與 privacy-preserving inventory digest，全程未覆寫 production。12 個產品 private bundles 已由 110 local Gitea container 產生完整集合、no-delete 同步至 188，remote manifest `12/12` 並完成 sample clone + `git fsck`。Velero weekly restore dry-run 的 `startingDeadlineSeconds` 已由 `600` 修正為 `604800`，停機期間漏掉的排程由 controller 自動補跑且成功。 + +Prometheus active/canonical rules 在 19:40 先通過排程週期；20:17 進一步發現 host active path 與 Docker read-only bind mount 曾因 inode replacement 分叉。drift guard 現在同時比較 canonical、host active 與 container runtime SHA，canonical 先經 promtool，只有 runtime mount 分叉才受控 recreate Prometheus container，並在 rules/health 未收斂時回滾 active file。live 自癒驗收後 source/canonical/active/runtime 四方 SHA 均為 `34c0359e01d608d6300b92ca878aed414180ec725aace3ade0a19d39cc362697`，guard `status=ok`、`repaired=1`、`missing_required_count=0`、`current_matches_canonical=1`、`runtime_matches_current=1`，active alerts file 載入 `145` rules，Backup/Gitea firing 或 pending alerts 為 `0`。20:20 下一個正式 cron 週期使用相同 container ID/StartedAt，回 `repaired=0`、三方 match `1`，證明穩態不會每五分鐘 recreate。這只完成本輪 Gitea/backup isolated verification；不等於執行 production disaster restore，也不等於所有產品版本與資料 freshness 已完成。 + v1.131 AI log triage controlled auto-recovery rule：每次全主機重啟後，10 分鐘 SLO verifier 不得只跑 read-only summary。`awoooi-reboot-auto-recovery-slo.service` 必須以 `AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply` 先執行 `scripts/reboot-recovery/ai-log-triage-auto-recovery.sh`，bounded 收集 99 / 110 / 120 / 121 / 188 logs、K3s events、VMware no-secret readback 與 public route status，自動輸出 findings / actions / summary，再接 host probe、post-reboot summary、scorecard 與 textfile metrics。2026-07-09 16:23 orchestrator check artifact `/tmp/awoooi-full-host-reboot-orchestrator-codex/20260709T082305Z` 的目前 truth 是 `RECOVERY_BLOCKED_P0`、`FINDING_COUNT=3`、`FINDING_P0_COUNT=2`：`STOCK_UPSTREAM_NOT_READY host=110_188`、`EDGE_502_UPSTREAM_REFUSED host=188_edge`，另有 `HOST188_POSTGRES_SYSTEMD_FAILED` P2 hygiene；public route readback 為 AWOOOI health `200`、AWOOOI web `307`、Stock freshness `200`、Gitea `200`、registry `/v2/` `401`、Harbor `200`。這些不得再被當成噪音或人工 SOP；必須進 AI findings、Prometheus/Telegram blocker 與下一步 action matrix。允許的 `--apply` 只限低風險 allowlist：start inactive docker/nginx、Harbor watchdog log-first repair、registry ready 後刪 failed-pull pods 讓 K3s 重建。仍禁止 reboot、node drain、firewall cutover、DB destructive action、Docker prune、secret/session/.env/raw-volume reads。 v1.130 Windows99 VMX backup/source search package rule：2026-07-03 18:19 最新 via-110 no-secret locator 已把目前 truth 從早先 `candidate_source_count=6` 修正為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`，且 `raw_path_output=false`、`path_fingerprint_only=true`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_performed=false`。因此 P0-4 下一步不得再引用 generic Ubuntu candidate；必須讀 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 priority readback 的 `windows99_vmx_source_backup_search_*` 欄位。當前 backup/source search receipt 顯示 120 / 121 / local 均未找到 verified 111 source，99 alias 無 Linux backup host mapping，110 / 188 / 112 SSH BatchMode 不可用，`candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`。固定下一步是離線備份 inventory 或授權 console source path，再取得 verified source path 後建立 scoped relink dry-run；仍不得 power on VM、restore、讀 VMX 內容、讀 secret、重啟 Windows / VMware service / host / Docker / Nginx / K3s / DB / firewall。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 608d3a52b..9311e559d 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.22 -> **最後更新**: 2026-07-09 (台北時間) +> **版本**: v5.23 +> **最後更新**: 2026-07-10 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-03 18:19 Windows99 VMX backup/source blocked-host truth 上卷 +> **觸發事件**: 2026-07-10 全主機重啟後備份/Gitea 可還原性與排程補跑收斂 --- @@ -21,6 +21,16 @@ ## 架構概覽與依賴圖 +### 2026-07-10 P0 備份/Gitea 自動驗證覆蓋 + +每次 host reboot 後，備份 lane 必須依序執行 `/backup/scripts/backup-status.sh --no-notify`、`/backup/scripts/verify-gitea-backup.sh --no-notify`，再由 110 的每日排程執行 `/backup/scripts/gitea-bundle-backup-sync-188.sh`。最後讀回 110/188 `backup_health.prom`、Gitea restore status、bundle sync status、188 sample restore metric 與 Prometheus Backup/Gitea alerts；不得只看檔案 mtime、HTTP 200 或命令 exit code。 + +本輪 live 驗證已證明：110 `13/13`、188 `4/4`、core blocker `0`；Gitea full dump 為 `6928031514` bytes，CRC、DB/config、15/15 regular repo inventory digest 一致；12/12 product private bundles 已從 110 local container 產生並完整同步 188，remote SHA-256 `12/12`、sample clone/fsck 成功；Velero missed weekly dry-run 已由 `startingDeadlineSeconds=604800` 自動 backfill 成功。20:17 drift guard 自動修復 read-only bind-mount inode drift 後，source/canonical/active/runtime rule SHA 一致，145 rules 已載入，Backup/Gitea 非 inactive alerts 為空；20:20 正式 cron 回 `repaired=0` 且 container ID/StartedAt 未變，穩態無重複 recreate。 + +Prometheus rules 部署不得用會置換 bind source inode 的 `install` / atomic rename 直接覆蓋 `/home/wooo/monitoring/alerts.yml`。正式 guard 會先 promtool canonical，再以保留 host path 的方式更新 active；若 container runtime SHA 仍不同，才用 `/home/wooo/monitoring/docker-compose.yml` 單獨 `--no-deps --force-recreate prometheus` 重新掛載，TSDB volume 保留。post-verifier 必須同時通過 `145` rules health、runtime SHA、required rules 與 active alert query；失敗則還原舊 active file 並重建 Prometheus，不得只看 reload HTTP 200。 + +失敗處置固定為 fail closed：full dump 或 bundle incomplete 不得更新 complete symlink；remote sync 不使用 delete；restore drill 只能在隔離暫存目錄；不得讀 raw backup contents、secret 或 private repo 名稱，不得 restore over production、prune 或刪遠端備份。Telegram 只在狀態改變或失敗時送一張繁中摘要卡，成功心跳維持每日一次。 + ### 2026-07-09 P0 AI Log Triage / Controlled Auto-Recovery 覆蓋 主機重啟後不得再靠人工逐台看 log 才判斷異常。固定第一步改為執行 `scripts/reboot-recovery/ai-log-triage-auto-recovery.sh --apply`；若只允許讀取，則用 `--check`，但 scorecard / 告警仍必須吃同一份 artifact。此 runner 會 bounded 收集 110 Docker / containerd / Harbor / Nginx、120 / 121 K3s events、188 edge Nginx / Docker / systemd、99 VMware no-secret readback 與 public route HTTP status，然後輸出 `findings.tsv`、`actions.tsv`、`summary.txt`。 diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 5e2789810..57c36d1c8 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -25,7 +25,7 @@ | P0-2 | DEPLOYED_RUNTIME_EDGE_DRIFT_BLOCKED | Deploy / reboot 期間 public 502 維護頁與外部 fallback | 2026-07-03 15:35 source fix 已部署讀回：production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 保留 `edge_public_maintenance_live_config_drift`，`runtime_metric_active_blocker_count=8`，且 route fallback ready 不再清掉 edge Nginx drift。110 SLO artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260703-151219` 與 textfile 證明 edge 未 ready：`awoooi_public_maintenance_edge_fallback_ready{scope="public_routes"} 0`、active blocker `edge_public_maintenance_live_config_drift`。 | 後續才進 privileged edge apply 或 L1 cloud/CDN fallback decision；完成條件是 edge fallback live config / maintenance file / `nginx -t` / reload / public header verifier 或外部 L1 fallback 全綠。不得讀密碼、不得用 app / Docker / host restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_DEPLOYED_SOURCE | 所有產品 / 網站版本與資料最新性 | Gitea `main=bc8d3af3f chore(cd): deploy dacdd90 [skip ci]`；CD `#4558` log 顯示 API/Web build/push wrapper 全部 `rc=0`、deploy marker `dacdd90`、production deploy readback matched。Production SLO route 已讀回最新 API 行為：`runtime_metric_runtime_readback_added_blockers=["windows99_vmware_autostart_config_not_ready","windows99_update_no_auto_reboot_policy_not_ready"]`、`windows99_update_no_auto_reboot_ready=false`。Stock freshness 仍為 `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。 | 版本層已補到 AWOOOI production；完成整體 P0 仍需收斂 Windows99 VMX / service / policy / power 與 host reboot-event blockers。後續每個 public product 仍要維持 source SHA、deploy marker、runtime endpoint、public route watch、freshness 四層一致。 | | P0-4 | BLOCKED_WINDOWS99_VMX_SOURCE_AND_TASK_NOT_TRIGGERED | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | 2026-07-03 17:34 已確認 110→99 SSH channel 可用，via-110 locator `rc=0`，`expected_vmx_path_present_count=0`、`candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`。full verify 在 90 秒曾 timeout，但 180 秒成功讀回 `VMX alias=111 present=0`、`MISSING_VMX_ALIASES=111`、`VMWARE_AUTOSTART_CONFIG_READY=0`；因此 source 已調整 Windows99 remote verify 預設 timeout 為 `180` 秒。前一輪 CD `#4618` 已將 `36536adcf` 投產，deploy marker `acbf96c`，production package endpoint live。 | 下一步固定為取得 111 既有 VMX source / verified backup / 授權 console candidate identity，產生 scoped relink dry-run + rollback + post-verifier，再 rerun no-secret collector + SLO scorecard。仍不得從 scorecard 直接 power on VM、重啟 Windows host / VMware service、改 scheduled task / registry、猜 generic Ubuntu candidate、restore 到 production 或讀 Windows 密碼；也不得把 90 秒 timeout 當成 SSH channel 故障。 | -| P0-5 | RUNTIME_READY_BACKUP_RECEIPT_GAP | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source / runtime 已部署 `awoooi_backup_alert_receipt_*` 指標與 Prometheus rules；110 exporter 讀回 88 個 stage requirement、188 讀回 12 個 stage requirement，`BackupAlertReceiptMetricMissing*` inactive，`BackupAlertReceiptStageMissing` 已修成每 `host / receipt_channel` 聚合 pending：110 一條、188 一條。 | 補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | +| P0-5 | LIVE_GREEN_SOURCE_BRANCH_PENDING_MAIN | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | 2026-07-10 live：backup status 動態分母讀回 110 `13/13`、188 `4/4`、`core_blockers=0`、`dr_warnings=0`；Gitea full dump isolated drill 驗證 `6928031514` bytes、CRC、DB/config 與 regular repo inventory `15/15`；12/12 product private bundles 由 110 local container 產生、no-delete 同步 188、remote SHA-256 `12/12` 並 sample clone/fsck；Velero missed weekly dry-run 自動 backfill 成功。20:17 guard 修復 Prometheus bind-mount inode drift 後，source/canonical/active/runtime SHA 一致、145 rules、Backup/Gitea non-inactive alerts `0`。 | 本輪 Gitea/backup slice 已 live verified；把 branch source 推進 Gitea main/CD 後，依總帳繼續補其餘 DB、registry、package、tool、log 與 AI artifact restore matrix，不得把本列誤報成全域 DR complete。 | | P0-6 | RUNTIME_READY_ALERT_RECEIPT_GAP | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Reboot per-blocker alert 與 backup receipt alert rules 已 deploy/readback；backup receipt 缺段不再產生 100 條 stage 噪音，現在聚合成 110 / 188 兩條 host-level pending。scorecard 目前有 10 個 reboot active blockers，其中 Windows99 config / policy blocker 已可供 per-blocker alert routing 使用；尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | | P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase`、`eta_or_wait_reason`、`primary_blocker`、fixed triage order 與 action matrix；2026-07-03 新增 `windows99_vmware_autostart_task_never_ran`，其 next action 固定為 `run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_then_rerun_no_secret_collector_no_vm_power_change`，不再退回泛用「再收一次 verify」。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | | P0-8 | BLOCKED_POLICY_READBACK | Windows99 禁止 Windows Update 無預警重啟 | 10:44 production readback 已 fail-closed：`windows99_update_no_auto_reboot_ready=false`，active blockers 含 `windows99_update_no_auto_reboot_policy_not_ready`，來源是 nested verifier 的 `policy_blockers=["windows_update_policy_readback_missing"]`。 | 保留 no-secret verifier，補週期性 policy readback 與 Telegram drift alert；完成條件是 Windows Update policy drift / missing readback 會自動告警且不需讀 secret，不得從 scorecard 直接 apply registry。 | @@ -225,6 +225,8 @@ Next: | P1-015 | DONE | 100 | Restore 188 MinIO / Velero backup freshness and DB exporters | 2026-06-24 06:35 resolved real backup / exporter red lights: 188 PostgreSQL exporter and Redis exporter now expose `pg_up=1` / `redis_up=1`; 188 MinIO health is live; 120 Velero BSL is `Available`; one-off backup `reboot-recovery-202606240456` completed; 110 backup-health textfile reports latest Velero backup fresh. 110 disk pressure was reduced from 92% to 73% by Docker image/build-cache cleanup only. | Reconcile MinIO `userns_mode: host` override into formal source-of-truth or data ownership fix; keep Docker volume prune forbidden without explicit owner approval. | `VeleroBackupNotRun`、`PostgreSQLDown`、`RedisDown`、110 disk-pressure alerts are resolved, and SOP includes restore helpers. | | P1-016 | DONE | 100 | Control repeated Telegram notification noise without hiding real alerts | 2026-06-24 confirmed MOMO Pro 5-minute spam came from a legacy 110 script checking `http://192.168.0.188/health`; live script now uses `https://mo.wooo.work/health` as primary truth. Heartbeat warning dedupe now hashes stable actionable fingerprints so HTTP status / timeout / latency drift does not create a new Telegram event every 30 minutes. `MoWoooWorkDown` now labels `component=momo-pro-system`, disables blind auto-repair, and requires public/local/container/data-freshness triage. Generic docker-health monitor keeps 5-minute repair checks but adds a separate 30-minute direct Telegram fallback cooldown. Bitan public-content cleanliness keeps failure notification with same-fingerprint cooldown and one recovery notice. | Fold remaining cross-product direct Telegram egress into the unified notification gateway over time; do not disable real warning/failure/recovery signals. Production deployment/readback must confirm the code and Prometheus rules are live before declaring runtime closure. | Healthy heartbeat is quiet, same actionable heartbeat warning is deduped, MOMO public health success produces no alert, repeated same-failure direct fallback paths are cooled, and real failure/recovery/new-warning notifications remain enabled. | | P1-017 | DONE | 100 | Restore 188 nginx-exporter and post-CD monitoring coverage | CD `#3294` deployed marker `622bc372` but failed post-deploy checks because `scripts/generate_monitoring.py --check` saw Prometheus job `nginx-exporter` down at `192.168.0.188:9113`. 188 `stub_status` and compose config were healthy, so the correct fix was restoring the stateless exporter from `/home/ollama/nginx-exporter.yml`, not reloading Nginx or restarting products. New helper `scripts/ops/188-nginx-exporter-restore.sh` defaults to read-only `--check` and exposes explicit `--apply` for maintenance-window restore. `high-value-config-change-gate.py` now classifies `scripts/ops/**/*exporter*` as `monitoring_alerting_observability` P1 / C1. | Keep this check in post-reboot and post-CD recovery. Do not mark historical CD `#3294` as success; use the next CD run plus monitoring coverage as future proof. | `bash scripts/ops/188-nginx-exporter-restore.sh --check` reports `nginx_up 1`; `python3 scripts/generate_monitoring.py --check --stabilization-sleep-seconds 0` reports `Jobs=14`, `全部 UP=14`, `真實問題=0`, coverage `100.0%`; high-value gate matches the helper as P1 / C1, not unmanaged. | +| P1-018 | DONE_LIVE_SOURCE_BRANCH_PENDING_MAIN | 95 | Gitea full dump isolated restore and metrics | The previous generic drill could report fallback success without proving the 6.9 GB Gitea dump, database/config artifacts or repository inventory. A dedicated verifier now restores to an isolated temporary directory, capacity-gates, checks ZIP CRC, required DB/config files and a privacy-preserving live/backup inventory digest, then deletes temporary data. | Merge and deploy the source branch through Gitea; retain isolated-only execution and never restore over production. | Live status is `success=1`, `reason=verified`, snapshot `29c8b7a8`, dump `6928031514`, backup/live regular repos `15/15`, inventory match `1`, CRC `1`. | +| P1-019 | DONE_LIVE_SOURCE_BRANCH_PENDING_MAIN | 95 | Gitea private bundle generation and 188 restore confidence | 188 did not have Gitea SSH key access, so generation moved to the 110 local Gitea container. The controller creates a complete 12-repo set, syncs to 188 without deletion, validates the remote manifest before symlink promotion, then runs sample clone/fsck and refreshes metrics. | Merge/deploy via Gitea and keep the 00:15 110 cron as the only generator; 188 remains verifier/exporter, not a key-bearing source client. | Live local and remote manifests are `12/12`, sync status `success=1`, sample restore metric `1`, and 188 bundle/mirror freshness metrics are green. | --- @@ -292,6 +294,17 @@ Do not run `truncate`, whole DB restore, force-push, DROP, or online root filesy ## 9. Progress Updates +```text +2026-07-10 19:40 Asia/Taipei +Phase: P0 backup/Gitea recovery truth-chain +Before: backup status used a stale hard-coded 188 denominator, generic Gitea restore evidence did not prove the full dump, the weekly Velero dry-run could be skipped permanently after downtime, and a five-minute drift guard could overwrite manually loaded Prometheus rules. +After: dynamic healthy/actual backup ratios are live; Gitea full-dump isolated restore and 110-to-188 complete private bundle flow are verified; Velero automatically backfilled the missed schedule; canonical/current Prometheus rules stay identical across the guard cycle. +Evidence: 110 `13/13`, 188 `4/4`, core blockers `0`; Gitea dump `6928031514` bytes, CRC/database/config/inventory match `1`, regular repositories `15/15`; bundles `12/12` and sample clone/fsck `1`; active rules `145`; Backup/Gitea non-inactive alerts `0`. +Additional verifier: remote bundle SHA-256 `12/12`; Prometheus source/canonical/active/runtime SHA `34c0359e01d608d6300b92ca878aed414180ec725aace3ade0a19d39cc362697`; guard runtime match `1` after controlled Prometheus-only recreate; next 20:20 cron kept the same container and returned `repaired=0`. +Blocked: no for this live Gitea/backup slice. Source still needs Gitea branch commit/main/CD convergence; full platform DR and credential escrow remain separate incomplete work items. +Next: finish source tests, commit/push the Gitea branch, then return to the ordered current Agent99 P0 without reopening this slice unless the verifier regresses. +``` + ```text 2026-06-18 14:20 Asia/Taipei Phase: P3 AI Ops runaway process automation diff --git a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md index f17e203f4..fd0bb9c1a 100644 --- a/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md +++ b/docs/workplans/2026-07-10-agent99-enterprise-ai-automation-master-plan.md @@ -1,6 +1,6 @@ # Agent99 全域 AI 自動化主計畫 -> 建立時間：2026-07-10 15:05 CST +> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-10 19:40 CST > 狀態：P0 執行中，尚未達成全域 AI 自動化 > 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json` > 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md` @@ -92,7 +92,7 @@ | 順序 | ID | 工作 | 狀態 | 驗收 | | --- | --- | --- | --- | --- | | 1 | AG99-P0-001 | 修正 `exitCode=0 => resolved` 假綠，對齊 99 runtime script 與 Gitea source SHA | **完成** | Production 99 outcome self-test 通過；runtime manifest `7/7` matched、drift `0`；operator replay 為 verified resolved，alert replay 缺 source-event resolved 時維持 verifying | -| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping 與重複事件 | **進行中，current P0** | `suggestedMode`/kind 優先於 source 字串；5 分鐘先 group；同 fingerprint 只建立一個 active incident | +| 2 | AG99-P0-002 | 修正 alert taxonomy、wrong-domain routing、bridge-before-grouping 與重複事件 | **進行中，current P0；本機 package/contract 綠，99 runtime 待部署** | `suggestedMode`/kind 優先於 source 字串；5 分鐘先 group；同 fingerprint 只建立一個 active incident | | 3 | AG99-P0-003 | 強化 ingress：relay token 必填、Telegram chat+sender allowlist、idempotency/single consumer | 待辦 | 未認證 LAN request 拒絕；producer `auto_repair`/policy 被保留；所有 request 可追 trace | | 4 | AG99-P0-004 | 新增 EvidenceCollect / RepairCandidate / MCP lane | 待辦 | 缺 evidence 的事件會建立可查 candidate，不再錯送 Perf/AwoooRepair；`INC-*` 可從收件追到 candidate | @@ -112,9 +112,9 @@ | 9 | AG99-P0-009 | 外部 down/up watcher、L0/L1 維護頁與 502 failover | 進行中 | 99/內網全斷仍能告警；edge 失聯自動顯示維護頁；恢復後撤回並驗證，不改正式 DNS 作為日常修復 | | 10 | AG99-P0-010 | 12 產品、全部網站/API 的版本與資料 freshness 矩陣 | 進行中 | 每列具 source SHA、CD、deploy marker、runtime、HTTP、功能 smoke、data timestamp；任何缺欄不宣稱最新 | | 11 | AG99-P0-011 | 全域 log source registry、AI 異常判讀與 correlation | 待辦 | host/product/site/service/package/tool logs 有 freshness、redaction、tag、cluster、primary/secondary/noise 判定 | -| 12 | AG99-P0-012 | 備份 observability、integrity、offsite 與 restore drill | 進行中 | 所有 backup domain 有 schedule/last success/freshness/restore receipt；失敗可定位且 TG 只報一次狀態變化 | +| 12 | AG99-P0-012 | 備份 observability、integrity、offsite 與 restore drill | 進行中；Gitea/Velero/110-188 slice live green | 所有 backup domain 有 schedule/last success/freshness/restore receipt；失敗可定位且 TG 只報一次狀態變化 | | 13 | AG99-P0-013 | 110/188 及全主機效能 AIOps 與安全降載 | 進行中 | CPU/memory/disk/IO/queue/DB attribution；allowlisted action 後 verifier，無空值 perf alert、無單次下降假結案 | -| 14 | AG99-P0-014 | Gitea/Harbor/Registry/Gitea-only CD 與完整 DR | 進行中 | repo/main/dev、full dump、DB/settings/issues/packages/LFS、registry、sample restore、runner/CD/runtime marker 全部可回讀 | +| 14 | AG99-P0-014 | Gitea/Harbor/Registry/Gitea-only CD 與完整 DR | 進行中；Gitea full dump/bundle restore slice live green | repo/main/dev、full dump、DB/settings/issues/packages/LFS、registry、sample restore、runner/CD/runtime marker 全部可回讀 | ### Wave 3：真正 AI Agent 與產品化 @@ -142,6 +142,14 @@ - Gitea inventory 有 12 個產品的 main/dev SSH refs；這只證明 repo/branch 存在，不證明每個產品 production 都是最新版本。 - Agent99 現況仍是規則式 SRE automation；在 planner、tool loop、mode verifier、callback、secure ingress 完成前，不得宣稱完整 AI Agent。 +### 2026-07-10 19:40 本輪已驗證增量 + +- Backup heartbeat 已以 live metric 動態分母讀回 110 `13/13`、188 `4/4`，`core_blockers=0`、`dr_warnings=0`。 +- Gitea full dump isolated restore 已驗證 6.9 GB archive、CRC、DB/config 與 15/15 regular repository inventory digest；production 未被 restore 或改寫。 +- 12/12 product private bundles 已由 110 local container 產生、no-delete 同步 188，remote SHA-256 `12/12` 與 sample clone/fsck 通過。 +- Velero missed weekly restore dry-run 已由 controller 自動 backfill 成功；20:17 Prometheus guard 修復 bind-mount inode drift 後，source/canonical/active/runtime 四方 SHA 一致、145 rules healthy，Backup/Gitea active alerts 為 0。 +- 以上只關閉 `AG99-P0-012/014` 的 Gitea/backup 子切片；全域 backup domain、Harbor/Registry/CD DR、99 Agent runtime 與 10 分鐘 reboot SLO 仍未完成。 + ## 7. 工作項強制欄位與治理 每個工作項必須先有：`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready`、`in_progress` 或 `executing` 前，還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update；缺一不可執行。 @@ -157,4 +165,4 @@ ## 8. 立即下一步 -目前唯一 P0 是 `AG99-P0-002`：以 explicit event kind / suggested mode 建立 routing precedence，將 5 分鐘 grouping、fingerprint dedupe 與 single-flight 移到 Agent99 dispatch 之前，並用 backup、provider、reboot、PostgreSQL、host、K3s replay corpus 證明不再 wrong-domain 或重複派工。完成後依序做 secure ingress、RepairCandidate，再回到 reboot state machine live rerun。 +目前唯一 P0 仍是 `AG99-P0-002`。本機 Agent99 package 已完成 routing/dedupe/card/reboot identity 契約與 synthetic replay；下一步是先完成本分支 source/test/Gitea receipt，再將測試過的 Agent99 package 原子部署到 99，重跑 backup、provider、reboot、PostgreSQL、host、K3s replay corpus 並讀回 runtime manifest。99 尚未取得可用的 no-secret authenticated management channel前，狀態只能是 local green/runtime pending；不能把 `exitCode=0` 或本機測試當 production resolved。其後依序做 secure ingress、RepairCandidate、mode verifier/callback，再回到 reboot state machine live rerun。 diff --git a/infra/ansible/playbooks/110-devops.yml b/infra/ansible/playbooks/110-devops.yml index 0ebab73a9..0d0d3f184 100644 --- a/infra/ansible/playbooks/110-devops.yml +++ b/infra/ansible/playbooks/110-devops.yml @@ -193,6 +193,30 @@ when: harbor_nginx.stat.exists and harbor_conf_check.stdout != "0" tags: nginx + # ======================================================================== + # Prometheus rule source/runtime convergence + # ======================================================================== + - name: "Monitoring | 部署 Prometheus rule drift guard" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../scripts/ops/prometheus-rule-drift-guard.sh" + dest: /home/wooo/scripts/prometheus-rule-drift-guard.sh + owner: wooo + group: wooo + mode: "0755" + tags: + - monitoring + - textfile_exporters + + - name: "Monitoring | 安裝 Prometheus rule drift guard cron" + ansible.builtin.cron: + name: "AWOOOI Prometheus rule drift guard" + user: wooo + minute: "*/5" + job: "/home/wooo/scripts/prometheus-rule-drift-guard.sh >/tmp/awoooi-prometheus-rule-drift-guard.log 2>&1" + tags: + - monitoring + - textfile_exporters + # ======================================================================== # Backup / offsite source-of-truth # ======================================================================== @@ -217,11 +241,15 @@ - backup-all.sh - backup-status.sh - backup-gitea.sh + - gitea-repo-bundle-backup.sh + - gitea-bundle-backup-sync-188.sh - backup-harbor.sh - backup-momo.sh - backup-awoooi.sh - backup-awoooi-frequent.sh - backup-configs.sh + - check-backup-integrity.sh + - verify-gitea-backup.sh - sync-offsite-backups.sh - offsite-escrow-evidence-report.sh - verify-offsite-full-sync.sh @@ -236,6 +264,18 @@ job: "/backup/scripts/backup-status.sh >/tmp/awoooi-backup-status.cron.log 2>&1" tags: backup_jobs + - name: "Backup | 安裝 Gitea bundle 每日本機產生、188 同步與還原驗證 cron" + ansible.builtin.cron: + name: "AWOOOI Gitea bundle daily backup sync and restore verifier" + user: wooo + minute: "15" + hour: "0" + job: >- + PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + /backup/scripts/gitea-bundle-backup-sync-188.sh + >> /backup/logs/gitea-bundle-sync-188.log 2>&1 + tags: backup_jobs + - name: "Backup | 安裝 offsite escrow evidence report cron" ansible.builtin.cron: name: "AWOOOI offsite escrow evidence report" diff --git a/infra/ansible/playbooks/188-ai-web.yml b/infra/ansible/playbooks/188-ai-web.yml index 53f712025..1091574b8 100644 --- a/infra/ansible/playbooks/188-ai-web.yml +++ b/infra/ansible/playbooks/188-ai-web.yml @@ -102,7 +102,9 @@ mode: "0755" loop: - /home/ollama/bin + - /home/ollama/scripts - /home/ollama/momo_backups + - /home/ollama/backup/110/gitea/git-bundles tags: backup_jobs - name: "Backup | 安裝 AwoooP ops 通知 helper" @@ -123,6 +125,20 @@ mode: "0755" tags: backup_jobs + - name: "Backup | 安裝 Gitea bundle 隔離還原與 exporter" + ansible.builtin.copy: + src: "{{ playbook_dir }}/../../../scripts/{{ item.src }}" + dest: "/home/ollama/scripts/{{ item.dest }}" + owner: ollama + group: ollama + mode: "0755" + loop: + - src: backup/gitea-bundle-sample-restore-dry-run.sh + dest: gitea-bundle-sample-restore-dry-run.sh + - src: ops/backup-health-textfile-exporter.py + dest: backup-health-textfile-exporter.py + tags: backup_jobs + - name: "Backup | 讀取既有 momo PostgreSQL crontab" ansible.builtin.command: cmd: "crontab -l -u ollama" diff --git a/k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml b/k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml index 5a5720384..06eb47feb 100644 --- a/k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml +++ b/k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml @@ -26,7 +26,9 @@ spec: concurrencyPolicy: Forbid successfulJobsHistoryLimit: 3 failedJobsHistoryLimit: 5 - startingDeadlineSeconds: 600 + # Recover one missed weekly dry-run after a host/cluster outage instead of + # permanently skipping the week's restore evidence. + startingDeadlineSeconds: 604800 jobTemplate: spec: backoffLimit: 0 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index a788d864f..12c43984d 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1591,11 +1591,12 @@ groups: team: ops alert_category: infrastructure notification_type: TYPE-3 - auto_repair: "false" + auto_repair: "true" + agent99_mode: BackupCheck annotations: summary: "188 Gitea private repo bundle 備份不完整" description: "Gitea private bundle manifest、bundle、checksum 或 freshness 有缺口；public repo search 不能代表完整備份。" - runbook: "先讀 awoooi_gitea_bundle_* 指標找缺少的 repo，執行 scripts/backup/gitea-repo-bundle-backup.sh 產生 bundle 並同步到 188；不得刪 repo、不得 restore 到 production、不得讀 token。" + runbook: "先讀 awoooi_gitea_bundle_* 聚合指標，再由 Agent99 路由到 110 執行 /backup/scripts/gitea-bundle-backup-sync-188.sh；全數 bundle/checksum/remote manifest/sample fsck 成功才 promote complete link，不得刪 repo、restore 到 production 或讀 token。" - alert: GiteaPrivateBundleRestoreDryRunMissing expr: absent(awoooi_gitea_bundle_sample_restore_dry_run_ok{host="188"}) or awoooi_gitea_bundle_sample_restore_dry_run_ok{host="188"} == 0 @@ -1608,7 +1609,8 @@ groups: team: ops alert_category: infrastructure notification_type: TYPE-3 - auto_repair: "false" + auto_repair: "true" + agent99_mode: BackupCheck annotations: summary: "188 Gitea private repo bundle 還原 dry-run 未通過" description: "Gitea bundle 備份需要 sample restore dry-run 證明可還原；bundle 存在不等於 restore confidence。" @@ -1665,7 +1667,19 @@ groups: runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188，刷新 /home/ollama/node_exporter_textfiles/backup_health.prom；不得送測試 Telegram、不得讀 token、不得 restore 到 production。" - alert: BackupAlertReceiptStageMissing - expr: sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0 + expr: | + max by (alertname, host) ( + ALERTS{ + alertstate="firing", + alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*", + alertname!="BackupAlertReceiptStageMissing" + } + ) + unless on (alertname, host) + label_replace( + max by (required_alert, host) (awoooi_backup_alert_receipt_fresh == 1), + "alertname", "$1", "required_alert", "(.*)" + ) for: 30m labels: severity: warning @@ -1676,9 +1690,9 @@ groups: notification_type: TYPE-1 auto_repair: "false" annotations: - summary: "{{ $labels.host }} 備份告警 receipt 缺少階段證據" - description: "{{ $labels.host }} / channel={{ $labels.receipt_channel }} 目前有 {{ $value }} 個 backup alert receipt stage 缺少新鮮 marker；這代表 Telegram/AwoooP sent/received/dedup/escalation 鏈尚未被證明，不能宣稱備份告警閉環。" - runbook: "先查 `awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",receipt_channel=\"{{ $labels.receipt_channel }}\"} == 0` 取得 required_alert/scope/stage/required_notification_type 明細；標準 receipt_channel 為 telegram_awooop，再讀 /backup/alert-receipts/*.last_success 或 AwoooP outbound receipt readback；補脫敏 evidence marker 時只允許 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" + summary: "{{ $labels.host }} 正在發生的備份告警尚未完成 receipt 閉環" + description: "{{ $labels.alertname }} 正在 firing，但尚無新鮮的 Telegram/AwoooP sent/received/dedup/escalation 完整 receipt；未 firing 的規則不再產生假告警。" + runbook: "查 awoooi_backup_alert_receipt_stage_fresh{host=\"{{ $labels.host }}\",required_alert=\"{{ $labels.alertname }}\"} 取得缺少階段；只補 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" - alert: BackupAggregateRunFailed expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0 @@ -1765,6 +1779,24 @@ groups: description: "每月 restore drill 沒有成功證據，備份雖可能新鮮，但尚未驗證可讀取還原。" runbook: "SSH 110 執行 `/backup/scripts/check-backup-integrity.sh --mode restore-drill`；只允許還原到隔離暫存目錄，不得覆蓋 production" + - alert: GiteaRestoreDrillMissingOrFailed + expr: absent(awoooi_gitea_restore_drill_fresh{host="110"}) or awoooi_gitea_restore_drill_fresh{host="110"} == 0 + for: 15m + labels: + severity: critical + layer: host-backup + component: gitea-restore-drill + host: "110" + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + agent99_mode: BackupCheck + annotations: + summary: "Gitea 隔離還原演練缺失、過期或失敗" + description: "Gitea 最新 dump 尚未同時通過 ZIP CRC、DB/config 必要資產、repo count 與隱私保護 inventory 集合比對；不能只用 Gitea HTTP 200 或 Restic pack check 宣稱可恢復。" + runbook: "Agent99/110 執行 `/backup/scripts/verify-gitea-backup.sh --no-notify`；只在隔離暫存區驗證，不停止 Gitea、不覆蓋 production、不輸出 repo 名稱或 secrets。" + - alert: BackupOffsiteCopyNotConfigured expr: sum by(host) (awoooi_backup_offsite_configured{host="110"}) == 0 for: 1m diff --git a/scripts/backup/backup-status.sh b/scripts/backup/backup-status.sh index 16c8d47b8..afa77a195 100644 --- a/scripts/backup/backup-status.sh +++ b/scripts/backup/backup-status.sh @@ -221,8 +221,10 @@ configured_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_conf configured_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_configured" 0)" script_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_script_present" 0)" script_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_script_present" 0)" -fresh_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")" -fresh_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")" +monitored_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")" +monitored_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")" +fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)" +fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)" stale_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 0)" stale_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 0)" failed_total_110="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_last_run_failed_count")" @@ -236,7 +238,12 @@ offsite_rclone_configured="$(awk '/^awoooi_backup_offsite_configured\{.*provider offsite_rclone_fresh="$(awk '/^awoooi_backup_offsite_fresh\{.*provider="rclone"/ { print $2; found=1; exit } END { if (!found) print 0 }' "${TEXTFILE_110}" 2>/dev/null || echo 0)" escrow_missing="$(metric_first "${TEXTFILE_110}" "awoooi_backup_dr_credential_escrow_missing_count")" -core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110)) +freshness_metrics_missing_110=0 +freshness_metrics_missing_188=0 +[ "${monitored_total_110}" -gt 0 ] || freshness_metrics_missing_110=1 +[ "${monitored_total_188}" -gt 0 ] || freshness_metrics_missing_188=1 + +core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + freshness_metrics_missing_110 + freshness_metrics_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110)) dr_warnings=0 if [ "${offsite_configured%.*}" -lt 1 ] 2>/dev/null; then dr_warnings=$((dr_warnings + 1)) @@ -265,7 +272,7 @@ missing_scripts_188="$(label_list_for_zero "${TEXTFILE_188_TMP}" "awoooi_backup_ backup_all_ts="$(metric_value_for_label "${TEXTFILE_110}" "awoooi_backup_job_last_success_timestamp" "job" "backup_all")" last_backup_all="$(human_timestamp "${backup_all_ts}")" -message="${headline}; 110備份=${fresh_total_110}/13 fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_total_188}/2 fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}" +message="${headline}; 110備份=${fresh_ok_110}/${monitored_total_110} fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_ok_188}/${monitored_total_188} fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; freshness_metrics_missing=${freshness_metrics_missing_110}/${freshness_metrics_missing_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}" if [ "${core_blockers}" -gt 0 ]; then message="${message}; stale110=${stale_jobs_110:-none}; stale188=${stale_jobs_188:-none}; missing_script110=${missing_scripts_110:-none}; missing_script188=${missing_scripts_188:-none}" @@ -273,7 +280,7 @@ fi { log_line "${message}" - log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188}" + log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188} freshness_metrics_missing_110=${freshness_metrics_missing_110} freshness_metrics_missing_188=${freshness_metrics_missing_188}" } | tee -a "${LOG_DIR}/backup-status.log" if [ "${NOTIFY}" -eq 1 ]; then @@ -285,7 +292,7 @@ if [ "${NOTIFY}" -eq 1 ]; then failed_interval_hours="${BACKUP_STATUS_FAILED_INTERVAL_HOURS:-6}" now_ts="$(date +%s)" notify_fingerprint="$( - printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \ + printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};fm110=${freshness_metrics_missing_110};fm188=${freshness_metrics_missing_188};fresh110=${fresh_ok_110}/${monitored_total_110};fresh188=${fresh_ok_188}/${monitored_total_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \ | cksum \ | awk '{print $1}' )" diff --git a/scripts/backup/check-backup-integrity.sh b/scripts/backup/check-backup-integrity.sh index 5bd95f71d..8f5a0a267 100755 --- a/scripts/backup/check-backup-integrity.sh +++ b/scripts/backup/check-backup-integrity.sh @@ -190,6 +190,18 @@ main() { failed=$((failed + 1)) fi else + if [ "${name}" = "gitea" ] && [ -x "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" ]; then + log_info "Gitea specialized restore drill: archive CRC, required assets, and live inventory match" + if "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" --no-notify >> "${LOG_FILE}" 2>&1; then + log_success "repo ${name} specialized restore drill OK" + echo "repo=${name} status=restore_drill_specialized_ok snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}" + else + log_error "repo ${name} specialized restore drill failed" + echo "repo=${name} status=restore_drill_specialized_failed snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}" + failed=$((failed + 1)) + fi + continue + fi local sample local sample_out sample=$(MAX_SAMPLE_BYTES="${MAX_SAMPLE_BYTES}" sample_path_for_repo "${repo}") diff --git a/scripts/backup/gitea-bundle-backup-sync-188.sh b/scripts/backup/gitea-bundle-backup-sync-188.sh new file mode 100755 index 000000000..b7043b51d --- /dev/null +++ b/scripts/backup/gitea-bundle-backup-sync-188.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash + +# Generate Gitea Git bundles on host 110 from the local Gitea container, sync +# the complete set to 188 without deletion, then run an isolated sample restore. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +LOCAL_OUTPUT_ROOT="${GITEA_BUNDLE_LOCAL_ROOT:-/backup/gitea-bundles}" +LOCAL_CONTAINER="${GITEA_BUNDLE_LOCAL_CONTAINER:-gitea}" +REMOTE_HOST="${GITEA_BUNDLE_REMOTE_HOST:-ollama@192.168.0.188}" +REMOTE_OUTPUT_ROOT="${GITEA_BUNDLE_REMOTE_ROOT:-/home/ollama/backup/110/gitea/git-bundles}" +REMOTE_VERIFY_SCRIPT="${GITEA_BUNDLE_REMOTE_VERIFY_SCRIPT:-/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh}" +REMOTE_EXPORTER_SCRIPT="${GITEA_BUNDLE_REMOTE_EXPORTER_SCRIPT:-/home/ollama/scripts/backup-health-textfile-exporter.py}" +BUNDLE_SCRIPT="${GITEA_BUNDLE_SCRIPT:-${SCRIPT_DIR}/gitea-repo-bundle-backup.sh}" +STATUS_FILE="${GITEA_BUNDLE_SYNC_STATUS_FILE:-/backup/status/gitea-bundle-sync-188.status}" +LOCK_FILE="${GITEA_BUNDLE_SYNC_LOCK_FILE:-${LOCAL_OUTPUT_ROOT}/.sync-188.lock}" +START_TIME="$(date +%s)" +SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new) +phase="preflight" +promoted=0 +previous_complete="" +previous_latest="" +expected_count=0 +stamp="unknown" + +mkdir -p "${LOCAL_OUTPUT_ROOT}" "$(dirname "${STATUS_FILE}")" +exec 9>"${LOCK_FILE}" +if ! flock -n 9; then + printf 'SKIP Gitea bundle sync already running\n' + exit 0 +fi + +write_status() { + local success="$1" + local reason="$2" + local temp_status="${STATUS_FILE}.tmp.$$" + { + printf 'timestamp=%s\n' "$(date +%s)" + printf 'success=%s\n' "${success}" + printf 'reason=%s\n' "${reason}" + printf 'phase=%s\n' "${phase}" + printf 'snapshot_stamp=%s\n' "${stamp}" + printf 'expected_repo_count=%s\n' "${expected_count}" + printf 'duration_seconds=%s\n' "$(($(date +%s) - START_TIME))" + } >"${temp_status}" + chmod 600 "${temp_status}" + mv "${temp_status}" "${STATUS_FILE}" +} + +refresh_remote_exporter() { + ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'" \ + >/dev/null 2>&1 || true +} + +rollback_remote_links() { + ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + python3 - "${REMOTE_OUTPUT_ROOT}" "${previous_complete}" "${previous_latest}" <<'PY' +import os +import sys +from pathlib import Path + +root = Path(sys.argv[1]) +previous = { + root / "latest-private-complete": sys.argv[2], + root / "latest": sys.argv[3], +} +for link, target in previous.items(): + if target: + temporary = link.with_name(f".{link.name}.rollback.{os.getpid()}") + temporary.unlink(missing_ok=True) + temporary.symlink_to(target) + os.replace(temporary, link) + elif link.is_symlink(): + link.unlink() +PY +} + +finish() { + local rc=$? + trap - EXIT + if [ "${rc}" -eq 0 ] && [ "${phase}" = "complete" ]; then + write_status 1 verified + else + if [ "${promoted}" -eq 1 ]; then + rollback_remote_links >/dev/null 2>&1 || true + fi + refresh_remote_exporter + write_status 0 "${phase}_failed" + fi + exit "${rc}" +} +trap finish EXIT + +[ -x "${BUNDLE_SCRIPT}" ] || { printf 'missing bundle script: %s\n' "${BUNDLE_SCRIPT}" >&2; exit 1; } +command -v docker >/dev/null +command -v rsync >/dev/null +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "test -x '${REMOTE_VERIFY_SCRIPT}' && test -x '${REMOTE_EXPORTER_SCRIPT}'" + +phase="bundle_local_container" +"${BUNDLE_SCRIPT}" \ + --local-container "${LOCAL_CONTAINER}" \ + --output-root "${LOCAL_OUTPUT_ROOT}" + +local_snapshot="$(readlink -f "${LOCAL_OUTPUT_ROOT}/latest-private-complete")" +stamp="$(basename "${local_snapshot}")" +manifest="${local_snapshot}/manifest.tsv" +[ -f "${manifest}" ] +expected_count="$(awk -F '\t' 'NR > 1 {total += 1; if ($2 != "ok") failed += 1} END {if (failed > 0 || total == 0) exit 1; print total}' "${manifest}")" + +phase="sync_complete_set_to_188" +previous_complete="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest-private-complete' 2>/dev/null || true")" +previous_latest="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest' 2>/dev/null || true")" +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "mkdir -p '${REMOTE_OUTPUT_ROOT}/${stamp}'" +rsync -a --checksum \ + -e "ssh ${SSH_OPTS[*]}" \ + "${local_snapshot}/" \ + "${REMOTE_HOST}:${REMOTE_OUTPUT_ROOT}/${stamp}/" + +phase="remote_manifest_verification" +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + python3 - "${REMOTE_OUTPUT_ROOT}/${stamp}" "${expected_count}" <<'PY' +import csv +import hashlib +import sys +from pathlib import Path + +snapshot = Path(sys.argv[1]) +expected = int(sys.argv[2]) +with (snapshot / "manifest.tsv").open(encoding="utf-8", newline="") as handle: + rows = list(csv.DictReader(handle, delimiter="\t")) +ok = len(rows) == expected and expected > 0 +verified_checksums = 0 +for row in rows: + bundle = snapshot / Path(row.get("bundle", "")).name + checksum = snapshot / Path(row.get("checksum", "")).name + row_ok = row.get("status") == "ok" and bundle.is_file() and checksum.is_file() + if row_ok: + expected_digest = checksum.read_text(encoding="utf-8", errors="replace").split()[0].lower() + digest = hashlib.sha256() + with bundle.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + row_ok = len(expected_digest) == 64 and digest.hexdigest() == expected_digest + if row_ok: + verified_checksums += 1 + ok = ok and row_ok +print( + f"remote_manifest_expected={expected} remote_manifest_rows={len(rows)} " + f"remote_manifest_checksum_ok={verified_checksums} remote_manifest_ok={int(ok)}" +) +raise SystemExit(0 if ok else 1) +PY + +phase="remote_promote_complete" +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "for link in '${REMOTE_OUTPUT_ROOT}/latest-private-complete' '${REMOTE_OUTPUT_ROOT}/latest'; do test ! -e \"\$link\" || test -L \"\$link\" || exit 1; done" +promoted=1 +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest-private-complete' && ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest'" + +phase="remote_sample_restore_fsck" +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "AIOPS_GITEA_BUNDLE_ROOT='${REMOTE_OUTPUT_ROOT}/latest-private-complete' AIOPS_HOST_LABEL=188 '${REMOTE_VERIFY_SCRIPT}' --write-textfile" + +phase="refresh_remote_backup_health" +ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \ + "AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'" + +phase="complete" +printf 'GITEA_BUNDLE_SYNC_COMPLETE=1 expected=%s evidence=%s\n' "${expected_count}" "${STATUS_FILE}" diff --git a/scripts/backup/gitea-repo-bundle-backup.sh b/scripts/backup/gitea-repo-bundle-backup.sh index b27f0fbfd..aadb389b5 100755 --- a/scripts/backup/gitea-repo-bundle-backup.sh +++ b/scripts/backup/gitea-repo-bundle-backup.sh @@ -7,6 +7,8 @@ set -euo pipefail GITEA_BASE_URL="${GITEA_BASE_URL:-http://192.168.0.110:3001}" GIT_REMOTE_BASE="${GIT_REMOTE_BASE:-}" +LOCAL_GITEA_CONTAINER="${LOCAL_GITEA_CONTAINER:-}" +LOCAL_REPO_ROOT="${LOCAL_REPO_ROOT:-/data/git/repositories}" OUTPUT_ROOT="${OUTPUT_ROOT:-/home/ollama/backup/110/gitea/git-bundles}" TIMEOUT_LS_REMOTE_SECONDS="${TIMEOUT_LS_REMOTE_SECONDS:-60}" TIMEOUT_CLONE_SECONDS="${TIMEOUT_CLONE_SECONDS:-180}" @@ -27,6 +29,11 @@ Options: --git-remote-base URL Git remote base, such as ssh://git@192.168.0.110:2222. When set, it is used instead of --gitea-base. + --local-container NAME + Read bare repos directly from a local Gitea container. + This avoids tokens and SSH keys on the Gitea host. + --local-repo-root PATH + Repo root inside the local container. --stamp STAMP Output directory stamp. Default: current local timestamp. --dry-run Print target URLs without cloning or writing bundles. -h, --help Show this help. @@ -56,6 +63,14 @@ while [ "$#" -gt 0 ]; do shift GIT_REMOTE_BASE="${1:?--git-remote-base requires URL}" ;; + --local-container) + shift + LOCAL_GITEA_CONTAINER="${1:?--local-container requires NAME}" + ;; + --local-repo-root) + shift + LOCAL_REPO_ROOT="${1:?--local-repo-root requires PATH}" + ;; --stamp) shift STAMP="${1:?--stamp requires value}" @@ -104,7 +119,11 @@ repo_remote_url() { if [ "$DRY_RUN" -eq 1 ]; then for repo in "${REPOS[@]}"; do - printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")" + if [ -n "$LOCAL_GITEA_CONTAINER" ]; then + printf 'DRY_RUN repo=%s transport=container container=%s\n' "$repo" "$LOCAL_GITEA_CONTAINER" + else + printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")" + fi done exit 0 fi @@ -116,19 +135,80 @@ mkdir -p "$backup_root" trap 'rm -rf "$tmp_root"' EXIT printf 'repo\tstatus\thead_count\tbundle\tchecksum\n' > "$manifest" +failed_count=0 +success_count=0 for repo in "${REPOS[@]}"; do slug="${repo//\//__}" - url="$(repo_remote_url "$repo")" refs_file="${backup_root}/${slug}.refs" err_file="${backup_root}/${slug}.err" bundle_file="${backup_root}/${slug}.bundle" checksum_file="${bundle_file}.sha256" repo_tmp="${tmp_root}/${slug}.git" + if [ -n "$LOCAL_GITEA_CONTAINER" ]; then + container_repo="${LOCAL_REPO_ROOT%/}/${repo}.git" + container_bundle="/tmp/awoooi-${slug}-${STAMP}.bundle" + if ! docker exec -u git "$LOCAL_GITEA_CONTAINER" test -d "$container_repo" >/dev/null 2>&1; then + owner="${repo%%/*}" + name="${repo#*/}" + resolved_paths="$(docker exec -u git "$LOCAL_GITEA_CONTAINER" sh -c \ + 'find "$1/$2" -mindepth 1 -maxdepth 1 -type d -iname "$3.git" -print' \ + sh "${LOCAL_REPO_ROOT%/}" "$owner" "$name" 2>>"$err_file" || true)" + resolved_count="$(printf '%s\n' "$resolved_paths" | awk 'NF {count += 1} END {print count + 0}')" + if [ "$resolved_count" -ne 1 ]; then + printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest" + failed_count=$((failed_count + 1)) + continue + fi + container_repo="$resolved_paths" + fi + if ! timeout "$TIMEOUT_LS_REMOTE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \ + git -C "$container_repo" for-each-ref --format='%(objectname)%09%(refname)' \ + >"$refs_file" 2>"$err_file"; then + printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest" + failed_count=$((failed_count + 1)) + continue + fi + + head_count="$(wc -l < "$refs_file" | tr -d ' ')" + if ! timeout "$TIMEOUT_BUNDLE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \ + git -C "$container_repo" bundle create "$container_bundle" --all \ + >/dev/null 2>>"$err_file"; then + docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true + printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + failed_count=$((failed_count + 1)) + continue + fi + + if ! timeout "$TIMEOUT_VERIFY_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \ + git -C "$container_repo" bundle verify "$container_bundle" \ + >/dev/null 2>>"$err_file"; then + docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true + printf '%s\tverify_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + failed_count=$((failed_count + 1)) + continue + fi + + if ! docker cp "${LOCAL_GITEA_CONTAINER}:${container_bundle}" "$bundle_file" 2>>"$err_file"; then + docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true + printf '%s\tcopy_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + failed_count=$((failed_count + 1)) + continue + fi + docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true + (sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file" + printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest" + success_count=$((success_count + 1)) + continue + fi + + url="$(repo_remote_url "$repo")" + if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_LS_REMOTE_SECONDS" \ git ls-remote --heads "$url" >"$refs_file" 2>"$err_file"; then printf '%s\tls_remote_failed\t0\t\t\n' "$repo" >> "$manifest" + failed_count=$((failed_count + 1)) continue fi @@ -137,30 +217,52 @@ for repo in "${REPOS[@]}"; do if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_CLONE_SECONDS" \ git clone --mirror --quiet "$url" "$repo_tmp" 2>>"$err_file"; then printf '%s\tclone_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + failed_count=$((failed_count + 1)) continue fi if ! timeout "$TIMEOUT_BUNDLE_SECONDS" \ git -C "$repo_tmp" bundle create "$bundle_file" --all >/dev/null 2>>"$err_file"; then printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest" + failed_count=$((failed_count + 1)) continue fi if ! timeout "$TIMEOUT_VERIFY_SECONDS" \ git -C "$repo_tmp" bundle verify "$bundle_file" >/dev/null 2>>"$err_file"; then printf '%s\tverify_failed\t%s\t%s\t\n' "$repo" "$head_count" "$bundle_file" >> "$manifest" + failed_count=$((failed_count + 1)) continue fi (sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file" rm -rf "$repo_tmp" printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest" + success_count=$((success_count + 1)) done -latest_link="${OUTPUT_ROOT%/}/latest" -if [ ! -e "$latest_link" ] || [ -L "$latest_link" ]; then - ln -sfn "$backup_root" "$latest_link" -else - echo "WARN latest path exists and is not a symlink: $latest_link" >&2 -fi +expected_count="${#REPOS[@]}" cat "$manifest" + +if [ "$failed_count" -ne 0 ] || [ "$success_count" -ne "$expected_count" ]; then + printf 'ERROR bundle backup incomplete success=%s expected=%s failed=%s; complete links unchanged\n' \ + "$success_count" "$expected_count" "$failed_count" >&2 + exit 1 +fi + +promotion_path_ready() { + link="$1" + if [ ! -e "$link" ] || [ -L "$link" ]; then + return 0 + fi + printf 'ERROR promotion path exists and is not a symlink: %s\n' "$link" >&2 + return 1 +} + +latest_link="${OUTPUT_ROOT%/}/latest" +complete_link="${OUTPUT_ROOT%/}/latest-private-complete" +promotion_path_ready "$latest_link" +promotion_path_ready "$complete_link" +ln -sfn "$backup_root" "$latest_link" +ln -sfn "$backup_root" "$complete_link" +printf 'BUNDLE_BACKUP_COMPLETE=1 success=%s expected=%s root=%s\n' "$success_count" "$expected_count" "$backup_root" diff --git a/scripts/backup/tests/test_backup_status_contract.py b/scripts/backup/tests/test_backup_status_contract.py index 035b643a6..0ffffd38a 100644 --- a/scripts/backup/tests/test_backup_status_contract.py +++ b/scripts/backup/tests/test_backup_status_contract.py @@ -19,3 +19,17 @@ def test_backup_status_keeps_aggregate_failure_out_of_core_blockers() -> None: assert "component_failed_188" in core_line assert "failed_total_110" not in core_line assert "failed_total_188" not in core_line + + +def test_backup_status_uses_dynamic_healthy_over_monitored_ratio() -> None: + text = BACKUP_STATUS.read_text(encoding="utf-8") + core_line = next(line for line in text.splitlines() if line.startswith("core_blockers=")) + + assert 'fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)"' in text + assert 'fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)"' in text + assert "110備份=${fresh_ok_110}/${monitored_total_110}" in text + assert "188備份=${fresh_ok_188}/${monitored_total_188}" in text + assert "freshness_metrics_missing_110" in core_line + assert "freshness_metrics_missing_188" in core_line + assert "/13 fresh" not in text + assert "/2 fresh" not in text diff --git a/scripts/backup/tests/test_gitea_backup_automation.py b/scripts/backup/tests/test_gitea_backup_automation.py new file mode 100644 index 000000000..c59d75fd5 --- /dev/null +++ b/scripts/backup/tests/test_gitea_backup_automation.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import os +import shutil +import subprocess +import textwrap +import zipfile +from pathlib import Path + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +BUNDLE_SCRIPT = ROOT / "scripts" / "backup" / "gitea-repo-bundle-backup.sh" +RESTORE_DRILL_SCRIPT = ROOT / "scripts" / "backup" / "verify-gitea-backup.sh" + + +def _fake_git(tmp_path: Path) -> Path: + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + git = fake_bin / "git" + git.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env bash + set -euo pipefail + if [ -n "${FAKE_GIT_FAIL_PATTERN:-}" ] && [[ "$*" == *"${FAKE_GIT_FAIL_PATTERN}"* ]]; then + exit 42 + fi + if [ "$1" = "ls-remote" ]; then + printf '0000000000000000000000000000000000000001\trefs/heads/main\n' + exit 0 + fi + if [ "$1" = "clone" ]; then + mkdir -p "${!#}" + exit 0 + fi + if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "create" ]; then + printf 'synthetic bundle\n' > "$5" + exit 0 + fi + if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "verify" ]; then + exit 0 + fi + printf 'unexpected fake git invocation: %s\n' "$*" >&2 + exit 64 + """ + ), + encoding="utf-8", + ) + git.chmod(0o755) + return fake_bin + + +def _fake_docker(fake_bin: Path, tmp_path: Path) -> None: + docker = fake_bin / "docker" + docker.write_text( + textwrap.dedent( + """\ + #!/usr/bin/env bash + set -euo pipefail + root="${FAKE_DOCKER_ROOT:?}" + if [ "$1" = "exec" ]; then + shift 4 + if [ "$1" = "test" ]; then + exit 0 + fi + if [ "$1" = "git" ] && [ "$4" = "for-each-ref" ]; then + printf '0000000000000000000000000000000000000001\trefs/heads/main\n' + exit 0 + fi + if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "create" ]; then + mkdir -p "${root}$(dirname "$6")" + printf 'synthetic container bundle\n' > "${root}$6" + exit 0 + fi + if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "verify" ]; then + exit 0 + fi + if [ "$1" = "rm" ]; then + rm -f "${root}${3}" + exit 0 + fi + fi + if [ "$1" = "cp" ]; then + source_path="${2#*:}" + cp "${root}${source_path}" "$3" + exit 0 + fi + printf 'unexpected fake docker invocation: %s\n' "$*" >&2 + exit 64 + """ + ), + encoding="utf-8", + ) + docker.chmod(0o755) + + +def test_bundle_backup_promotes_only_a_complete_repo_set(tmp_path: Path) -> None: + fake_bin = _fake_git(tmp_path) + output_root = tmp_path / "bundles" + env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"} + + complete = subprocess.run( + [ + "bash", + str(BUNDLE_SCRIPT), + "--output-root", + str(output_root), + "--git-remote-base", + "ssh://git@example.invalid:2222", + "--stamp", + "complete", + "--repo", + "owner/one", + "--repo", + "owner/two", + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert complete.returncode == 0, complete.stderr + assert (output_root / "latest-private-complete").resolve() == output_root / "complete" + assert (output_root / "latest").resolve() == output_root / "complete" + assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in complete.stdout + + failed = subprocess.run( + [ + "bash", + str(BUNDLE_SCRIPT), + "--output-root", + str(output_root), + "--git-remote-base", + "ssh://git@example.invalid:2222", + "--stamp", + "partial", + "--repo", + "owner/one", + "--repo", + "owner/fail", + ], + env=env | {"FAKE_GIT_FAIL_PATTERN": "owner/fail"}, + capture_output=True, + text=True, + check=False, + ) + assert failed.returncode != 0 + assert "complete links unchanged" in failed.stderr + assert (output_root / "latest-private-complete").resolve() == output_root / "complete" + assert "owner/fail\tls_remote_failed" in (output_root / "partial" / "manifest.tsv").read_text( + encoding="utf-8" + ) + + +def test_bundle_backup_supports_local_gitea_container_without_credentials(tmp_path: Path) -> None: + fake_bin = _fake_git(tmp_path) + _fake_docker(fake_bin, tmp_path) + output_root = tmp_path / "local-bundles" + docker_root = tmp_path / "fake-docker" + docker_root.mkdir() + env = os.environ | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "FAKE_DOCKER_ROOT": str(docker_root), + } + + result = subprocess.run( + [ + "bash", + str(BUNDLE_SCRIPT), + "--output-root", + str(output_root), + "--local-container", + "gitea", + "--stamp", + "local-complete", + "--repo", + "owner/one", + "--repo", + "owner/two", + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode == 0, result.stderr + assert (output_root / "latest-private-complete").resolve() == output_root / "local-complete" + assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in result.stdout + + +def test_bundle_backup_preflights_both_promotion_links(tmp_path: Path) -> None: + fake_bin = _fake_git(tmp_path) + output_root = tmp_path / "blocked-promotion" + output_root.mkdir() + (output_root / "latest-private-complete").mkdir() + env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"} + + result = subprocess.run( + [ + "bash", + str(BUNDLE_SCRIPT), + "--output-root", + str(output_root), + "--git-remote-base", + "ssh://git@example.invalid:2222", + "--stamp", + "candidate", + "--repo", + "owner/one", + ], + env=env, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + assert "promotion path exists and is not a symlink" in result.stderr + assert not (output_root / "latest").exists() + assert (output_root / "latest-private-complete").is_dir() + + +@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier") +def test_gitea_restore_drill_blocks_same_count_inventory_mismatch(tmp_path: Path) -> None: + archive = tmp_path / "gitea-dump.zip" + with zipfile.ZipFile(archive, "w") as payload: + payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n") + payload.writestr("gitea-repo/owner/two.git/HEAD", "ref: refs/heads/main\n") + payload.writestr("gitea-db.sql", "synthetic database placeholder\n") + payload.writestr("config/app.ini", "synthetic config placeholder\n") + + state_dir = tmp_path / "state" + work_dir = tmp_path / "work" + state_dir.mkdir() + work_dir.mkdir() + matching_inventory = tmp_path / "matching.txt" + matching_inventory.write_text("owner/one.git\nowner/two.git\n", encoding="utf-8") + mismatch_inventory = tmp_path / "mismatch.txt" + mismatch_inventory.write_text("owner/one.git\nowner/different.git\n", encoding="utf-8") + base_env = os.environ | { + "GITEA_RESTORE_DRILL_TEST_MODE": "1", + "GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive), + "GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir), + "GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir), + "BACKUP_BASE": str(tmp_path), + } + + success = subprocess.run( + ["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"], + env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(matching_inventory)}, + capture_output=True, + text=True, + check=False, + ) + assert success.returncode == 0, success.stderr + status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8") + assert "success=1" in status + assert "inventory_digest_match=1" in status + + mismatch = subprocess.run( + ["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"], + env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(mismatch_inventory)}, + capture_output=True, + text=True, + check=False, + ) + assert mismatch.returncode != 0 + status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8") + assert "success=0" in status + assert "reason=repository_inventory_mismatch" in status + + +@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier") +def test_gitea_restore_drill_records_live_inventory_read_failure(tmp_path: Path) -> None: + archive = tmp_path / "gitea-dump.zip" + with zipfile.ZipFile(archive, "w") as payload: + payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n") + payload.writestr("gitea-db.sql", "synthetic database placeholder\n") + payload.writestr("config/app.ini", "synthetic config placeholder\n") + + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + docker = fake_bin / "docker" + docker.write_text("#!/usr/bin/env bash\nexit 42\n", encoding="utf-8") + docker.chmod(0o755) + state_dir = tmp_path / "state" + work_dir = tmp_path / "work" + state_dir.mkdir() + work_dir.mkdir() + result = subprocess.run( + ["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"], + env=os.environ + | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "GITEA_RESTORE_DRILL_TEST_MODE": "1", + "GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive), + "GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir), + "GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir), + "BACKUP_BASE": str(tmp_path), + }, + capture_output=True, + text=True, + check=False, + ) + assert result.returncode != 0 + status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8") + assert "success=0" in status + assert "reason=live_inventory_read_failed" in status diff --git a/scripts/backup/verify-gitea-backup.sh b/scripts/backup/verify-gitea-backup.sh new file mode 100755 index 000000000..63a4309b1 --- /dev/null +++ b/scripts/backup/verify-gitea-backup.sh @@ -0,0 +1,288 @@ +#!/bin/bash + +# Isolated Gitea restore drill. It validates the latest dump without touching +# the running container or exposing repository names, database rows, or config. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEST_MODE="${GITEA_RESTORE_DRILL_TEST_MODE:-0}" +CALLER_BACKUP_BASE="${BACKUP_BASE:-}" +CALLER_BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-}" +CALLER_RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-}" +if [ "${TEST_MODE}" != "1" ] && [ -f "${SCRIPT_DIR}/common.sh" ]; then + # shellcheck disable=SC1091 + source "${SCRIPT_DIR}/common.sh" +else + BACKUP_BASE="${BACKUP_BASE:-/backup}" + RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/nonexistent}" + log_info() { printf '[INFO] %s\n' "$*"; } + log_success() { printf '[SUCCESS] %s\n' "$*"; } + log_error() { printf '[ERROR] %s\n' "$*" >&2; } + notify_clawbot() { return 0; } +fi +if [ -n "${CALLER_BACKUP_BASE}" ]; then + BACKUP_BASE="${CALLER_BACKUP_BASE}" + BACKUP_LOG_DIR="${CALLER_BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}" + RESTIC_PASSWORD_FILE="${CALLER_RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}" + export BACKUP_BASE BACKUP_LOG_DIR RESTIC_PASSWORD_FILE +fi + +LOCAL_REPO="${GITEA_BACKUP_REPO:-${BACKUP_BASE}/gitea}" +STATE_DIR="${GITEA_RESTORE_DRILL_STATE_DIR:-${BACKUP_BASE}/integrity}" +WORK_ROOT="${GITEA_RESTORE_DRILL_WORK_ROOT:-${BACKUP_BASE}/tmp}" +STATUS_FILE="${STATE_DIR}/gitea-restore-drill.status" +MIN_FREE_BUFFER_BYTES="${GITEA_RESTORE_DRILL_MIN_FREE_BUFFER_BYTES:-2147483648}" +NOTIFY=0 +SOURCE_ZIP="${GITEA_RESTORE_DRILL_SOURCE_ZIP:-}" +LIVE_COUNTS="${GITEA_RESTORE_DRILL_LIVE_COUNTS:-}" +LIVE_INVENTORY_FILE="${GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE:-}" +START_TIME="$(date +%s)" +WORK_DIR="" + +while [ "$#" -gt 0 ]; do + case "$1" in + --notify) NOTIFY=1 ;; + --no-notify) NOTIFY=0 ;; + -h|--help) + printf 'Usage: verify-gitea-backup.sh [--notify|--no-notify]\n' + exit 0 + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + exit 2 + ;; + esac + shift +done + +cleanup() { + if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then + rm -rf "${WORK_DIR}" + fi +} +trap cleanup EXIT + +low_priority() { + if command -v ionice >/dev/null 2>&1; then + ionice -c2 -n7 nice -n 10 "$@" + else + nice -n 10 "$@" + fi +} + +file_size() { + stat -c '%s' "$1" 2>/dev/null || stat -f '%z' "$1" +} + +write_status() { + local success="$1" + local reason="$2" + local snapshot_id="$3" + local snapshot_timestamp="$4" + local dump_size="$5" + local backup_regular="$6" + local backup_wiki="$7" + local live_regular="$8" + local live_wiki="$9" + local database_present="${10}" + local config_present="${11}" + local crc_ok="${12}" + local inventory_match="${13}" + local duration + local temp_status + + duration=$(($(date +%s) - START_TIME)) + install -d -m 700 "${STATE_DIR}" + temp_status="${STATUS_FILE}.tmp.$$" + { + printf 'timestamp=%s\n' "$(date +%s)" + printf 'success=%s\n' "${success}" + printf 'reason=%s\n' "${reason}" + printf 'snapshot_id=%s\n' "${snapshot_id}" + printf 'snapshot_timestamp=%s\n' "${snapshot_timestamp}" + printf 'dump_size_bytes=%s\n' "${dump_size}" + printf 'backup_regular_repo_count=%s\n' "${backup_regular}" + printf 'backup_wiki_repo_count=%s\n' "${backup_wiki}" + printf 'live_regular_repo_count=%s\n' "${live_regular}" + printf 'live_wiki_repo_count=%s\n' "${live_wiki}" + printf 'database_dump_present=%s\n' "${database_present}" + printf 'config_present=%s\n' "${config_present}" + printf 'archive_crc_ok=%s\n' "${crc_ok}" + printf 'inventory_digest_match=%s\n' "${inventory_match}" + printf 'duration_seconds=%s\n' "${duration}" + } > "${temp_status}" + chmod 600 "${temp_status}" + mv "${temp_status}" "${STATUS_FILE}" +} + +fail_drill() { + local reason="$1" + local snapshot_id="${2:-unknown}" + local snapshot_timestamp="${3:-unknown}" + local dump_size="${4:-0}" + local backup_regular="${5:-0}" + local backup_wiki="${6:-0}" + local live_regular="${7:-0}" + local live_wiki="${8:-0}" + local database_present="${9:-0}" + local config_present="${10:-0}" + local crc_ok="${11:-0}" + local inventory_match="${12:-0}" + + write_status 0 "${reason}" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \ + "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \ + "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" + log_error "Gitea restore drill failed: ${reason}; evidence=${STATUS_FILE}" + if [ "${NOTIFY}" -eq 1 ]; then + notify_clawbot "failed" "gitea-restore-drill" "Gitea 備份還原演練失敗：${reason}" "$(($(date +%s) - START_TIME))" + fi + exit 1 +} + +if [ "${TEST_MODE}" != "1" ] && { [ -n "${SOURCE_ZIP}" ] || [ -n "${LIVE_COUNTS}" ] || [ -n "${LIVE_INVENTORY_FILE}" ]; }; then + fail_drill "test_override_rejected" +fi + +install -d -m 700 "${WORK_ROOT}" "${STATE_DIR}" +WORK_DIR="$(mktemp -d "${WORK_ROOT%/}/gitea-restore-drill.XXXXXX")" +chmod 700 "${WORK_DIR}" +RESTORED_ZIP="${WORK_DIR}/gitea-dump.zip" + +snapshot_id="unknown" +snapshot_timestamp="unknown" +dump_size=0 + +if [ "${TEST_MODE}" = "1" ]; then + [ -f "${SOURCE_ZIP}" ] || fail_drill "test_source_zip_missing" + cp "${SOURCE_ZIP}" "${RESTORED_ZIP}" + dump_size="$(file_size "${RESTORED_ZIP}")" + snapshot_id="synthetic" + snapshot_timestamp="synthetic" +else + [ -d "${LOCAL_REPO}/data" ] || fail_drill "restic_repo_missing" + [ -r "${RESTIC_PASSWORD_FILE}" ] || fail_drill "restic_password_file_unavailable" + + snapshot_meta="" + if ! snapshot_meta="$( + restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ + python3 -c 'import json,sys +rows=json.load(sys.stdin) +if rows: + row=rows[-1] + print("%s\t%s" % (row.get("short_id", "unknown"), row.get("time", "unknown")))' + )"; then + fail_drill "snapshot_metadata_read_failed" + fi + [ -n "${snapshot_meta}" ] || fail_drill "snapshot_metadata_missing" + IFS=$'\t' read -r snapshot_id snapshot_timestamp <<<"${snapshot_meta}" + + dump_meta="" + if ! dump_meta="$( + restic -r "${LOCAL_REPO}" ls latest --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \ + python3 -c 'import json,os,sys +for line in sys.stdin: + try: item=json.loads(line) + except json.JSONDecodeError: continue + if item.get("struct_type")=="node" and os.path.basename(item.get("path", ""))=="gitea-dump.zip": + print("%s\t%s" % (item.get("path", ""), item.get("size", 0))) + raise SystemExit' + )"; then + fail_drill "dump_metadata_read_failed" "${snapshot_id}" "${snapshot_timestamp}" + fi + [ -n "${dump_meta}" ] || fail_drill "gitea_dump_member_missing" "${snapshot_id}" "${snapshot_timestamp}" + IFS=$'\t' read -r dump_path dump_size <<<"${dump_meta}" + + available_bytes="$(df -Pk "${WORK_ROOT}" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}')" + required_bytes=$((dump_size + MIN_FREE_BUFFER_BYTES)) + if [ "${available_bytes}" -lt "${required_bytes}" ]; then + fail_drill "insufficient_free_space" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" + fi + + log_info "Restoring isolated Gitea dump snapshot=${snapshot_id} size_bytes=${dump_size}" + if ! low_priority restic -r "${LOCAL_REPO}" dump latest "${dump_path}" \ + --password-file "${RESTIC_PASSWORD_FILE}" > "${RESTORED_ZIP}"; then + fail_drill "restic_dump_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" + fi +fi + +actual_size="$(file_size "${RESTORED_ZIP}")" +if [ "${actual_size}" -ne "${dump_size}" ]; then + fail_drill "dump_size_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" +fi + +if ! low_priority unzip -tqq "${RESTORED_ZIP}"; then + fail_drill "archive_crc_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" +fi +crc_ok=1 + +inventory="" +if ! inventory="$( + unzip -Z1 "${RESTORED_ZIP}" | awk ' + /\.wiki\.git\/HEAD$/ { wiki += 1 } + /\.git\/HEAD$/ && $0 !~ /\.wiki\.git\/HEAD$/ { regular += 1 } + /(^|\/)gitea-db\.sql$/ { database += 1 } + /(^|\/)app\.ini$/ { config += 1 } + END { printf "%d %d %d %d", regular, wiki, database, config } + ' +)"; then + fail_drill "archive_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 0 0 0 0 0 "${crc_ok}" +fi +read -r backup_regular backup_wiki database_present config_present <<<"${inventory}" + +backup_inventory_file="${WORK_DIR}/backup-repo-inventory" +live_inventory_file="${WORK_DIR}/live-repo-inventory" +if ! unzip -Z1 "${RESTORED_ZIP}" | awk -F/ ' + /\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) } +' | LC_ALL=C sort -u > "${backup_inventory_file}"; then + fail_drill "backup_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" +fi + +if [ -n "${LIVE_INVENTORY_FILE}" ]; then + [ -f "${LIVE_INVENTORY_FILE}" ] || fail_drill "test_live_inventory_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" + LC_ALL=C sort -u "${LIVE_INVENTORY_FILE}" > "${live_inventory_file}" +else + if ! docker exec gitea sh -c 'find /data/git/repositories -type f -path "*.git/HEAD"' 2>/dev/null | awk -F/ ' + /\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) } + ' | LC_ALL=C sort -u > "${live_inventory_file}"; then + fail_drill "live_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}" + fi +fi + +live_inventory="$(awk ' + /\.wiki\.git$/ { wiki += 1; next } + /\.git$/ { regular += 1 } + END { printf "%d %d", regular, wiki } +' "${live_inventory_file}")" +read -r live_regular live_wiki <<<"${live_inventory}" + +hash_file() { + if command -v sha256sum >/dev/null 2>&1; then + sha256sum "$1" | awk '{print $1}' + else + shasum -a 256 "$1" | awk '{print $1}' + fi +} + +backup_inventory_digest="$(hash_file "${backup_inventory_file}")" +live_inventory_digest="$(hash_file "${live_inventory_file}")" +inventory_match=0 +[ "${backup_inventory_digest}" = "${live_inventory_digest}" ] && inventory_match=1 + +[ "${database_present}" -gt 0 ] || fail_drill "database_dump_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" 0 "${config_present}" "${crc_ok}" +[ "${config_present}" -gt 0 ] || fail_drill "config_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" 0 "${crc_ok}" +[ "${backup_regular}" -gt 0 ] || fail_drill "repository_inventory_empty" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" + +if [ "${backup_regular}" -ne "${live_regular}" ] || [ "${backup_wiki}" -ne "${live_wiki}" ]; then + fail_drill "repository_count_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" +fi +if [ "${inventory_match}" -ne 1 ]; then + fail_drill "repository_inventory_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" 0 +fi + +write_status 1 "verified" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \ + "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \ + "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}" +log_success "Gitea restore drill verified: regular=${backup_regular}/${live_regular} wiki=${backup_wiki}/${live_wiki}; evidence=${STATUS_FILE}" +if [ "${NOTIFY}" -eq 1 ]; then + notify_clawbot "success" "gitea-restore-drill" "Gitea 備份還原演練完成，repo=${backup_regular}/${live_regular}" "$(($(date +%s) - START_TIME))" +fi diff --git a/scripts/ops/backup-alert-label-contract-check.py b/scripts/ops/backup-alert-label-contract-check.py index 6107cbc43..3fcfb658b 100755 --- a/scripts/ops/backup-alert-label-contract-check.py +++ b/scripts/ops/backup-alert-label-contract-check.py @@ -6,6 +6,9 @@ Node exporter textfile metrics use labels such as job="backup_all" locally, but Prometheus rewrites that metric label to exported_job because the scrape target already has job="node-exporter-110". Backup alerts must therefore use $labels.exported_job in user-facing text and exported_job="..." in expressions. +Receipt alerts must also be demand-driven: only an actively firing backup or +Gitea alert may require a fresh receipt. Missing markers for inactive rules are +inventory evidence, not user-facing incidents. """ from __future__ import annotations @@ -142,20 +145,30 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: lines.append("OK BackupAlertReceiptMetricMissing188 checks 188 receipt requirement metric") rule = _require_alert(alerts, "BackupAlertReceiptStageMissing") - _require_contains( - str(rule.get("expr", "")), + expr = str(rule.get("expr", "")) + for required_fragment in [ + "ALERTS{", + 'alertstate="firing"', + 'alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*"', + 'alertname!="BackupAlertReceiptStageMissing"', + "unless on (alertname, host)", + "label_replace(", + "awoooi_backup_alert_receipt_fresh == 1", + ]: + _require_contains(expr, required_fragment, "BackupAlertReceiptStageMissing expr") + _require_not_contains( + expr, "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0", "BackupAlertReceiptStageMissing expr", ) text = _annotation_text(rule) for required_label in [ "$labels.host", - "$labels.receipt_channel", - "$value", - "required_alert/scope/stage/required_notification_type", + "$labels.alertname", + "required_alert", ]: _require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations") - lines.append("OK BackupAlertReceiptStageMissing aggregates by host/receipt channel and points to detail labels") + lines.append("OK BackupAlertReceiptStageMissing requires receipts only for actively firing backup/Gitea alerts") return lines diff --git a/scripts/ops/backup-health-textfile-exporter.py b/scripts/ops/backup-health-textfile-exporter.py index 280b49704..77572b77c 100755 --- a/scripts/ops/backup-health-textfile-exporter.py +++ b/scripts/ops/backup-health-textfile-exporter.py @@ -34,6 +34,10 @@ BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/sc OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite")) ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence")) CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json")) +GITEA_RESTORE_DRILL_STATUS_FILE = Path( + os.environ.get("AIOPS_GITEA_RESTORE_DRILL_STATUS_FILE", "/backup/integrity/gitea-restore-drill.status") +) +GITEA_RESTORE_DRILL_MAX_AGE_HOURS = float(os.environ.get("AIOPS_GITEA_RESTORE_DRILL_MAX_AGE_HOURS", "744")) BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts")) BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168")) DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [ @@ -73,6 +77,7 @@ BACKUP_ALERT_RECEIPT_REQUIREMENTS = [ {"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"}, {"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"}, {"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"}, + {"host": "110", "alertname": "GiteaRestoreDrillMissingOrFailed", "scope": "gitea_restore_drill"}, {"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"}, {"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"}, {"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"}, @@ -573,6 +578,46 @@ def _integrity_metric_lines(host: str) -> list[str]: return lines +def _gitea_restore_drill_metric_lines(host: str) -> list[str]: + values = _read_key_value_status(str(GITEA_RESTORE_DRILL_STATUS_FILE)) + now = int(time.time()) + timestamp = int(values.get("timestamp", 0)) + success = int(values.get("success", 0)) + dump_size = int(values.get("dump_size_bytes", 0)) + backup_regular = int(values.get("backup_regular_repo_count", 0)) + backup_wiki = int(values.get("backup_wiki_repo_count", 0)) + live_regular = int(values.get("live_regular_repo_count", 0)) + live_wiki = int(values.get("live_wiki_repo_count", 0)) + database_present = int(values.get("database_dump_present", 0)) + config_present = int(values.get("config_present", 0)) + crc_ok = int(values.get("archive_crc_ok", 0)) + inventory_match = int(values.get("inventory_digest_match", 0)) + age = max(0, now - timestamp) if timestamp else 0 + count_match = int(backup_regular == live_regular and backup_wiki == live_wiki and backup_regular > 0) + required_artifacts_ok = int(database_present > 0 and config_present > 0) + verified = int(success == 1 and crc_ok == 1 and required_artifacts_ok == 1 and count_match == 1 and inventory_match == 1) + fresh = int(timestamp > 0 and age <= int(GITEA_RESTORE_DRILL_MAX_AGE_HOURS * 3600) and verified == 1) + reason = str(values.get("reason", "status_missing")) + labels = f'host="{_escape_label(host)}",max_age_hours="{GITEA_RESTORE_DRILL_MAX_AGE_HOURS:g}"' + + return [ + f'awoooi_gitea_restore_drill_status_info{{{labels},reason="{_escape_label(reason)}"}} 1', + f"awoooi_gitea_restore_drill_last_success_timestamp{{{labels}}} {timestamp if verified else 0}", + f"awoooi_gitea_restore_drill_age_seconds{{{labels}}} {age}", + f"awoooi_gitea_restore_drill_fresh{{{labels}}} {fresh}", + f"awoooi_gitea_restore_drill_verified{{{labels}}} {verified}", + f"awoooi_gitea_restore_drill_archive_crc_ok{{{labels}}} {crc_ok}", + f"awoooi_gitea_restore_drill_required_artifacts_ok{{{labels}}} {required_artifacts_ok}", + f"awoooi_gitea_restore_drill_repo_count_match{{{labels}}} {count_match}", + f"awoooi_gitea_restore_drill_inventory_match{{{labels}}} {inventory_match}", + f"awoooi_gitea_restore_drill_dump_size_bytes{{{labels}}} {dump_size}", + f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="regular"}} {backup_regular}', + f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="wiki"}} {backup_wiki}', + f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="regular"}} {live_regular}', + f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="wiki"}} {live_wiki}', + ] + + def _config_capture_metric_lines(host: str) -> list[str]: now = int(time.time()) labels = f'host="{_escape_label(host)}"' @@ -975,6 +1020,28 @@ def _base_lines(host: str) -> list[str]: "# TYPE awoooi_backup_integrity_failed_repo_count gauge", "# HELP awoooi_backup_integrity_checked_repo_count Checked repository count from backup integrity or restore drill run.", "# TYPE awoooi_backup_integrity_checked_repo_count gauge", + "# HELP awoooi_gitea_restore_drill_status_info Bounded result reason from the isolated Gitea restore drill.", + "# TYPE awoooi_gitea_restore_drill_status_info gauge", + "# HELP awoooi_gitea_restore_drill_last_success_timestamp Unix timestamp of the latest fully verified isolated Gitea restore drill.", + "# TYPE awoooi_gitea_restore_drill_last_success_timestamp gauge", + "# HELP awoooi_gitea_restore_drill_age_seconds Age of the latest isolated Gitea restore drill evidence.", + "# TYPE awoooi_gitea_restore_drill_age_seconds gauge", + "# HELP awoooi_gitea_restore_drill_fresh Whether the isolated Gitea restore drill is verified and within max age.", + "# TYPE awoooi_gitea_restore_drill_fresh gauge", + "# HELP awoooi_gitea_restore_drill_verified Whether CRC, required assets, counts, and inventory all passed.", + "# TYPE awoooi_gitea_restore_drill_verified gauge", + "# HELP awoooi_gitea_restore_drill_archive_crc_ok Whether every Gitea dump ZIP member passed CRC verification.", + "# TYPE awoooi_gitea_restore_drill_archive_crc_ok gauge", + "# HELP awoooi_gitea_restore_drill_required_artifacts_ok Whether database dump and configuration are present.", + "# TYPE awoooi_gitea_restore_drill_required_artifacts_ok gauge", + "# HELP awoooi_gitea_restore_drill_repo_count_match Whether backup and live regular/wiki repo counts match.", + "# TYPE awoooi_gitea_restore_drill_repo_count_match gauge", + "# HELP awoooi_gitea_restore_drill_inventory_match Whether privacy-preserving backup and live repo inventories match.", + "# TYPE awoooi_gitea_restore_drill_inventory_match gauge", + "# HELP awoooi_gitea_restore_drill_dump_size_bytes Size of the verified Gitea dump archive.", + "# TYPE awoooi_gitea_restore_drill_dump_size_bytes gauge", + "# HELP awoooi_gitea_restore_drill_repo_count Regular or wiki repo count in backup or live inventory.", + "# TYPE awoooi_gitea_restore_drill_repo_count gauge", "# HELP awoooi_backup_config_capture_status_timestamp Unix timestamp of the latest config-capture coverage status.", "# TYPE awoooi_backup_config_capture_status_timestamp gauge", "# HELP awoooi_backup_config_capture_status_age_seconds Age of the latest config-capture coverage status.", @@ -1104,6 +1171,7 @@ def _collect_110(host: str) -> list[str]: "offsite_full_sync_verify": "/backup/scripts/verify-offsite-full-sync.sh --write-textfile", "backup_integrity_check": "/backup/scripts/check-backup-integrity.sh --mode check", "backup_restore_drill": "/backup/scripts/check-backup-integrity.sh --mode restore-drill", + "gitea_bundle_sync_188": "/backup/scripts/gitea-bundle-backup-sync-188.sh", } for job, pattern in expected_crons.items(): labels = f'host="{_escape_label(host)}",job="{_escape_label(job)}"' @@ -1124,6 +1192,9 @@ def _collect_110(host: str) -> list[str]: "verify-offsite-full-sync.sh", "mark-credential-escrow-verified.sh", "check-backup-integrity.sh", + "verify-gitea-backup.sh", + "gitea-repo-bundle-backup.sh", + "gitea-bundle-backup-sync-188.sh", "backup-gitea.sh", "backup-harbor.sh", "backup-momo.sh", @@ -1193,6 +1264,7 @@ def _collect_110(host: str) -> list[str]: lines.append(f"awoooi_backup_last_run_failed_count{{{labels}}} {failed_count}") lines.append(f"awoooi_backup_job_last_success_timestamp{{{labels},type=\"aggregate\",source=\"110-cron-log\",target=\"/backup/logs/cron.log\",max_age_hours=\"48\"}} {backup_all_ts if failed_count == 0 else 0}") lines.extend(_integrity_metric_lines(host)) + lines.extend(_gitea_restore_drill_metric_lines(host)) lines.extend(_config_capture_metric_lines(host)) lines.extend(_offsite_and_escrow_metric_lines(host)) lines.extend(_retention_metric_lines(host)) @@ -1215,11 +1287,13 @@ def _collect_188(host: str) -> list[str]: for script in [ "/home/ollama/bin/backup-from-110.sh", "/home/ollama/bin/momo-pg-backup.sh", + "/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh", "/home/ollama/awoooi-ops/pg-backup.sh", ]: labels = f'host="{_escape_label(host)}",script="{_escape_label(Path(script).name)}"' lines.append(f"awoooi_backup_script_present{{{labels}}} {int(Path(script).exists() and os.access(script, os.X_OK))}") + backup_from_110_ts = _read_backup_110_timestamp() lines.extend( _metric_lines_for_job( host=host, @@ -1227,12 +1301,13 @@ def _collect_188(host: str) -> list[str]: source="188-rsync", target="/home/ollama/backup/110", backup_type="rsync", - last_success=_read_backup_110_timestamp(), + last_success=backup_from_110_ts, max_age_hours=25, sample_count=1, ) ) - gitea_mirror_ts, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea")) + _, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea")) + gitea_mirror_ts = backup_from_110_ts gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0 lines.extend( _metric_lines_for_job( @@ -1248,6 +1323,20 @@ def _collect_188(host: str) -> list[str]: ) gitea_bundle_lines, gitea_bundle_ok = _gitea_bundle_metric_lines(host) lines.extend(gitea_bundle_lines) + gitea_bundle_ts, gitea_bundle_count = _newest_tree_timestamp(GITEA_BUNDLE_ROOT, max_entries=20000) + gitea_bundle_fresh = 1 if gitea_bundle_ts and int(time.time()) - gitea_bundle_ts <= 25 * 3600 else 0 + lines.extend( + _metric_lines_for_job( + host=host, + job="gitea_private_bundle", + source="188-git-bundle", + target=str(GITEA_BUNDLE_ROOT), + backup_type="git_bundle", + last_success=gitea_bundle_ts, + max_age_hours=25, + sample_count=gitea_bundle_count, + ) + ) coverage_labels = ( f'host="{_escape_label(host)}",' 'domain="service",' @@ -1256,7 +1345,7 @@ def _collect_188(host: str) -> list[str]: lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1") lines.append( "awoooi_backup_coverage_domain_fresh" - f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_ok else 0}" + f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_fresh and gitea_bundle_ok else 0}" ) momo_ts = _newest_file_timestamp([ "/home/ollama/momo_backups/*.sql.gz", diff --git a/scripts/ops/prometheus-rule-drift-guard.sh b/scripts/ops/prometheus-rule-drift-guard.sh index a8bf9b19b..4fb195dd7 100755 --- a/scripts/ops/prometheus-rule-drift-guard.sh +++ b/scripts/ops/prometheus-rule-drift-guard.sh @@ -13,10 +13,16 @@ CURRENT_RULES="${CURRENT_RULES:-/home/wooo/monitoring/alerts.yml}" CANONICAL_RULES="${CANONICAL_RULES:-/home/wooo/monitoring/alerts-unified.canonical.yml}" TEXTFILE="${TEXTFILE:-/home/wooo/node_exporter_textfiles/prometheus_rule_drift_guard.prom}" LOG_FILE="${LOG_FILE:-/home/wooo/logs/prometheus-rule-drift-guard.log}" +PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}" +PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}" +PROMETHEUS_COMPOSE_FILE="${PROMETHEUS_COMPOSE_FILE:-/home/wooo/monitoring/docker-compose.yml}" +PROMETHEUS_COMPOSE_SERVICE="${PROMETHEUS_COMPOSE_SERVICE:-prometheus}" +ALLOW_RUNTIME_RECREATE="${ALLOW_RUNTIME_RECREATE:-1}" REQUIRED_RULES=( "BackupCredentialEscrowEvidenceMissing" "BackupExpectedJobMissing" + "GiteaRestoreDrillMissingOrFailed" "awoooi_recovery_core_ready" "awoooi_recovery_dr_offsite_ready" "ColdStartRecoveryBlocked" @@ -32,6 +38,7 @@ write_textfile() { local repaired="$2" local missing_count="$3" local matches_canonical="$4" + local runtime_matches_current="$5" local tmp mkdir -p "$(dirname "$TEXTFILE")" 2>/dev/null || true tmp="$(mktemp "${TEXTFILE}.tmp.XXXXXX")" || return 0 @@ -48,6 +55,9 @@ awoooi_prometheus_rule_drift_guard_missing_required_count{host="${HOST_LABEL}"} # HELP awoooi_prometheus_rule_drift_guard_current_matches_canonical Whether active alerts.yml matches canonical copy. # TYPE awoooi_prometheus_rule_drift_guard_current_matches_canonical gauge awoooi_prometheus_rule_drift_guard_current_matches_canonical{host="${HOST_LABEL}"} ${matches_canonical} +# HELP awoooi_prometheus_rule_drift_guard_runtime_matches_current Whether the Prometheus container mount matches the active host rules file. +# TYPE awoooi_prometheus_rule_drift_guard_runtime_matches_current gauge +awoooi_prometheus_rule_drift_guard_runtime_matches_current{host="${HOST_LABEL}"} ${runtime_matches_current} EOF chmod 0644 "$tmp" 2>/dev/null || true mv "$tmp" "$TEXTFILE" 2>/dev/null || rm -f "$tmp" @@ -67,16 +77,32 @@ try: if payload.get("status") != "success": raise RuntimeError(payload) loaded = { - str(rule.get("name") or rule.get("alert") or rule.get("record")) + str(rule.get("name") or rule.get("alert") or rule.get("record")): rule for group in payload.get("data", {}).get("groups") or [] for rule in group.get("rules") or [] } - print(len(required - loaded)) + missing = required - set(loaded) + unhealthy = {name for name in required & set(loaded) if loaded[name].get("health") != "ok"} + print(len(missing | unhealthy)) except Exception as exc: print(f"QUERY_FAILED:{exc}") PY } +wait_for_required_rules() { + local attempt missing + for attempt in $(seq 1 30); do + missing="$(rules_missing_count)" + if [ "$missing" = "0" ]; then + log "runtime rules converged after attempt=${attempt}" + return 0 + fi + sleep 2 + done + log "runtime rules did not converge within 60s: ${missing}" + return 1 +} + matches_canonical() { if [ ! -f "$CURRENT_RULES" ] || [ ! -f "$CANONICAL_RULES" ]; then echo 0 @@ -89,60 +115,136 @@ matches_canonical() { fi } +file_sha256() { + sha256sum "$1" 2>/dev/null | awk '{print $1}' +} + +runtime_rules_sha256() { + docker exec "${PROMETHEUS_CONTAINER}" sha256sum "${PROMETHEUS_RUNTIME_RULES}" 2>/dev/null | awk '{print $1}' +} + +runtime_matches_current() { + local current_hash runtime_hash + current_hash="$(file_sha256 "${CURRENT_RULES}")" + runtime_hash="$(runtime_rules_sha256)" + if [ -n "${current_hash}" ] && [ "${current_hash}" = "${runtime_hash}" ]; then + echo 1 + else + echo 0 + fi +} + +validate_canonical_rules() { + local candidate_path rc + candidate_path="/tmp/awoooi-prometheus-rule-guard-candidate.$$.yml" + docker cp "${CANONICAL_RULES}" "${PROMETHEUS_CONTAINER}:${candidate_path}" >/dev/null 2>&1 || return 1 + docker exec "${PROMETHEUS_CONTAINER}" promtool check rules "${candidate_path}" >>"${LOG_FILE}" 2>&1 + rc=$? + docker exec -u 0 "${PROMETHEUS_CONTAINER}" rm -f "${candidate_path}" >/dev/null 2>&1 || true + return "${rc}" +} + +recreate_prometheus_runtime() { + [ "${ALLOW_RUNTIME_RECREATE}" = "1" ] || return 1 + [ -f "${PROMETHEUS_COMPOSE_FILE}" ] || return 1 + docker compose -f "${PROMETHEUS_COMPOSE_FILE}" up -d --no-deps --force-recreate \ + "${PROMETHEUS_COMPOSE_SERVICE}" >>"${LOG_FILE}" 2>&1 +} + restore_rules() { - local backup_path + local backup_path needs_recreate backup_path="${CURRENT_RULES}.guard.bak.$(date +%Y%m%d%H%M%S)" - cp "$CURRENT_RULES" "$backup_path" 2>/dev/null || true - cp "$CANONICAL_RULES" "$CURRENT_RULES" - curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null + if ! cp "$CURRENT_RULES" "$backup_path" 2>/dev/null; then + log "failed to create active rules rollback copy" + return 1 + fi + if ! validate_canonical_rules; then + log "canonical promtool validation failed" + return 1 + fi + + if ! cat "$CANONICAL_RULES" >"$CURRENT_RULES"; then + log "failed to update active rules file" + return 1 + fi + + needs_recreate=0 + [ "$(runtime_matches_current)" -eq 1 ] || needs_recreate=1 + if [ "${needs_recreate}" -eq 1 ]; then + log "runtime bind mount differs from active file; recreating Prometheus only" + if ! recreate_prometheus_runtime; then + cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true + log "Prometheus recreate failed; active file rolled back" + return 1 + fi + elif ! curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null; then + cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true + curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true + log "Prometheus reload failed; active file rolled back" + return 1 + fi + + if wait_for_required_rules && [ "$(runtime_matches_current)" -eq 1 ]; then + return 0 + fi + + cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true + if [ "${needs_recreate}" -eq 1 ]; then + recreate_prometheus_runtime >/dev/null 2>&1 || true + else + curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true + fi + log "post-apply verifier failed; active rules rolled back" + return 1 } main() { if [ ! -f "$CANONICAL_RULES" ]; then log "canonical rules missing: ${CANONICAL_RULES}" - write_textfile "canonical_missing" 0 999 0 + write_textfile "canonical_missing" 0 999 0 0 return 1 fi - local missing before_matches repaired after_missing after_matches + local missing before_matches before_runtime_matches repaired after_missing after_matches after_runtime_matches missing="$(rules_missing_count)" before_matches="$(matches_canonical)" + before_runtime_matches="$(runtime_matches_current)" repaired=0 if [[ "$missing" == QUERY_FAILED:* ]]; then log "Prometheus query failed: ${missing}" - write_textfile "query_failed" 0 999 "$before_matches" + write_textfile "query_failed" 0 999 "$before_matches" "$before_runtime_matches" return 1 fi - if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ]; then - log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches}; restoring" + if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ] || [ "$before_runtime_matches" -eq 0 ]; then + log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches} runtime_matches_current=${before_runtime_matches}; restoring" if restore_rules; then repaired=1 - sleep 3 else log "restore failed" - write_textfile "restore_failed" 0 "$missing" "$before_matches" + write_textfile "restore_failed" 0 "$missing" "$before_matches" "$before_runtime_matches" return 1 fi fi after_missing="$(rules_missing_count)" after_matches="$(matches_canonical)" + after_runtime_matches="$(runtime_matches_current)" if [[ "$after_missing" == QUERY_FAILED:* ]]; then log "post-restore Prometheus query failed: ${after_missing}" - write_textfile "post_query_failed" "$repaired" 999 "$after_matches" + write_textfile "post_query_failed" "$repaired" 999 "$after_matches" "$after_runtime_matches" return 1 fi - if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ]; then - write_textfile "ok" "$repaired" "$after_missing" "$after_matches" + if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ] && [ "$after_runtime_matches" -eq 1 ]; then + write_textfile "ok" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches" log "ok repaired=${repaired}" return 0 fi - log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches}" - write_textfile "drifted" "$repaired" "$after_missing" "$after_matches" + log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches} runtime_matches_current=${after_runtime_matches}" + write_textfile "drifted" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches" return 1 } diff --git a/scripts/ops/tests/test_backup_alert_label_contract.py b/scripts/ops/tests/test_backup_alert_label_contract.py new file mode 100644 index 000000000..70ffe5c6d --- /dev/null +++ b/scripts/ops/tests/test_backup_alert_label_contract.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +CHECKER = ROOT / "scripts" / "ops" / "backup-alert-label-contract-check.py" +RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml" +BASELINE = ROOT / "ops" / "reboot-recovery" / "full-stack-backup-baseline.yml" + + +def _run(rules: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(CHECKER), + "--rules", + str(rules), + "--baseline", + str(BASELINE), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=False, + ) + + +def test_checker_accepts_firing_only_receipt_contract() -> None: + result = _run(RULES) + + assert result.returncode == 0, result.stderr + assert "requires receipts only for actively firing backup/Gitea alerts" in result.stdout + assert "BACKUP_ALERT_LABEL_CONTRACT_OK" in result.stdout + + +def test_checker_rejects_non_firing_receipt_selector(tmp_path: Path) -> None: + payload = yaml.safe_load(RULES.read_text(encoding="utf-8")) + for group in payload["groups"]: + for rule in group.get("rules", []): + if rule.get("alert") == "BackupAlertReceiptStageMissing": + rule["expr"] = str(rule["expr"]).replace( + 'alertstate="firing"', + 'alertstate="pending"', + ) + candidate = tmp_path / "alerts.yml" + candidate.write_text(yaml.safe_dump(payload, allow_unicode=True, sort_keys=False), encoding="utf-8") + + result = _run(candidate) + + assert result.returncode != 0 + assert 'alertstate="firing"' in result.stderr diff --git a/scripts/ops/tests/test_backup_health_textfile_exporter.py b/scripts/ops/tests/test_backup_health_textfile_exporter.py index 4c77eebb7..06b163a1a 100644 --- a/scripts/ops/tests/test_backup_health_textfile_exporter.py +++ b/scripts/ops/tests/test_backup_health_textfile_exporter.py @@ -35,6 +35,50 @@ def test_default_gitea_bundle_expected_repos_cover_current_project_set() -> None "wooo/bitan-pharmacy", "wooo/vtuber", ] + assert any( + row["alertname"] == "GiteaRestoreDrillMissingOrFailed" + for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS + ) + + +def test_gitea_restore_drill_metrics_require_inventory_and_artifacts(tmp_path: Path, monkeypatch) -> None: + exporter = load_exporter() + status_file = tmp_path / "gitea-restore-drill.status" + status_file.write_text( + "\n".join( + [ + "timestamp=2000000000", + "success=1", + "reason=verified", + "dump_size_bytes=6928031514", + "backup_regular_repo_count=15", + "backup_wiki_repo_count=0", + "live_regular_repo_count=15", + "live_wiki_repo_count=0", + "database_dump_present=1", + "config_present=2", + "archive_crc_ok=1", + "inventory_digest_match=1", + ] + ) + + "\n", + encoding="utf-8", + ) + monkeypatch.setattr(exporter, "GITEA_RESTORE_DRILL_STATUS_FILE", status_file) + monkeypatch.setattr(exporter.time, "time", lambda: 2000000060) + + rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110")) + assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 1' in rendered + assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 1' in rendered + assert 'awoooi_gitea_restore_drill_inventory_match{host="110",max_age_hours="744"} 1' in rendered + + status_file.write_text( + status_file.read_text(encoding="utf-8").replace("inventory_digest_match=1", "inventory_digest_match=0"), + encoding="utf-8", + ) + rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110")) + assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 0' in rendered + assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 0' in rendered def test_gitea_bundle_metrics_require_all_expected_repos(tmp_path: Path, monkeypatch) -> None: diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 7a844d84a..727244100 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -446,6 +446,7 @@ def test_backup_exporter_emits_domain_level_backup_coverage() -> None: "GiteaPrivateBundleBackupIncomplete", "GiteaPrivateBundleRestoreDryRunMissing", "BackupRestoreDrillMissingOrFailed", + "GiteaRestoreDrillMissingOrFailed", "BackupCredentialEscrowEvidenceMissing", ]: assert alertname in exporter @@ -462,16 +463,19 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert "BackupAlertReceiptMetricMissing" in alerts assert "BackupAlertReceiptStageMissing" in alerts assert "awoooi_backup_alert_receipt_expected_info" in alerts - assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0" in alerts - assert "$labels.receipt_channel" in alerts - assert "$value" in alerts - assert "required_alert/scope/stage/required_notification_type" in alerts - assert "telegram_awooop" in alerts + assert 'ALERTS{' in alerts + assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts + assert "label_replace(" in alerts + assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts + assert "$labels.alertname" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: script = read("scripts/backup/gitea-repo-bundle-backup.sh") restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh") + wrapper = read("scripts/backup/gitea-bundle-backup-sync-188.sh") + exporter = read("scripts/ops/backup-health-textfile-exporter.py") + playbook = read("infra/ansible/playbooks/110-devops.yml") assert "GIT_TERMINAL_PROMPT=0" in script assert "GIT_REMOTE_BASE" in script @@ -481,6 +485,9 @@ def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: assert "bundle verify" in script assert ".sha256" in script assert "manifest.tsv" in script + assert "latest-private-complete" in script + assert "complete links unchanged" in script + assert 'exit 1' in script assert "gitea dump" in script assert "does not replace `gitea dump`" in script assert "tokens or passwords" in script @@ -488,3 +495,39 @@ def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: assert "fsck --connectivity-only" in restore_script assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script assert "never reads tokens" in restore_script + assert "flock -n" in wrapper + assert "latest-private-complete" in wrapper + assert "--local-container" in wrapper + assert "rsync -a --checksum" in wrapper + assert "remote_sample_restore_fsck" in wrapper + assert "refresh_remote_backup_health" in wrapper + assert "rollback_remote_links" in wrapper + assert "remote_manifest_checksum_ok" in wrapper + assert "promoted=1" in wrapper + assert '"gitea_bundle_sync_188"' in exporter + assert 'job="gitea_private_bundle"' in exporter + assert "gitea-bundle-backup-sync-188.sh" in playbook + assert 'minute: "15"' in playbook + assert 'hour: "0"' in playbook + + +def test_prometheus_rule_guard_repairs_container_bind_mount_drift() -> None: + guard = read("scripts/ops/prometheus-rule-drift-guard.sh") + playbook = read("infra/ansible/playbooks/110-devops.yml") + + assert "runtime_matches_current" in guard + assert "awoooi_prometheus_rule_drift_guard_runtime_matches_current" in guard + assert "promtool check rules" in guard + assert "--no-deps --force-recreate" in guard + assert "failed to create active rules rollback copy" in guard + assert "post-apply verifier failed; active rules rolled back" in guard + assert "prometheus-rule-drift-guard.sh" in playbook + assert 'minute: "*/5"' in playbook + + +def test_velero_restore_test_recovers_a_missed_weekly_schedule() -> None: + manifest = read("k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml") + + assert "startingDeadlineSeconds: 604800" in manifest + assert 'schedule: "0 18 * * 6"' in manifest + assert "concurrencyPolicy: Forbid" in manifest