diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 2b3fceb16..a28112ddc 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -626,6 +626,8 @@ jobs: ;; ops/alertmanager/alertmanager.yml) ;; + ops/reboot-recovery/full-stack-backup-baseline.yml) + ;; scripts/dev/awoooi-navigation-coverage-guard.py) ;; scripts/ci/wait-host-web-build-pressure.sh) @@ -648,6 +650,8 @@ jobs: ;; scripts/ops/backup-alert-label-contract-check.py) ;; + scripts/ops/backup-alert-live-visibility-check.py) + ;; scripts/ops/backup-from-110.sh) ;; scripts/ops/backup-health-textfile-exporter.py) @@ -945,6 +949,7 @@ jobs: ../../scripts/reboot-recovery/validate-windows99-console-verify-artifact.py \ ../../scripts/reboot-recovery/windows99-management-channel-probe.py \ ../../scripts/ops/backup-alert-label-contract-check.py \ + ../../scripts/ops/backup-alert-live-visibility-check.py \ ../../scripts/ops/backup-health-textfile-exporter.py \ ../../scripts/ops/docker-disk-pressure-retention-cleanup.py \ ../../scripts/ops/gitea-queue-hook-backlog-playbook.py \ @@ -960,6 +965,7 @@ jobs: python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/alertmanager/alertmanager.yml')); print('alertmanager YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-cold-start-baseline.yml')); print('full-stack-cold-start-baseline YAML OK')" + python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-backup-baseline.yml')); print('full-stack-backup-baseline YAML OK')" bash -n \ ../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh \ ../../ops/runner/check-awoooi-non110-runner-readiness.sh \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 545237f87..ef9308d75 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -53780,3 +53780,24 @@ production browser smoke: **下一步**： - 跑 `guard-gitea-runner-pressure.py`、`git diff --check` 後 commit / push 到 Gitea main；deploy 後讀回 `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 與 `/api/v1/agents/awoooi-priority-work-order-readback`，目標是看到 `gitea_repo_bundle_backup_ready=true`、`gitea_repo_bundle_expected_repo_count=12`、`gitea_repo_bundle_repo_row_count=12`、`backup_core_green_false_but_gitea_repo_bundle_ready=true`。 + +## 2026-07-03 — P0-5 / P0-6 備份告警 receipt source contract + +**完成內容**： +- `backup-health-textfile-exporter` 新增 backup alert receipt requirement，對每個 required backup alert 輸出 `sent / received / dedup / escalation` 四段 evidence stage，並用 `required_alert` / `required_notification_type` 避免覆蓋 Prometheus meta alert 的 `alertname` / `notification_type`。 +- Prometheus rules 新增 `BackupAlertReceiptMetricMissing`、`BackupAlertReceiptMetricMissing188`、`BackupAlertReceiptStageMissing`，讓 110 / 188 任一 backup alert receipt 缺段時會以具名 alert 暴露，不再只剩「可能沒告警」的盲點。 +- baseline、label contract checker、live visibility checker、CD controlled-runtime allowlist 與 P0 operational contract tests 已同步，將 backup freshness / restore drill / alert receipt 分成不同 evidence 層，避免用 exporter 存在誤判為 Telegram / AwoooP receipt 完成。 +- `FULL-STACK-COLD-START-SOP`、`REBOOT-RECOVERY-SOP`、reboot cold-start workplan、commander inserted priority ledger 已寫入本輪主機重啟後的備份告警 receipt 判讀與禁做事項。 + +**本地驗證結果**： +- `python3.11 -m pytest scripts/ops/tests/test_backup_health_textfile_exporter.py scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py ops/runner/test_cd_controlled_runtime_profile.py -q -p no:cacheprovider`：`65 passed`。 +- `python3.11 scripts/ops/backup-alert-label-contract-check.py`：`BACKUP_ALERT_LABEL_CONTRACT_OK`。 +- `python3.11 -m py_compile scripts/ops/backup-health-textfile-exporter.py scripts/ops/backup-alert-label-contract-check.py scripts/ops/backup-alert-live-visibility-check.py`、Prometheus / baseline YAML parse、`guard-gitea-runner-pressure.py --root .`、`git diff --check`：通過。 + +**仍維持**： +- 沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有讀 runner token 或 Telegram token。 +- 沒有使用 GitHub / gh / GitHub API / GitHub Actions。 +- 沒有重啟主機，沒有 Docker / Nginx / K3s / DB / firewall restart，沒有 workflow_dispatch，沒有 DROP / TRUNCATE / restore / prune，沒有送 Telegram 測試訊息。 + +**下一步**： +- 整合 Gitea main 最新 commit 後 push；CD 完成後讀回 Prometheus / Alertmanager / production scorecard，確認新 receipt metrics / rules 進 runtime，再往下一個 P0 blocker（99 VMware autostart / full Gitea dump / DB/settings/issues/packages/LFS / service green）推進。 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index b0da7bf39..485b3c70f 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.105 +> Version: v1.106 > Last updated: 2026-07-03 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 @@ -2565,3 +2565,51 @@ docker update --cpus=1.5 gitea - active CI load 只能走 runner / queue / pressure gate lane；不得直接 `SIGKILL`、`docker restart`、`systemctl restart`、DB restart 或恢復 generic runner。 - evidence output 仍必須保持 sanitized，不輸出 hostedtoolcache 完整路徑、raw command、URL、workspace path 或 secret。 + +### 16.6 2026-07-03 backup alert receipt 四段證據 + +P0-5 / P0-6 不得再只用「backup health exporter 有跑」或「Prometheus rule 存在」宣稱備份告警閉環。`scripts/ops/backup-health-textfile-exporter.py` 必須輸出 backup alert receipt requirement，且每個 required backup alert 都要追蹤四段不含 secret 的 evidence marker： + +- `sent` +- `received` +- `dedup` +- `escalation` + +標準 marker 位置： + +```bash +/backup/alert-receipts/..last_success +/backup/alert-receipts//.last_success +/backup/alert-receipts/.last_success +/backup/alert-receipts//last_success +``` + +shared marker 只作兼容讀回；正式 closeout 仍應逐 stage 補齊。marker 內容只允許 timestamp 或脫敏 evidence id，不得寫入 Telegram token、chat raw payload、使用者訊息、runner token、DB DSN、`.env`、cookie、authorization header 或任何 secret。 + +新增 metrics： + +```text +awoooi_backup_alert_receipt_expected_info{host,required_alert,scope,required_notification_type,receipt_channel,max_age_hours} +awoooi_backup_alert_receipt_fresh{...} +awoooi_backup_alert_receipt_stage_expected_info{...,stage} +awoooi_backup_alert_receipt_stage_fresh{...,stage} +``` + +Prometheus 必須載入： + +- `BackupAlertReceiptMetricMissing` +- `BackupAlertReceiptMetricMissing188` +- `BackupAlertReceiptStageMissing` + +判讀規則： + +- `awoooi_backup_alert_receipt_stage_fresh == 0` 代表該 backup alert 的 `sent / received / dedup / escalation` 某段沒有新鮮證據；不能宣稱 Telegram/AwoooP receipt 完成。 +- `required_alert` 是原始 backup alert 名稱；不得用 Prometheus 保留 label `alertname` 承載原始 alert，避免被 meta alert 覆蓋。 +- `required_notification_type` 是原始 backup alert 應走的類型；meta alert 自己的 `notification_type` 可維持 `TYPE-1`。 +- `scripts/ops/backup-alert-label-contract-check.py` 是 source rule contract；`scripts/ops/backup-alert-live-visibility-check.py` 是 Prometheus / Alertmanager live visibility contract。 + +禁止事項： + +- 不得為了產生 receipt 而送測試 Telegram、呼叫 Bot API、改 Alertmanager receiver route、改 silence 或 workflow_dispatch。 +- 不得讀取 token / chat raw payload / secret volume。 +- 不得用 receipt marker 取代真正 backup freshness、restore drill、Gitea bundle completeness 或 offsite verifier。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 677255db8..685d35d71 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.2 -> **最後更新**: 2026-07-02 (台北時間) +> **版本**: v5.3 +> **最後更新**: 2026-07-03 (台北時間) > **更新者**: Codex > **觸發事件**: 110 control-path / Harbor recovery receipt、Gitea stale queue blocker、controlled CD lane fail-closed enforcer staging 收斂 @@ -399,6 +399,24 @@ scripts/backup/gitea-bundle-sample-restore-dry-run.sh --repo wooo/awoooi --write - `awoooi_gitea_bundle_sample_restore_dry_run_ok` absent / `0` 是 restore confidence blocker；只允許在暫存目錄 clone bundle + `git fsck --connectivity-only`，不得把 bundle restore 回 Gitea production。 - public repo search 只代表 public visibility；private/internal repo 要看 authenticated inventory、SSH heads 或 bundle metrics，不能用 404 直接推論 history 消失。 +**Backup alert receipt 判讀 (2026-07-03)**: + +備份監控不只看 freshness / failed / age / restore drill，還必須能證明告警進入 Telegram / AwoooP receipt 鏈。重啟後若使用者問「備份到底有沒有告警」，固定順序如下： + +```bash +python3 scripts/ops/backup-alert-label-contract-check.py +curl -G -s http://192.168.0.110:9090/api/v1/query --data-urlencode 'query=awoooi_backup_alert_receipt_stage_fresh{host=~"110|188"}' | python3 -m json.tool +scripts/ops/backup-alert-live-visibility-check.py --prometheus-url http://192.168.0.110:9090 --alertmanager-url http://192.168.0.110:9093 --host 110 +scripts/ops/backup-alert-live-visibility-check.py --prometheus-url http://192.168.0.110:9090 --alertmanager-url http://192.168.0.110:9093 --host 188 +``` + +判讀規則： + +- `required_alert` 是原始 backup alert，例如 `BackupJobStale`、`BackupCoverageDomainStale`、`GiteaPrivateBundleBackupIncomplete`；不得用 Prometheus 保留 label `alertname` 承載它。 +- 每個 required alert 都要有 `sent / received / dedup / escalation` 四段 marker；缺一段就是 `BackupAlertReceiptStageMissing`。 +- marker 只能是 `/backup/alert-receipts/*.last_success` 這類 timestamp / evidence id；不得寫 Telegram token、raw payload、chat id 原文或 secret。 +- 這個 receipt proof 不取代真正 backup / restore proof；Gitea bundle、Gitea full dump、DB/settings/issues/packages/LFS、offsite、restore drill 仍要各自綠。 + **Harbor Exited 128 修復**: ```bash # 等 harbor-log healthy diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index d88f11570..0a2f6ce7a 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -23,8 +23,8 @@ | P0-2 | BLOCKED_EDGE_PRIVILEGED_APPLY | Deploy / reboot 期間 public 502 維護頁與外部 fallback | Gitea CD `#4519` 已推 deploy marker `3aca484 -> a94ddd5`，但 marker 後 public probe 仍讀到 `https://awoooi.wooo.work/api/v1/health` raw `502`、fallback header/body 空；live 188 `/etc/nginx/sites-enabled/awoooi.wooo.work.conf` 缺 maintenance fallback，`/var/www/maintenance/maintenance.html` 缺失，`ollama@188` 無 passwordless sudo。 | 先跑 `scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh --check` 留 drift receipt；具備 privileged channel 後執行 `--apply`，要求 backup、`nginx -t`、reload、public route probe 全綠。不得讀密碼、不得用 app restart 掩蓋 edge fallback drift。 | | P0-3 | PARTIAL_GREEN_SOURCE_RUNTIME | 所有產品 / 網站版本與資料最新性 | Gitea `main=89d4d6112`；Production SLO readback 對齊 deploy marker `89d4d6112 chore(cd): deploy 17ba08c [skip ci]`；Stock freshness `status=ok`、`latest_trading_date=2026-07-02`、blockers `[]`。AWOOOI health HTTP 200 但整體 `degraded`，SignOz / local Ollama 仍需列為 runtime degraded evidence。 | 將 source SHA / deploy marker / runtime endpoint / public route watch 固定進 scorecard；完成條件是每個 public product 都有 source、deploy、runtime、freshness 四層 readback，且 degraded components 有 owner lane。 | | P0-4 | BLOCKED_WINDOWS99_AUTOSTART | 192.168.0.99 VMware 自動啟動與 VM guest 111 / 188 / 120 / 121 / 112 | Scorecard：`windows99_update_no_auto_reboot_ready=true`、`windows99_vmware_verify_ready=false`、VMX missing alias `111`、powered off aliases `111/112/120/121/188`、WinRM unavailable、SSH BatchMode permission denied、RDP / Hyper-V console channel reachable。 | 只用 no-secret console / management collector 取得 `windows99-vmware-autostart.ps1 -Mode Verify` stdout，先以 `validate-windows99-console-verify-artifact.py` 轉成 normalized artifact，再重跑 SLO scorecard；完成條件是 VMX config ready、guest power ready、99 uptime known、all required host reachable。 | -| P0-5 | PARTIAL_GREEN_BACKUP_MONITORING | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。這只證明 repo bundle / core backup，不等於 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控完成。 | 補齊 backup health textfile / Prometheus / Telegram receipt matrix：每個 backup scope 必須有 fresh、failed、age、restore-drill、alert-receipt；完成條件是沒有「不知道備份有沒有跑」的盲點。 | -| P0-6 | SOURCE_READY_ALERT_RECEIPT_BLOCKED | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Source 已有 reboot / backup alert rules 與 per-blocker projection；scorecard `telegram_active_blocker_alert_required_count=8`。尚未完成 shutdown / reboot / backup alert 的脫敏 Telegram delivery receipt 全矩陣。 | 補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | +| P0-5 | SOURCE_READY_BACKUP_RECEIPT_CONTRACT | Gitea / 主機 / DB / 網站 / 服務 / 套件 / 工具 / log 備份監控告警 | Gitea repo bundle readback ready：expected `12`、rows `12`、missing `0`、failed `0`、sample restore dry-run ok；backup core green。2026-07-03 source 已新增 `awoooi_backup_alert_receipt_*` 指標、`BackupAlertReceiptMetricMissing*` / `BackupAlertReceiptStageMissing`、baseline contract 與 tests，固定每個 required backup alert 需有 `sent / received / dedup / escalation` receipt marker。 | 推 CD 後讀回 Prometheus / Alertmanager 是否載入新 rules，並補 `/backup/alert-receipts/*.last_success` 脫敏 marker；下一層仍要補 Gitea full dump、DB/settings/issues/packages/LFS、所有工具與 log 全量備份監控。 | +| P0-6 | SOURCE_READY_ALERT_RECEIPT_DEPLOY_PENDING | 主機關機 / 重啟 / SLO miss / backup failure Telegram 告警 | Source 已有 reboot / backup alert rules、per-blocker projection、backup alert receipt requirement 與 live visibility checker；scorecard `telegram_active_blocker_alert_required_count=8`。尚未完成 shutdown / reboot / backup alert 的 production 脫敏 delivery receipt 全矩陣。 | CD / Prometheus readback 後補 alert receipt readback：host down、host up、SLO miss、Windows99 blocker、backup stale/failed、deploy 502、freshness stale；完成條件是每類告警都有 sent / received / dedup / escalation evidence。 | | P0-7 | SOURCE_READY_SLA_AUTOMATION | 固定排查順序、ETA / wait reason、自動化判斷與修復 | Scorecard 已固定 `current_phase=host_boot_detection_blocked`、`eta_or_wait_reason=reboot_event_readback_missing_eta_unavailable`、`fixed_triage_order`；但 10 分鐘內自動恢復仍未達標。 | 把每個 blocker 的 next_safe_action、post_verifier、forbidden_actions 接到自動 work item / Telegram / scorecard；完成條件是重啟後自動判斷、主動告警、主動 rerun verifier，不再人工臨場猜流程。 | | P0-8 | PARTIAL_READY_POLICY | Windows99 禁止 Windows Update 無預警重啟 | Scorecard：`windows99_update_no_auto_reboot_ready=true`；但 Windows99 no-secret remote execution channel blocked，policy 證據仍需納入持續監控。 | 保留 readback，補週期性 verifier 與 Telegram drift alert；完成條件是 Windows Update policy drift 會自動告警且不需讀 secret。 | diff --git a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md index 1a592f152..2d6101fe1 100644 --- a/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md +++ b/docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md @@ -62,8 +62,8 @@ | 3 | CIR-P0-RBT-003 | P0 | 「192.168.0.99 VMWare 要自動啟動，裡面 111/188/120/121/112 也自動啟動」 | Windows 99 VMware host autostart + guest VM autostart contract；VM host 111/188/120/121/112 開機順序與 readback | Source verifier / parser / API readback / collection packet 已完成；management probe 讀回 `host_reachable=true`、RDP open、`2179` VMConnect / console channel open、SSH BatchMode `permission_denied`、WinRM timeout；snapshot active blockers=`windows99_remote_execution_channel_unavailable`、`windows99_vmware_autostart_readback_missing` | 收集 local console Verify output 或恢復 no-secret management channel，再確認 `VMRUN_PRESENT`、scheduled task、VMware services、VM power、VMX present 全綠 | | 4 | CIR-P0-RBT-004 | P0 | 「192.168.0.99 不可因 Windows Update 無預警重開」 | Windows Update reboot policy：active hours / no auto-restart / maintenance window / update notification audit | Source verifier 已補 `WINDOWS_UPDATE_POLICY` 與 `WINDOWS_UPDATE_NO_AUTO_REBOOT_READY`；collection packet 已列 forbidden actions；99 management channel 尚不能收 policy readback | 取得 Verify output；若 policy 不綠，再走 controlled apply，禁止要求或記錄 Windows 密碼 | | 5 | CIR-P0-RBT-005 | P0 | 「網站重啟後 502 嚴重影響體驗，要維護頁，外部雲端或專業做法」 | Public maintenance fallback：Nginx / edge / external static maintenance page / status page / fail-open UX，避免 502 直出 | Source + runtime verifier 已實作；Gitea CD `#4459` Success、deploy marker `8d7a6faaf`，production API scorecard 已讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、unreachable without L1=`0`，P0 blockers `11`、readiness `47`；仍不可宣稱整體 10 分鐘 SLO 完成 | 若未來 public route 出現 raw 5xx，先走 L0 Nginx intercept / header verifier；若 edge unreachable，才產生 L1 external static origin / CDN decision record + rollback | -| 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；本輪補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker` 與 metric-missing guard，避免只看到總 SLO 告警；仍需完整 shutdown/up E2E receipt | 部署 alert rules 後讀回 Prometheus / Alertmanager active/resolved 與 outbound receipt；不送測試 secret，不重啟主機 | -| 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram receipt | 部分已有 backup health exporter / alert rules；全域 coverage 與 restore drill 未全綠 | 建立 backup coverage matrix：host / DB / website / service config / package list / tool scripts / logs，每列有 metric、alert、last_success、restore_verifier | +| 6 | CIR-P0-RBT-006 | P0 | 「所有主機關機立刻 Telegram 告警，重啟後也要告警，其他告警一併完整思考」 | Down / shutdown suspected / reboot detected / reboot recovered / SLO missed / backup failed / freshness stale / CPU pressure / Gitea queue 告警矩陣 | HostDown / HostRebootEventDetected / RebootAutoRecoverySLOMissed 已存在；已補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker`、metric-missing guard，以及 2026-07-03 backup alert receipt requirement / stage metrics；仍需完整 shutdown/up E2E receipt | 部署 alert rules 後讀回 Prometheus / Alertmanager active/resolved 與 outbound receipt；backup alert 先補 `/backup/alert-receipts/*.last_success` 脫敏 marker，不送測試 secret、不重啟主機 | +| 7 | CIR-P0-RBT-007 | P0 | 「所有備份包含主機、DB、網站、服務、套件、工具、日誌都沒有監控告警」 | Backup observability coverage：backup job inventory、last success、freshness、offsite、restore drill、Telegram/AwoooP receipt | 已有 backup health exporter / alert rules / Gitea bundle restore dry-run；2026-07-03 新增 `awoooi_backup_alert_receipt_expected_info`、`awoooi_backup_alert_receipt_stage_fresh`、`BackupAlertReceiptStageMissing` 與 source/live contract checker，固定 `sent / received / dedup / escalation` 四段 receipt | 推 CD 後讀回 Prometheus / Alertmanager 是否載入新 rules；再補 Gitea full dump / DB / settings / issues / packages / LFS 與所有工具/log 全量備份監控 | | 8 | CIR-P0-RBT-008 | P0 | 「每次重啟排查都不一樣，也不知道多久恢復，不符合 SLA」 | 固定化 reboot runbook：fixed triage order、ETA、active blocker、remaining seconds、owner lane、next command | Production scorecard readback 已固定 `status=blocked_reboot_auto_recovery_slo_not_ready`、readiness `47%`、active blockers `11`；本輪再補 per-blocker Prometheus metric / Telegram alert 投影，避免只剩總 blocker_count；`next_safe_action=restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot` | 優先收斂 99 no-secret Verify / 111 reachability / backup_core / service_green / Wazuh degraded；不得用不同排查路徑繞過 scorecard | | 9 | CIR-P0-RBT-009 | P0 | 「所有產品、網站都要是最新版本；版本和數據是否最新要驗證」 | Product freshness/version matrix：source commit、deploy marker、runtime image、public health、data freshness、latest source availability | AWOOOI Gitea main 已到 deploy marker `c68b74686`，production source readback `5d5bc86fed` verified；StockPlatform public freshness / ingestion 讀回 `ok`，latest trading date `2026-07-02`，core price/chips/margin/AI recommendations 都是 `2026-07-02` | 建立全產品 readback 表：product、canonical repo、main SHA、deploy marker、public URL、data freshness、blocked reason | | 10 | CIR-P0-GIT-001 | P0 | 「Gitea 儲存庫都不見了？Gitea 沒完整備份嗎？」 | Gitea repository identity + backup proof + restore drill：不能只看 UI visible，要比對 SSH heads、repo path、bundle backup、restore sample | 2026-07-02 production `/api/v1/agents/gitea-repo-bundle-backup-readback` 已 ready：9 expected repos present/ok、missing=0、failed=0、checksum_missing=0、bundle_fresh=true、all_expected_ok=true、sample_restore_dry_run_ok=true；repo bundle / restore dry-run 層已關閉，不是 repo missing。 | 維持每日 bundle backup + restore dry-run monitoring；另補 Gitea full dump / DB / settings / issues / packages / LFS 備份 readback。禁止刪 repo / 改 visibility / 讀 token / restore 到 production | @@ -114,6 +114,7 @@ | Telegram alert AI Loop post-apply verifier + coverage surface | `telegram-alert-learning-context-post-apply-verifier` API / service / tests 已完成；`telegram-alert-monitoring-coverage-readback` API / service / UI 已新增，會把 monitoring inventory、alert_operation_log、AwoooP outbound mirror、AI automation matrix 與 post verifier 合成 coverage readback | | Public maintenance fallback runtime readback | Gitea CD `#4459` / deploy marker `8d7a6faaf` 已讓 production scorecard 讀回 `public_maintenance_fallback.ready=true`、raw 5xx=`0`、P0 blockers `11`、readiness `47` | | Reboot SLO per-blocker 告警投影 | Source 已補 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}`、`RebootAutoRecoveryActiveBlocker`、`RebootAutoRecoveryActiveBlockerMetricMissing` 與契約測試 | +| Backup alert receipt source contract | Source 已補 `awoooi_backup_alert_receipt_expected_info`、`awoooi_backup_alert_receipt_stage_fresh`、`BackupAlertReceiptMetricMissing*`、`BackupAlertReceiptStageMissing`、baseline contract、live visibility checker 與 focused tests；不送 Telegram、不讀 token | ### In Progress @@ -135,7 +136,7 @@ | 10 分鐘 reboot auto-recovery SLA | scorecard source / snapshot / API 已補固定 phase / ETA / blocker / next action 欄位；仍缺 fresh all-host reboot/drill proof | 補 production API readback，缺事件則明確標等待下一次 reboot 或 approved drill | | 99 Windows / VMware autostart | Source verifier / parser / API readback / collection packet 已完成；live 99 Verify output 尚未收集，scorecard 以 `windows99_vmware_autostart_readback_missing` fail-closed；collection packet 顯示 99 reachable、可收 no-secret Verify，但 uptime unknown | 收集 99 no-secret Verify output，確認 VM 111/188/120/121/112 running、scheduled task / services / Windows Update policy 全綠 | | Reboot SLO blocker 收斂 | production scorecard 已固定 `11` blockers / readiness `47`，且 source 已補 per-blocker metric/alert；還沒部署本輪告警規則 | 推 Gitea CD 後部署/讀回 Prometheus rules，再依具名 blocker 收斂 99/111/backup/service/Wazuh | -| 全備份監控告警 coverage | 部分 exporter/rule 已存在，但 host/DB/site/service/package/tool/log coverage 未全列 | 建立 backup coverage matrix 與 restore drill verifier | +| 全備份監控告警 coverage | exporter/rule 已有 host/DB/site/service/package/tool/log coverage 與 backup alert receipt requirement；production receipt markers / full Gitea dump / DB/settings/issues/packages/LFS 尚未全綠 | CD 後讀 Prometheus / Alertmanager；補 `/backup/alert-receipts/*.last_success`、Gitea full dump 與 restore drill verifier | | Stock/Postgres hot pressure | 110 live 已導向 Stock/Postgres playbook；尚未完成 hot query / backup export playbook closure | 下一步執行 read-only Stock/Postgres evidence 與 source freshness / query attribution | ### Blocked / Deferred diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index cd06ea79b..071d2f683 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -1614,6 +1614,56 @@ groups: description: "{{ $labels.domain }} 需要的備份證據 {{ $labels.required_jobs }} 未全部新鮮；重啟恢復不能宣稱備份完整。" runbook: "先跑 `scripts/backup/backup-status.sh --no-notify` 和 backup-health exporter readback，修復對應 required_jobs；禁止刪除、prune、restore 或覆蓋 production。" + - alert: BackupAlertReceiptMetricMissing + expr: absent(awoooi_backup_alert_receipt_expected_info{host="110"}) + for: 20m + labels: + severity: warning + layer: host-backup + component: backup-alert-receipt + host: "110" + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "110 備份告警 receipt requirement 指標缺失" + description: "backup-health exporter 沒有輸出備份告警 sent/received/dedup/escalation receipt requirement；無法證明 backup stale/failed/restore drill 類告警進入 Telegram/AwoooP receipt 鏈。" + runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py，確認 /backup/alert-receipts 只放不含 secret 的 *.last_success marker；不得送測試 Telegram、不得讀 token、不得改 receiver route。" + + - alert: BackupAlertReceiptMetricMissing188 + expr: absent(awoooi_backup_alert_receipt_expected_info{host="188"}) + for: 20m + labels: + severity: warning + layer: host-backup + component: backup-alert-receipt + host: "188" + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "188 備份告警 receipt requirement 指標缺失" + description: "backup-health exporter 沒有輸出 188 Gitea bundle / restore dry-run 告警的 Telegram/AwoooP receipt requirement。" + runbook: "部署新版 scripts/ops/backup-health-textfile-exporter.py 到 188，刷新 /home/ollama/node_exporter_textfiles/backup_health.prom；不得送測試 Telegram、不得讀 token、不得 restore 到 production。" + + - alert: BackupAlertReceiptStageMissing + expr: awoooi_backup_alert_receipt_stage_fresh == 0 + for: 30m + labels: + severity: warning + layer: host-backup + component: backup-alert-receipt + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "備份告警 {{ $labels.required_alert }} 缺少 {{ $labels.stage }} receipt" + description: "{{ $labels.required_alert }} / scope={{ $labels.scope }} 尚無新鮮 {{ $labels.stage }} receipt marker；required_notification_type={{ $labels.required_notification_type }}，channel={{ $labels.receipt_channel }}，max_age_hours={{ $labels.max_age_hours }}。這代表 Telegram/AwoooP sent/received/dedup/escalation 鏈尚未被證明，不能宣稱備份告警閉環。" + runbook: "先讀 /backup/alert-receipts/{{ $labels.required_alert }}.*.last_success 或 AwoooP outbound receipt readback，標準 receipt_channel 為 telegram_awooop；補脫敏 evidence marker 時只允許 timestamp/evidence id，不得寫入 token、chat raw payload 或 secret。" + - alert: BackupAggregateRunFailed expr: sum by (host) (awoooi_backup_last_run_failed_count{host="110",exported_job!="backup_all"}) > 0 for: 10m diff --git a/ops/reboot-recovery/full-stack-backup-baseline.yml b/ops/reboot-recovery/full-stack-backup-baseline.yml index 775149955..19984a7fd 100644 --- a/ops/reboot-recovery/full-stack-backup-baseline.yml +++ b/ops/reboot-recovery/full-stack-backup-baseline.yml @@ -256,6 +256,9 @@ monitoring_contract: - BackupRestoreTestStale - BackupOffsiteCopyNotConfigured - BackupOffsiteCopyStale + - BackupAlertReceiptMetricMissing + - BackupAlertReceiptMetricMissing188 + - BackupAlertReceiptStageMissing - BackupCredentialEscrowEvidenceMissing - BackupRetentionPolicyNotLatestOnly - BackupSnapshotRetentionExceeded @@ -273,6 +276,8 @@ monitoring_contract: - "如果任一 awoooi_backup_cron_singular_entry_ok{host=\"110\"} == 0，Prometheus 與 Alertmanager 必須看得到 BackupScheduleSingletonMismatch。" - "如果任一 awoooi_backup_config_capture_ok{host=\"110\",critical=\"true\"} == 0，Prometheus 與 Alertmanager 必須看得到 BackupConfigCapturePartial，且 target label 必須指出缺哪個設定來源。" - "如果 awoooi_backup_config_capture_status_timestamp 缺失或超過 48 小時，Prometheus 與 Alertmanager 必須看得到 BackupConfigCaptureStatusStale。" + - "如果 awoooi_backup_alert_receipt_expected_info{host=\"110\"} 或 {host=\"188\"} 缺失，Prometheus 與 Alertmanager 必須看得到 BackupAlertReceiptMetricMissing / BackupAlertReceiptMetricMissing188。" + - "如果任一 awoooi_backup_alert_receipt_stage_fresh == 0，Prometheus 與 Alertmanager 必須看得到 BackupAlertReceiptStageMissing，且 required_alert / scope / stage label 必須指出缺哪個 sent / received / dedup / escalation receipt。" - "live visibility check 只讀 Prometheus / Alertmanager API，不送測試告警、不改 silence、不改 route、不觸發修復。" prometheus_recording_rules: - awoooi_recovery_core_ready diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index e7c10644b..279eba52e 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -163,7 +163,9 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "scripts/backup/tests/test_backup_status_contract.py)", "scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py)", "scripts/ops/backup-alert-label-contract-check.py)", + "scripts/ops/backup-alert-live-visibility-check.py)", "scripts/ops/backup-health-textfile-exporter.py)", + "ops/reboot-recovery/full-stack-backup-baseline.yml)", "scripts/ops/tests/test_backup_health_textfile_exporter.py)", "scripts/ops/gitea-queue-hook-backlog-playbook.py)", "scripts/ops/host-runaway-process-exporter.py)", @@ -180,6 +182,7 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "../../scripts/backup/tests/test_backup_status_contract.py", "../../scripts/backup/tests/test_gitea_bundle_sample_restore_dry_run.py", "../../scripts/ops/backup-alert-label-contract-check.py", + "../../scripts/ops/backup-alert-live-visibility-check.py", "../../scripts/ops/gitea-queue-hook-backlog-playbook.py", "../../scripts/ops/host-runaway-process-exporter.py", "../../scripts/ops/host-sustained-load-controller.py", @@ -902,6 +905,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "ops/monitoring/alerts-unified.yml)", "ops/monitoring/alerts.yml)", "ops/alertmanager/alertmanager.yml)", + "ops/reboot-recovery/full-stack-backup-baseline.yml)", "ops/reboot-recovery/full-stack-cold-start-baseline.yml)", "ops/runner/awoooi-cd-lane-drain.service)", "ops/runner/README.md)", @@ -910,6 +914,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "ops/runner/test_check_awoooi_110_controlled_cd_lane_readiness.py)", "ops/runner/test_register_awoooi_110_controlled_cd_lane_drain.py)", "scripts/backup/gitea-repo-bundle-backup.sh)", + "scripts/ops/backup-alert-live-visibility-check.py)", "scripts/ops/backup-health-textfile-exporter.py)", "scripts/ops/docker-disk-pressure-retention-cleanup.py)", "scripts/ops/gitea-queue-hook-backlog-playbook.py)", @@ -946,6 +951,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "scripts/reboot-recovery/tests/test_recover_110_control_path_and_harbor_local.py)", "scripts/reboot-recovery/tests/test_momo_source_arrival_gate.py)", "../../scripts/ops/backup-health-textfile-exporter.py", + "../../scripts/ops/backup-alert-live-visibility-check.py", "../../scripts/ops/docker-disk-pressure-retention-cleanup.py", "../../scripts/ops/gitea-queue-hook-backlog-playbook.py", "../../scripts/ops/host-runaway-process-exporter.py", @@ -956,6 +962,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../ops/monitoring/alerts-unified.yml", "../../ops/monitoring/alerts.yml", "../../ops/alertmanager/alertmanager.yml", + "../../ops/reboot-recovery/full-stack-backup-baseline.yml", "../../ops/reboot-recovery/full-stack-cold-start-baseline.yml", "../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh", "../../ops/runner/register-awoooi-110-controlled-cd-lane-drain.sh", diff --git a/scripts/ops/backup-alert-label-contract-check.py b/scripts/ops/backup-alert-label-contract-check.py index 22a8c1261..8f08c6ebb 100755 --- a/scripts/ops/backup-alert-label-contract-check.py +++ b/scripts/ops/backup-alert-label-contract-check.py @@ -125,6 +125,39 @@ def static_check(path: Path, baseline_path: Path) -> list[str]: _require_contains(_annotation_text(rule), "$labels.item", "BackupCredentialEscrowEvidenceMissing annotations") lines.append("OK BackupCredentialEscrowEvidenceMissing uses item label") + rule = _require_alert(alerts, "BackupAlertReceiptMetricMissing") + _require_contains( + str(rule.get("expr", "")), + "awoooi_backup_alert_receipt_expected_info", + "BackupAlertReceiptMetricMissing expr", + ) + lines.append("OK BackupAlertReceiptMetricMissing checks receipt requirement metric") + + rule = _require_alert(alerts, "BackupAlertReceiptMetricMissing188") + _require_contains( + str(rule.get("expr", "")), + "awoooi_backup_alert_receipt_expected_info", + "BackupAlertReceiptMetricMissing188 expr", + ) + lines.append("OK BackupAlertReceiptMetricMissing188 checks 188 receipt requirement metric") + + rule = _require_alert(alerts, "BackupAlertReceiptStageMissing") + _require_contains( + str(rule.get("expr", "")), + "awoooi_backup_alert_receipt_stage_fresh == 0", + "BackupAlertReceiptStageMissing expr", + ) + text = _annotation_text(rule) + for required_label in [ + "$labels.required_alert", + "$labels.scope", + "$labels.stage", + "$labels.required_notification_type", + "$labels.receipt_channel", + ]: + _require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations") + lines.append("OK BackupAlertReceiptStageMissing uses required_alert/scope/stage/receipt labels") + return lines @@ -230,6 +263,11 @@ def live_check(base_url: str, baseline_path: Path) -> list[str]: 'awoooi_backup_config_capture_ok{host="110"}', {"host", "target", "source", "critical"}, ), + _require_live_label( + base_url, + 'awoooi_backup_alert_receipt_stage_expected_info{host="110"}', + {"host", "required_alert", "scope", "stage", "required_notification_type", "receipt_channel"}, + ), ] lines.extend(_require_live_rules(base_url, _expected_backup_alerts(baseline_path))) return lines diff --git a/scripts/ops/backup-alert-live-visibility-check.py b/scripts/ops/backup-alert-live-visibility-check.py index 7ec765ec5..de3d3d782 100755 --- a/scripts/ops/backup-alert-live-visibility-check.py +++ b/scripts/ops/backup-alert-live-visibility-check.py @@ -187,6 +187,54 @@ def _escrow_required_alerts(prometheus_url: str, host: str, timeout: int) -> lis return sorted(required, key=lambda alert: alert.labels["item"]) +def _receipt_stage_required_alerts(prometheus_url: str, host: str, timeout: int) -> list[RequiredAlert]: + expr = f'awoooi_backup_alert_receipt_stage_fresh{{host="{host}"}} == 0' + rows = _prom_query(prometheus_url, expr, timeout) + required: list[RequiredAlert] = [] + for row in rows: + labels = _metric_labels(row) + required_alert = labels.get("required_alert") + scope = labels.get("scope") + stage = labels.get("stage") + receipt_channel = labels.get("receipt_channel") + required_notification_type = labels.get("required_notification_type") + missing = [ + label + for label, value in { + "required_alert": required_alert, + "scope": scope, + "stage": stage, + "receipt_channel": receipt_channel, + "required_notification_type": required_notification_type, + }.items() + if not value + ] + if missing: + raise VisibilityError(f"Backup alert receipt gap metric missing labels {missing}: {row}") + required.append( + RequiredAlert( + "BackupAlertReceiptStageMissing", + { + **COMMON_LABELS, + "host": host, + "required_alert": str(required_alert), + "scope": str(scope), + "stage": str(stage), + "receipt_channel": str(receipt_channel), + "required_notification_type": str(required_notification_type), + }, + ) + ) + return sorted( + required, + key=lambda alert: ( + alert.labels["required_alert"], + alert.labels["scope"], + alert.labels["stage"], + ), + ) + + def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: int) -> list[str]: required_alerts: list[RequiredAlert] = [] lines: list[str] = [] @@ -205,6 +253,20 @@ def live_check(prometheus_url: str, alertmanager_url: str, host: str, timeout: i else: lines.append("OK credential escrow markers are fresh; no escrow gap alert required") + receipt_stage_alerts = _receipt_stage_required_alerts(prometheus_url, host, timeout) + required_alerts.extend(receipt_stage_alerts) + if receipt_stage_alerts: + samples = ", ".join( + f"{alert.labels['required_alert']}:{alert.labels['stage']}" + for alert in receipt_stage_alerts[:8] + ) + suffix = "" if len(receipt_stage_alerts) <= 8 else f", ... +{len(receipt_stage_alerts) - 8}" + lines.append( + f"OK backup alert receipt gap metrics require {len(receipt_stage_alerts)} alert(s): {samples}{suffix}" + ) + else: + lines.append("OK backup alert receipt markers are fresh; no receipt stage alert required") + prom_alerts = _prom_alerts(prometheus_url, timeout) for required in required_alerts: _require_prom_alert(prom_alerts, required) diff --git a/scripts/ops/backup-health-textfile-exporter.py b/scripts/ops/backup-health-textfile-exporter.py index bc7ef6998..280b49704 100755 --- a/scripts/ops/backup-health-textfile-exporter.py +++ b/scripts/ops/backup-health-textfile-exporter.py @@ -34,6 +34,8 @@ BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/sc OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite")) ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence")) CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json")) +BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts")) +BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168")) DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [ "wooo/awoooi", "wooo/ewoooc", @@ -55,6 +57,34 @@ ESCROW_ITEMS = [ "dns_registrar_recovery", "oauth_ai_provider_recovery", ] +BACKUP_ALERT_RECEIPT_STAGES = ("sent", "received", "dedup", "escalation") +BACKUP_ALERT_RECEIPT_REQUIREMENTS = [ + {"host": "110", "alertname": "BackupHealthMonitorMissing110", "scope": "monitor"}, + {"host": "188", "alertname": "BackupHealthMonitorMissing188", "scope": "monitor"}, + {"host": "110", "alertname": "BackupHealthMonitorStale", "scope": "monitor"}, + {"host": "110", "alertname": "BackupExpectedJobMissing", "scope": "schedule"}, + {"host": "110", "alertname": "BackupScriptMissing", "scope": "script"}, + {"host": "110", "alertname": "BackupJobStale", "scope": "freshness"}, + {"host": "110", "alertname": "BackupCoverageDomainStale", "scope": "coverage"}, + {"host": "188", "alertname": "GiteaPrivateBundleBackupIncomplete", "scope": "gitea_bundle"}, + {"host": "188", "alertname": "GiteaPrivateBundleRestoreDryRunMissing", "scope": "gitea_bundle_restore"}, + {"host": "110", "alertname": "BackupAggregateRunFailed", "scope": "aggregate"}, + {"host": "110", "alertname": "BackupConfigCapturePartial", "scope": "config_capture"}, + {"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"}, + {"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"}, + {"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"}, + {"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"}, + {"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"}, + {"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"}, + {"host": "110", "alertname": "BackupRestoreTestStale", "scope": "velero_restore_test"}, + {"host": "110", "alertname": "BackupOffsiteCopyNotConfigured", "scope": "offsite"}, + {"host": "110", "alertname": "BackupOffsiteCopyStale", "scope": "offsite"}, + {"host": "110", "alertname": "BackupRetentionPolicyNotLatestOnly", "scope": "retention"}, + {"host": "110", "alertname": "BackupSnapshotRetentionExceeded", "scope": "retention"}, + {"host": "110", "alertname": "BackupOffsiteFullVerifyFailed", "scope": "offsite_verify"}, + {"host": "110", "alertname": "BackupOffsiteRemoteSnapshotRetentionExceeded", "scope": "offsite_retention"}, + {"host": "110", "alertname": "BackupCredentialEscrowEvidenceMissing", "scope": "credential_escrow"}, +] def _escape_label(value: str) -> str: @@ -106,6 +136,55 @@ def _marker_timestamp(paths: list[Path]) -> int: return 0 +def _backup_alert_receipt_marker_timestamp(alertname: str, stage: str) -> int: + return _marker_timestamp( + [ + BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.{stage}.last_success", + BACKUP_ALERT_RECEIPT_DIR / alertname / f"{stage}.last_success", + BACKUP_ALERT_RECEIPT_DIR / f"{alertname}.last_success", + BACKUP_ALERT_RECEIPT_DIR / alertname / "last_success", + ] + ) + + +def _backup_alert_receipt_metric_lines(host: str) -> list[str]: + now = int(time.time()) + lines: list[str] = [] + for requirement in BACKUP_ALERT_RECEIPT_REQUIREMENTS: + if requirement["host"] != host: + continue + alertname = requirement["alertname"] + scope = requirement["scope"] + base_labels = ( + f'host="{_escape_label(host)}",' + f'required_alert="{_escape_label(alertname)}",' + f'scope="{_escape_label(scope)}",' + 'required_notification_type="TYPE-3",' + 'receipt_channel="telegram_awooop",' + f'max_age_hours="{BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS:g}"' + ) + lines.append(f"awoooi_backup_alert_receipt_expected_info{{{base_labels}}} 1") + + stage_fresh_values: list[int] = [] + for stage in BACKUP_ALERT_RECEIPT_STAGES: + timestamp = _backup_alert_receipt_marker_timestamp(alertname, stage) + age = now - timestamp if timestamp else 0 + fresh = 1 if timestamp and age <= int(BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS * 3600) else 0 + stage_fresh_values.append(fresh) + stage_labels = f'{base_labels},stage="{_escape_label(stage)}"' + lines.extend( + [ + f"awoooi_backup_alert_receipt_stage_expected_info{{{stage_labels}}} 1", + f"awoooi_backup_alert_receipt_stage_last_success_timestamp{{{stage_labels}}} {timestamp}", + f"awoooi_backup_alert_receipt_stage_age_seconds{{{stage_labels}}} {age}", + f"awoooi_backup_alert_receipt_stage_fresh{{{stage_labels}}} {fresh}", + ] + ) + + lines.append(f"awoooi_backup_alert_receipt_fresh{{{base_labels}}} {1 if all(stage_fresh_values) else 0}") + return lines + + def _shell_export_value(path: Path, key: str) -> str: try: lines = path.read_text(encoding="utf-8", errors="replace").splitlines() @@ -860,6 +939,18 @@ def _base_lines(host: str) -> list[str]: "# TYPE awoooi_backup_coverage_domain_expected_info gauge", "# HELP awoooi_backup_coverage_domain_fresh Whether every required evidence job for a backup coverage domain is fresh.", "# TYPE awoooi_backup_coverage_domain_fresh gauge", + "# HELP awoooi_backup_alert_receipt_expected_info Backup alert receipt requirements that must have Telegram/AwoooP evidence.", + "# TYPE awoooi_backup_alert_receipt_expected_info gauge", + "# HELP awoooi_backup_alert_receipt_fresh Whether every required receipt stage for a backup alert is fresh.", + "# TYPE awoooi_backup_alert_receipt_fresh gauge", + "# HELP awoooi_backup_alert_receipt_stage_expected_info Required sent/received/dedup/escalation receipt stage for a backup alert.", + "# TYPE awoooi_backup_alert_receipt_stage_expected_info gauge", + "# HELP awoooi_backup_alert_receipt_stage_last_success_timestamp Unix timestamp of the backup alert receipt stage evidence marker.", + "# TYPE awoooi_backup_alert_receipt_stage_last_success_timestamp gauge", + "# HELP awoooi_backup_alert_receipt_stage_age_seconds Age of the backup alert receipt stage evidence marker.", + "# TYPE awoooi_backup_alert_receipt_stage_age_seconds gauge", + "# HELP awoooi_backup_alert_receipt_stage_fresh Whether a backup alert receipt stage marker is fresh.", + "# TYPE awoooi_backup_alert_receipt_stage_fresh gauge", "# HELP awoooi_backup_job_configured Whether the expected backup cron/config is present.", "# TYPE awoooi_backup_job_configured gauge", "# HELP awoooi_backup_script_present Whether the backup script exists on this host.", @@ -1107,6 +1198,7 @@ def _collect_110(host: str) -> list[str]: lines.extend(_retention_metric_lines(host)) lines.extend(_cron_duplicate_metric_lines(host, cron)) lines.extend(_velero_metric_lines(host)) + lines.extend(_backup_alert_receipt_metric_lines(host)) return lines @@ -1183,6 +1275,7 @@ def _collect_188(host: str) -> list[str]: sample_count=1 if momo_ts else 0, ) ) + lines.extend(_backup_alert_receipt_metric_lines(host)) return lines diff --git a/scripts/ops/tests/test_backup_health_textfile_exporter.py b/scripts/ops/tests/test_backup_health_textfile_exporter.py index 97d342b35..4c77eebb7 100644 --- a/scripts/ops/tests/test_backup_health_textfile_exporter.py +++ b/scripts/ops/tests/test_backup_health_textfile_exporter.py @@ -165,3 +165,65 @@ def test_dr_phase_ready_when_full_offsite_is_fresh_and_escrow_is_complete( ) assert 'awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0' in rendered assert 'awoooi_backup_dr_phase{host="110",next_step="offsite_and_escrow_ready"} 5' in rendered + + +def test_backup_alert_receipt_metrics_require_all_stages(tmp_path: Path, monkeypatch) -> None: + exporter = load_exporter() + receipt_dir = tmp_path / "alert-receipts" + receipt_dir.mkdir() + now = 1_782_900_000 + + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir) + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168) + monkeypatch.setattr(exporter.time, "time", lambda: now) + + for stage in exporter.BACKUP_ALERT_RECEIPT_STAGES[:-1]: + (receipt_dir / f"BackupJobStale.{stage}.last_success").write_text( + str(now - 60), + encoding="utf-8", + ) + + metrics = exporter._backup_alert_receipt_metric_lines("110") + rendered = "\n".join(metrics) + + assert 'awoooi_backup_alert_receipt_expected_info{host="110",required_alert="BackupJobStale"' in rendered + assert 'awoooi_backup_alert_receipt_stage_expected_info{host="110",required_alert="BackupJobStale"' in rendered + assert 'stage="sent"' in rendered + assert 'stage="received"' in rendered + assert 'stage="dedup"' in rendered + assert 'stage="escalation"' in rendered + assert ( + 'awoooi_backup_alert_receipt_stage_fresh{host="110",required_alert="BackupJobStale",scope="freshness",' + 'required_notification_type="TYPE-3",receipt_channel="telegram_awooop",max_age_hours="168",stage="escalation"} 0' + in rendered + ) + assert ( + 'awoooi_backup_alert_receipt_fresh{host="110",required_alert="BackupJobStale",scope="freshness",' + 'required_notification_type="TYPE-3",receipt_channel="telegram_awooop",max_age_hours="168"} 0' + in rendered + ) + + +def test_backup_alert_receipt_shared_marker_can_satisfy_all_stages(tmp_path: Path, monkeypatch) -> None: + exporter = load_exporter() + receipt_dir = tmp_path / "alert-receipts" + receipt_dir.mkdir() + now = 1_782_900_000 + + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_DIR", receipt_dir) + monkeypatch.setattr(exporter, "BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", 168) + monkeypatch.setattr(exporter.time, "time", lambda: now) + (receipt_dir / "GiteaPrivateBundleRestoreDryRunMissing.last_success").write_text( + str(now - 60), + encoding="utf-8", + ) + + metrics = exporter._backup_alert_receipt_metric_lines("188") + rendered = "\n".join(metrics) + + assert ( + 'awoooi_backup_alert_receipt_fresh{host="188",required_alert="GiteaPrivateBundleRestoreDryRunMissing",' + 'scope="gitea_bundle_restore",required_notification_type="TYPE-3",receipt_channel="telegram_awooop",' + 'max_age_hours="168"} 1' + in rendered + ) diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index a9bd83b0b..00f15058a 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -333,10 +333,23 @@ def test_backup_exporter_emits_domain_level_backup_coverage() -> None: assert "awoooi_backup_coverage_domain_expected_info" in exporter assert "awoooi_backup_coverage_domain_fresh" in exporter + assert "awoooi_backup_alert_receipt_expected_info" in exporter + assert "awoooi_backup_alert_receipt_stage_fresh" in exporter + assert "BACKUP_ALERT_RECEIPT_STAGES = (\"sent\", \"received\", \"dedup\", \"escalation\")" in exporter + assert "AIOPS_BACKUP_ALERT_RECEIPT_DIR" in exporter assert "gitea_repo_mirror_from_110" in exporter assert "/home/ollama/backup/110/gitea" in exporter for domain in ["host", "database", "website", "service", "package", "tool", "log"]: assert f'"{domain}"' in exporter + for alertname in [ + "BackupJobStale", + "BackupCoverageDomainStale", + "GiteaPrivateBundleBackupIncomplete", + "GiteaPrivateBundleRestoreDryRunMissing", + "BackupRestoreDrillMissingOrFailed", + "BackupCredentialEscrowEvidenceMissing", + ]: + assert alertname in exporter def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: @@ -347,6 +360,13 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None: assert "awoooi_backup_coverage_domain_fresh == 0" in alerts assert "GiteaPrivateBundleRestoreDryRunMissing" in alerts assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in alerts + assert "BackupAlertReceiptMetricMissing" in alerts + assert "BackupAlertReceiptStageMissing" in alerts + assert "awoooi_backup_alert_receipt_expected_info" in alerts + assert "awoooi_backup_alert_receipt_stage_fresh == 0" in alerts + assert "$labels.required_alert" in alerts + assert "$labels.required_notification_type" in alerts + assert "telegram_awooop" in alerts def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None: