wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/main' into codex/p0-obs-002-20260715

Browse Source
This commit is contained in:
ogt
2026-07-15 02:30:04 +08:00
parent bfb2d2d040 3d8fa0200e
commit 4cf66c2a7e
17 changed files with 524 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
ops/monitoring/alerts-unified.yml
View File

@@ -24,16 +24,16 @@ groups:
- record: awoooi_recovery_core_ready
expr: |
sum without(result) (
awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} == bool 1
awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} == bool 1
)
* on(host,scope) (
awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} == bool 0
awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} == bool 0
)
* on(host,scope) (
awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} == bool 0
awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} == bool 0
)
* on(host,scope) (
(time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"}) < bool 3600
(time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"}) < bool 3600
)
- record: awoooi_recovery_dr_offsite_ready
@@ -2326,7 +2326,7 @@ groups:
runbook: "檢查 `/home/wooo/logs/prometheus-rule-drift-guard.log` 與 `/home/wooo/monitoring/alerts.yml.guard.bak.*`，找出誰覆寫了 active rules。"
- alert: ColdStartMonitorMissing
expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188"})
expr: absent(awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188"})
for: 15m
labels:
severity: warning
@@ -2342,7 +2342,7 @@ groups:
runbook: "執行 scripts/reboot-recovery/install-cold-start-monitor-110.sh，確認 /home/wooo/node_exporter_textfiles/cold_start_recovery.prom"
- alert: ColdStartMonitorStale
expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_120_121_188"} > 900
expr: time() - awoooi_cold_start_last_run_timestamp{host="110",scope="110_112_120_121_188"} > 900
for: 10m
labels:
severity: warning
@@ -2358,7 +2358,7 @@ groups:
runbook: "SSH 110 檢查 crontab、/tmp/awoooi-cold-start-monitor.cron.log、cold-start-last.log"
- alert: ColdStartRecoveryBlocked
expr: awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"} > 0
expr: awoooi_cold_start_blocked_gates{host="110",scope="110_112_120_121_188"} > 0
for: 5m
labels:
severity: critical
@@ -2374,7 +2374,7 @@ groups:
runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log；依 docs/runbooks/FULL-STACK-COLD-START-SOP.md 的 P0→P2 順序修復"
- alert: K3sNodeFilesystemErrorGateBlocked
expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0
expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="k3s_node_filesystem_error",target="120"} > 0
for: 5m
labels:
severity: critical
@@ -2392,7 +2392,7 @@ groups:
runbook: "查看 110 `/home/wooo/reboot-recovery/cold-start-last.log`，執行 `scripts/reboot-recovery/120-fsck-maintenance-checklist.sh` 做維護前只讀檢查；維護窗內用 console/rescue 對 120 root LV 執行 fsck，禁止 online fsck。"
- alert: ColdStartHost120Unreachable
expr: awoooi_cold_start_blocker_reason{host="110",scope="110_120_121_188",reason="host_unreachable",target="120"} > 0
expr: awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="host_unreachable",target="120"} > 0
for: 3m
labels:
severity: critical
@@ -2410,7 +2410,7 @@ groups:
runbook: "查看 120 console。若停在 initramfs/manual fsck，先對 root LV 做離線 fsck；若主機關機或網卡異常，先恢復電源/網路，再重跑 full-stack cold-start gate。禁止從自動修復直接重啟其他服務掩蓋主機離線。"
- alert: ColdStartRecoveryDegraded
expr: awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"} > 0
expr: awoooi_cold_start_warn_gates{host="110",scope="110_112_120_121_188"} > 0
for: 15m
labels:
severity: warning
@@ -2426,7 +2426,7 @@ groups:
runbook: "查看 /home/wooo/reboot-recovery/cold-start-last.log，修到 PASS/WARN/BLOCKED = green"
- alert: ColdStartLastGreenTooOld
expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_120_121_188"} > 3600
expr: time() - awoooi_cold_start_last_green_timestamp{host="110",scope="110_112_120_121_188"} > 3600
for: 15m
labels:
severity: warning
@@ -2438,7 +2438,7 @@ groups:
auto_repair: "false"
annotations:
summary: "Full-stack cold-start gate has not been GREEN recently"
description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/120/121/188 與排程/網站 gate。"
description: "距離上次 GREEN 已超過 {{ $value | humanizeDuration }}；需要確認 110/112/120/121/188 與排程/網站 gate。"
runbook: "執行 SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test"
# =========================================================================

2
ops/reboot-recovery/full-stack-cold-start-baseline.yml
View File

@@ -351,7 +351,7 @@ authoritative_checks:
expected: PASS=64 WARN=0 BLOCKED=0
textfile_metric:
path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom
green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1
green_metric: awoooi_cold_start_last_result{host="110",scope="110_112_120_121_188",result="green"} 1
backup_baseline:
path: ops/reboot-recovery/full-stack-backup-baseline.yml
required_metrics: