From a21d0161331c5ae93f9afa80dda52474d5723db4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 2 Jul 2026 20:01:49 +0800 Subject: [PATCH] fix(alerts): restore alertmanager webhook route [metadata-only] --- .gitea/workflows/cd.yaml | 9 +++++ docs/LOGBOOK.md | 20 ++++++++++ ops/alertmanager/alertmanager.yml | 13 +++---- .../test_cd_controlled_runtime_profile.py | 10 +++++ .../tests/test_alertmanager_webhook_config.py | 39 +++++++++++++++++++ 5 files changed, 84 insertions(+), 7 deletions(-) create mode 100644 scripts/ops/tests/test_alertmanager_webhook_config.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index e0b4bc5cb..394cf4dac 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -589,6 +589,8 @@ jobs: ;; ops/monitoring/alerts.yml) ;; + ops/alertmanager/alertmanager.yml) + ;; scripts/dev/awoooi-navigation-coverage-guard.py) ;; scripts/ci/wait-host-web-build-pressure.sh) @@ -625,6 +627,8 @@ jobs: ;; scripts/ops/host-sustained-load-evidence.py) ;; + scripts/ops/deploy-alertmanager-config.sh) + ;; scripts/ops/notify-awoooi-ops.sh) ;; scripts/ops/pg-backup.sh) @@ -639,6 +643,8 @@ jobs: ;; scripts/ops/tests/test_host_pressure_alert_contract.py) ;; + scripts/ops/tests/test_alertmanager_webhook_config.py) + ;; scripts/reboot-recovery/deploy-to-110.sh) ;; scripts/reboot-recovery/deploy-to-188.sh) @@ -903,6 +909,7 @@ jobs: ../../scripts/security/telegram-notification-egress-owner-response-acceptance.py python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts-unified.yml')); print('alerts-unified YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/monitoring/alerts.yml')); print('alerts YAML OK')" + python3.11 -c "import yaml; yaml.safe_load(open('../../ops/alertmanager/alertmanager.yml')); print('alertmanager YAML OK')" python3.11 -c "import yaml; yaml.safe_load(open('../../ops/reboot-recovery/full-stack-cold-start-baseline.yml')); print('full-stack-cold-start-baseline YAML OK')" bash -n \ ../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh \ @@ -935,6 +942,7 @@ jobs: ../../scripts/backup/backup-status.sh \ ../../scripts/backup/gitea-repo-bundle-backup.sh \ ../../scripts/ops/backup-from-110.sh \ + ../../scripts/ops/deploy-alertmanager-config.sh \ ../../scripts/ops/docker-health-monitor.sh \ ../../scripts/ops/dr-drill.sh \ ../../scripts/ops/notify-awoooi-ops.sh \ @@ -996,6 +1004,7 @@ jobs: ../../scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py \ ../../scripts/ops/tests/test_host_runaway_process_exporter.py \ ../../scripts/ops/tests/test_host_pressure_alert_contract.py \ + ../../scripts/ops/tests/test_alertmanager_webhook_config.py \ ../../scripts/reboot-recovery/tests/test_dr_escrow_evidence_checklist.py \ ../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py \ ../../scripts/reboot-recovery/tests/test_momo_source_arrival_gate.py \ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 23c30e66c..d8f8ba156 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,23 @@ +## 2026-07-02 — 20:02 P0-006 Alertmanager 主告警鏈路恢復 public webhook + +**完成內容**： +- 110 CPU / reboot SLO 告警已在 Prometheus 偵測並 `firing`，問題不是未偵測，而是 Alertmanager 主 webhook 仍指向 `192.168.0.125:32334` dead VIP / NodePort，導致告警停在 Prometheus / Alertmanager，未穩定進 AWOOOI API -> TelegramGateway receipt path。 +- `ops/alertmanager/alertmanager.yml` 主 receiver `awoooi-webhook` 已改為 `https://awoooi.wooo.work/api/v1/webhooks/alertmanager`，並保留 emergency `telegram-direct` 只限 `AWOOOIApiDown` / `AlertmanagerDown` critical matchers。 +- `.gitea/workflows/cd.yaml` 已把 Alertmanager config / deploy script / webhook config test 納入 controlled-runtime allowlist、YAML parse、`bash -n` 與 focused pytest，避免這類告警鏈修復掉回 full/B5 或重型 Docker build。 +- `scripts/ops/tests/test_alertmanager_webhook_config.py` 新增 regression，禁止主 webhook 回到 dead VIP，並鎖住 emergency direct Telegram 只限告警鏈自救。 +- live 110 已受控套用同一 URL 修復：備份 `/home/wooo/monitoring/alertmanager.yml`、`amtool check-config` 通過、Alertmanager HUP reload 完成、container 維持 `running`。 + +**驗證**： +- `python3.11 -m pytest scripts/ops/tests/test_alertmanager_webhook_config.py scripts/ops/tests/test_host_pressure_alert_contract.py ops/runner/test_cd_controlled_runtime_profile.py -q -p no:cacheprovider`：`50 passed`。 +- `python3.11 -c "import yaml; yaml.safe_load(open('ops/alertmanager/alertmanager.yml'))"`：通過。 +- `bash -n scripts/ops/deploy-alertmanager-config.sh`：通過。 +- `python3 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK ... auto_branch_events_on_110=0 generic_runner_labels=0`。 +- live readback：Alertmanager ready HTTP `200`；public webhook probe HTTP `200` / `No firing alerts to process`；Prometheus 仍讀回 `Host110SustainedModeratePressure` firing `3`、`HostRebootEventDetected` firing `1`、`RebootAutoRecoverySLOMissed` firing `1`。 + +**仍維持**： +- 未讀 Telegram token / secret / `.env` / raw sessions / SQLite / auth；未呼叫 direct Bot API；未 workflow_dispatch；未重啟主機 / VM / Docker daemon / Nginx / K3s / DB / firewall；未 fake DB rows；未宣稱 10 分鐘全恢復完成。 +- P0-006 仍 blocked：reboot scorecard `readiness_percent=21`、`active_blocker_count=15`；priority readback `active_p0_readiness_percent=43`，StockPlatform freshness / ingestion 仍因 `core_margin_short_daily_missing`、`ai_recommendations_stale`、`core.margin_short_daily_incomplete` blocked。 + ## 2026-07-02 — 19:58 CIR-P0-TG-001 AI alert card learning writeback refs **完成內容**： diff --git a/ops/alertmanager/alertmanager.yml b/ops/alertmanager/alertmanager.yml index fcfcf64f5..760140cb2 100644 --- a/ops/alertmanager/alertmanager.yml +++ b/ops/alertmanager/alertmanager.yml @@ -1,7 +1,7 @@ # AWOOOI Alertmanager 配置 # 2026-04-05 Claude Code: 修正 webhook URL # 修正前: http://192.168.0.188:8088/api/v1/webhook/alertmanager (OpenClaw，舊系統，錯誤) -# 修正後: http://192.168.0.121:32334/api/v1/webhooks/alertmanager (AWOOOI API，複數，正確) +# 修正後: https://awoooi.wooo.work/api/v1/webhooks/alertmanager (AWOOOI public API，複數，正確) # 根據 feedback_alertmanager_awoooi_flow.md 鐵律 # 2026-04-09 Claude Sonnet 4.6 Asia/Taipei: 新增 Telegram Fallback (ADR-035) # 架構: awoooi-webhook (主路徑) + telegram-direct (告警鏈路緊急旁路) @@ -53,14 +53,13 @@ route: receivers: # 主路徑: AWOOOI API 處理所有告警 (AI 分析 + 去重 + Telegram) - # 2026-04-16 ogt + Claude Sonnet 4.6: 改指向 VIP 192.168.0.125 - # 根因: 121:32334 Connection Refused，120:32334 也 Refused - # 只有 VIP 125:32334 可連通（kube-proxy NodePort 路由正常） - # ⚠️ SPF-1 風險：VIP 125 為單點，VIP host 整機 down → 主鏈斷 - # 緩解計畫見 critic SPF 治理設計（中度方案：webhook_configs 多 url round-robin） + # 2026-07-02 Codex: 125:32334 已 connection refused；告警仍 firing 但 + # Alertmanager → AWOOOI API → TelegramGateway 主鏈斷。主路徑改走 public + # HTTPS route，避免單一 K8s VIP / NodePort 漂移讓 110 CPU / reboot SLO + # 告警只停在 Prometheus / Alertmanager。 - name: 'awoooi-webhook' webhook_configs: - - url: 'http://192.168.0.125:32334/api/v1/webhooks/alertmanager' + - url: 'https://awoooi.wooo.work/api/v1/webhooks/alertmanager' send_resolved: true # Emergency 路徑: AWOOOI API / AlertChain 自身異常時直接送 Telegram 到 SRE 群組。 diff --git a/ops/runner/test_cd_controlled_runtime_profile.py b/ops/runner/test_cd_controlled_runtime_profile.py index 1707a37c6..33ff23af5 100644 --- a/ops/runner/test_cd_controlled_runtime_profile.py +++ b/ops/runner/test_cd_controlled_runtime_profile.py @@ -149,9 +149,11 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "scripts/ops/host-runaway-process-exporter.py)", "scripts/ops/host-sustained-load-controller.py)", "scripts/ops/host-sustained-load-evidence.py)", + "scripts/ops/deploy-alertmanager-config.sh)", "scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py)", "scripts/ops/tests/test_host_runaway_process_exporter.py)", "scripts/ops/tests/test_host_pressure_alert_contract.py)", + "scripts/ops/tests/test_alertmanager_webhook_config.py)", "../../scripts/backup/backup-awoooi-frequent.sh", "../../scripts/backup/backup-status.sh", "../../scripts/backup/tests/test_backup_status_contract.py", @@ -160,10 +162,12 @@ def test_backup_freshness_sources_stay_on_controlled_runtime_profile() -> None: "../../scripts/ops/host-runaway-process-exporter.py", "../../scripts/ops/host-sustained-load-controller.py", "../../scripts/ops/host-sustained-load-evidence.py", + "../../scripts/ops/deploy-alertmanager-config.sh", "../../scripts/ops/tests/test_backup_health_textfile_exporter.py", "../../scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py", "../../scripts/ops/tests/test_host_runaway_process_exporter.py", "../../scripts/ops/tests/test_host_pressure_alert_contract.py", + "../../scripts/ops/tests/test_alertmanager_webhook_config.py", ] for source in expected_sources: assert source in text @@ -830,6 +834,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "apps/api/src/api/v1/platform/operator_runs.py)", "ops/monitoring/alerts-unified.yml)", "ops/monitoring/alerts.yml)", + "ops/alertmanager/alertmanager.yml)", "ops/reboot-recovery/full-stack-cold-start-baseline.yml)", "ops/runner/awoooi-cd-lane-drain.service)", "ops/runner/README.md)", @@ -843,8 +848,10 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "scripts/ops/gitea-queue-hook-backlog-playbook.py)", "scripts/ops/host-runaway-process-exporter.py)", "scripts/ops/host-sustained-load-evidence.py)", + "scripts/ops/deploy-alertmanager-config.sh)", "scripts/ops/tests/test_gitea_queue_hook_backlog_playbook.py)", "src/api/v1/platform/operator_runs.py", + "scripts/ops/tests/test_alertmanager_webhook_config.py)", "scripts/reboot-recovery/deploy-to-110.sh)", "scripts/reboot-recovery/enforce-110-runner-failclosed.sh)", "scripts/reboot-recovery/recover-110-control-path-and-harbor-local.sh)", @@ -877,9 +884,11 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/ops/host-runaway-process-exporter.py", "../../scripts/ops/host-sustained-load-controller.py", "../../scripts/ops/host-sustained-load-evidence.py", + "../../scripts/ops/deploy-alertmanager-config.sh", "../../scripts/backup/gitea-repo-bundle-backup.sh", "../../ops/monitoring/alerts-unified.yml", "../../ops/monitoring/alerts.yml", + "../../ops/alertmanager/alertmanager.yml", "../../ops/reboot-recovery/full-stack-cold-start-baseline.yml", "../../ops/runner/check-awoooi-110-controlled-cd-lane-readiness.sh", "../../ops/runner/register-awoooi-110-controlled-cd-lane-drain.sh", @@ -904,6 +913,7 @@ def test_post_start_recovery_verifiers_stay_on_controlled_runtime_profile() -> N "../../scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py", "../../scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py", "../../scripts/ops/tests/test_host_runaway_process_exporter.py", + "../../scripts/ops/tests/test_alertmanager_webhook_config.py", "../../scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py", "../../scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py", "../../scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py", diff --git a/scripts/ops/tests/test_alertmanager_webhook_config.py b/scripts/ops/tests/test_alertmanager_webhook_config.py new file mode 100644 index 000000000..2c17eab16 --- /dev/null +++ b/scripts/ops/tests/test_alertmanager_webhook_config.py @@ -0,0 +1,39 @@ +from __future__ import annotations + +from pathlib import Path + +import yaml + + +ROOT = Path(__file__).resolve().parents[3] +ALERTMANAGER_CONFIG = ROOT / "ops" / "alertmanager" / "alertmanager.yml" + + +def _config() -> dict: + return yaml.safe_load(ALERTMANAGER_CONFIG.read_text(encoding="utf-8")) + + +def _receiver(name: str) -> dict: + return next(item for item in _config()["receivers"] if item["name"] == name) + + +def test_awoooi_webhook_uses_public_api_route_not_dead_vip() -> None: + receiver = _receiver("awoooi-webhook") + urls = [ + item["url"] + for item in receiver.get("webhook_configs", []) + if item.get("url") + ] + + assert urls == ["https://awoooi.wooo.work/api/v1/webhooks/alertmanager"] + assert all("192.168.0.125:32334" not in url for url in urls) + assert all("/api/v1/webhooks/alertmanager" in url for url in urls) + + +def test_emergency_direct_telegram_stays_limited_to_alert_chain() -> None: + receiver = _receiver("telegram-direct") + assert receiver.get("telegram_configs") + + route_text = str(_config()["route"]) + assert "AWOOOIApiDown" in route_text + assert 'severity="critical"' in route_text