diff --git a/apps/api/src/plugins/mcp/providers/ssh_provider.py b/apps/api/src/plugins/mcp/providers/ssh_provider.py index 2e7458466..f21b1f8bc 100644 --- a/apps/api/src/plugins/mcp/providers/ssh_provider.py +++ b/apps/api/src/plugins/mcp/providers/ssh_provider.py @@ -126,8 +126,13 @@ GROUP_B_TOOLS = { "ssh_clear_docker_logs", "ssh_renew_ssl", "ssh_reload_nginx", + "ssh_docker_prune", } +# Disk usage gate for ssh_docker_prune (only run when usage >= this %) +# 2026-05-02 ogt + Claude Sonnet 4.6: prevent accidental prune on healthy hosts +DOCKER_PRUNE_DISK_GATE_PCT = 75 + ALL_TOOLS = GROUP_A_TOOLS | GROUP_B_TOOLS MIN_TRUST_SCORE_FOR_GROUP_B = 0.8 @@ -335,6 +340,19 @@ class SSHProvider(MCPToolProvider): }, "required": ["host", "trust_score"]}, server_name=self.name, ), + MCPTool( + name="ssh_docker_prune", + description=( + "Reclaim disk space via docker image+volume+builder prune. " + f"Gated: only runs when root filesystem usage >= {DOCKER_PRUNE_DISK_GATE_PCT}%. " + "Requires trust_score >= 0.8." + ), + input_schema={"type": "object", "properties": { + "host": {"type": "string"}, + "trust_score": {"type": "number"}, + }, "required": ["host", "trust_score"]}, + server_name=self.name, + ), ] # ========================================================================= @@ -551,6 +569,21 @@ class SSHProvider(MCPToolProvider): if tool_name == "ssh_reload_nginx": return "nginx -t 2>&1 && systemctl reload nginx && echo 'Nginx reloaded'" + if tool_name == "ssh_docker_prune": + # Disk-gated docker prune: only acts when the alerting condition still holds. + # 2026-05-02 ogt + Claude Sonnet 4.6: ADR-068 飛輪 — disk full SOP + gate = DOCKER_PRUNE_DISK_GATE_PCT + return ( + "USAGE=$(df --output=pcent / | tail -1 | tr -dc '0-9'); " + f"if [ -z \"$USAGE\" ] || [ \"$USAGE\" -lt {gate} ]; then " + f"echo \"skip: disk usage ${{USAGE}}% < {gate}% gate\"; exit 0; fi; " + "echo '=== DISK BEFORE ==='; df -h /; " + "echo '=== IMAGE PRUNE ==='; docker image prune -a -f 2>&1 | tail -3; " + "echo '=== VOLUME PRUNE ==='; docker volume prune -f 2>&1 | tail -3; " + "echo '=== BUILDER PRUNE ==='; docker builder prune -a -f 2>&1 | tail -3; " + "echo '=== DISK AFTER ==='; df -h /" + ) + raise ValueError(f"No command builder for tool: {tool_name}") async def _ssh_exec( diff --git a/apps/api/src/services/decision_manager.py b/apps/api/src/services/decision_manager.py index 9538cd760..8c946c540 100644 --- a/apps/api/src/services/decision_manager.py +++ b/apps/api/src/services/decision_manager.py @@ -3307,6 +3307,11 @@ class DecisionManager: elif _action_lower.startswith("ssh ") and "docker restart" in _action_lower and target != "unknown": _tool = "ssh_docker_restart" _container = target + elif _action_lower.startswith("ssh ") and "docker" in _action_lower and "prune" in _action_lower: + # 主機磁碟告警自動修復:docker image/volume/builder prune + # 2026-05-02 ogt + Claude Sonnet 4.6: ADR-068 飛輪 — disk full SOP + # ssh_docker_prune 內含 ≥75% 磁碟使用率守衛,低於閾值會 no-op + _tool = "ssh_docker_prune" elif _action_lower.startswith("ssh ") and ("ps aux" in _action_lower or "top" in _action_lower or "free" in _action_lower or "df -h" in _action_lower or "uptime" in _action_lower): # 主機診斷指令:自動收集 CPU/記憶體/磁碟,不修改任何狀態 _tool = "ssh_diagnose" @@ -3339,6 +3344,7 @@ class DecisionManager: elif _tool == "ssh_systemctl_restart": params["service"] = _service if _tool in {"ssh_docker_restart", "ssh_systemctl_restart"}: + # Codex: 用 token 的 trust_score / confidence 餵給 Group B 守衛 try: params["trust_score"] = float( token.proposal_data.get("trust_score") @@ -3347,6 +3353,10 @@ class DecisionManager: ) except Exception: params["trust_score"] = 0.0 + elif _tool == "ssh_docker_prune": + # 2026-05-02 ogt + Claude Sonnet 4.6: docker prune 飛輪 + # Group B 工具需要 trust_score;自動修復鏈用 0.85(高於 0.8 門檻) + params["trust_score"] = 0.85 # ssh_diagnose: read-only, host only. try: diff --git a/apps/api/tests/test_decision_manager_docker_prune_routing.py b/apps/api/tests/test_decision_manager_docker_prune_routing.py new file mode 100644 index 000000000..2f0d2e6a7 --- /dev/null +++ b/apps/api/tests/test_decision_manager_docker_prune_routing.py @@ -0,0 +1,136 @@ +""" +DecisionManager._ssh_execute docker prune 路由測試 +================================================= +ADR-068 飛輪 — disk full SOP(2026-05-02 ogt + Claude Sonnet 4.6) + +驗證 LLM 提案 action 含 "docker prune" 時,會路由到 ssh_provider 的 +ssh_docker_prune 工具,並帶上 trust_score=0.85 + host=instance label。 +""" + +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from src.plugins.mcp.interfaces import MCPToolResult +from src.services.decision_manager import DecisionManager + + +def _fake_incident(host: str = "192.168.0.110") -> SimpleNamespace: + """最小可用 Incident stub — 只有 _ssh_execute 用到的欄位""" + signal = SimpleNamespace(labels={"instance": host}) + return SimpleNamespace( + incident_id="INC-TEST-PRUNE", + signals=[signal], + ) + + +def _fake_token() -> SimpleNamespace: + """最小可用 DecisionToken stub""" + return SimpleNamespace( + state=None, + proposal_data={}, + error=None, + ) + + +@pytest.fixture +def manager(monkeypatch): + """避免實例化重型依賴(openclaw / redis / knowledge_service)""" + # get_knowledge_service 在 __init__ 內 import,patch 原始 module + with patch("src.services.decision_manager.get_openclaw"), \ + patch("src.services.knowledge_service.get_knowledge_service"), \ + patch("src.plugins.mcp.providers.k8s_provider.K8sProvider"), \ + patch("src.plugins.mcp.providers.ssh_provider.SSHProvider"): + mgr = DecisionManager() + # 用 AsyncMock 攔截 SSH execute、token 存檔、telegram push + mgr._ssh = MagicMock() + mgr._ssh.execute = AsyncMock( + return_value=MCPToolResult(success=True, execution_id="exec-1", output={"stdout": "pruned"}) + ) + mgr._save_token = AsyncMock() + monkeypatch.setattr( + "src.services.decision_manager._fire_and_forget", + lambda *a, **k: None, + ) + # _fire_and_forget 已經 patch 為 no-op,但這兩個函式被「呼叫」會建出 coroutine + # 改用同步 stub 直接回 None 避免 unawaited coroutine warning + monkeypatch.setattr( + "src.services.decision_manager._push_decision_to_telegram", + lambda *a, **k: None, + ) + monkeypatch.setattr( + "src.services.decision_manager._push_auto_repair_result", + lambda *a, **k: None, + ) + monkeypatch.setenv("SSH_MCP_ALLOWED_HOSTS", "192.168.0.110,192.168.0.188") + return mgr + + +class TestDockerPruneActionRouting: + """LLM action 字串 → ssh_docker_prune 工具路由""" + + @pytest.mark.asyncio + async def test_ssh_docker_prune_action_routes_to_tool(self, manager): + incident = _fake_incident() + token = _fake_token() + + await manager._ssh_execute( + incident=incident, + token=token, + action="ssh 192.168.0.110 'docker image prune -a -f && docker volume prune -f'", + target="root", + ) + + manager._ssh.execute.assert_awaited_once() + call = manager._ssh.execute.call_args + assert call.kwargs["tool_name"] == "ssh_docker_prune", ( + f"expected ssh_docker_prune, got {call.kwargs.get('tool_name')}" + ) + params = call.kwargs["parameters"] + assert params["host"] == "192.168.0.110" + # Group B 必須帶 trust_score >= 0.8 + assert params.get("trust_score", 0) >= 0.8 + + @pytest.mark.asyncio + async def test_short_form_docker_prune_routes(self, manager): + """簡寫格式(無 ssh prefix 但有 docker prune 關鍵字)也應路由""" + # 注意:必須以 ssh 開頭才命中現有路由家族;非 ssh 開頭走別的分支 + incident = _fake_incident() + token = _fake_token() + await manager._ssh_execute( + incident=incident, + token=token, + action="ssh wooo@192.168.0.110 docker prune -a", + target="docker", + ) + assert manager._ssh.execute.call_args.kwargs["tool_name"] == "ssh_docker_prune" + + @pytest.mark.asyncio + async def test_docker_restart_still_routes_to_docker_restart(self, manager): + """加新分支不能誤傷既有 docker restart 路由""" + incident = _fake_incident() + token = _fake_token() + await manager._ssh_execute( + incident=incident, + token=token, + action="ssh 192.168.0.110 docker restart awoooi-api", + target="awoooi-api", + ) + # Codex 已把 _tool 改為 SSH MCP 全名(之前 short name 與 ssh_provider 不對齊) + assert manager._ssh.execute.call_args.kwargs["tool_name"] == "ssh_docker_restart" + + @pytest.mark.asyncio + async def test_diagnose_still_routes_to_ssh_diagnose(self, manager): + """加新分支不能誤傷 diagnose 路由""" + incident = _fake_incident() + token = _fake_token() + await manager._ssh_execute( + incident=incident, + token=token, + action="ssh 192.168.0.110 'df -h && free -h'", + target="unknown", + ) + assert manager._ssh.execute.call_args.kwargs["tool_name"] == "ssh_diagnose" diff --git a/apps/api/tests/test_ssh_provider_docker_prune.py b/apps/api/tests/test_ssh_provider_docker_prune.py new file mode 100644 index 000000000..faf21318d --- /dev/null +++ b/apps/api/tests/test_ssh_provider_docker_prune.py @@ -0,0 +1,83 @@ +""" +ssh_docker_prune 工具測試 +========================== +ADR-068 飛輪 — disk full SOP(2026-05-02 ogt + Claude Sonnet 4.6) + +測試項目: +- ssh_docker_prune 在 GROUP_B_TOOLS 白名單內 +- list_tools() 回傳含 ssh_docker_prune 且 schema 正確 +- _build_command() 產生包含 75% 磁碟守衛 + image/volume/builder prune 鏈 +- _build_command() 不會引入 FORBIDDEN_PATTERNS(rm -rf, $( ), backtick 等) +""" + +from __future__ import annotations + +import asyncio + +from src.plugins.mcp.providers.ssh_provider import ( + DOCKER_PRUNE_DISK_GATE_PCT, + GROUP_B_TOOLS, + SSHProvider, +) + + +class TestDockerPruneToolRegistration: + """白名單與 list_tools() 註冊""" + + def test_in_group_b(self): + assert "ssh_docker_prune" in GROUP_B_TOOLS + + def test_disk_gate_constant(self): + # 守衛閾值合理(不會太低誤觸,不會太高永遠不跑) + assert 50 <= DOCKER_PRUNE_DISK_GATE_PCT <= 90 + + def test_listed_with_schema(self): + provider = SSHProvider() + tools = asyncio.run(provider.list_tools()) + prune = next((t for t in tools if t.name == "ssh_docker_prune"), None) + assert prune is not None, "ssh_docker_prune not registered in list_tools()" + assert prune.server_name == provider.name + assert "host" in prune.input_schema["required"] + assert "trust_score" in prune.input_schema["required"] + + +class TestDockerPruneCommand: + """命令字串建構""" + + def _cmd(self) -> str: + provider = SSHProvider() + return provider._build_command("ssh_docker_prune", {"host": "192.168.0.110"}) + + def test_includes_disk_gate(self): + cmd = self._cmd() + # 應該先檢查磁碟使用率,低於閾值就 skip + assert "df --output=pcent" in cmd + assert f"{DOCKER_PRUNE_DISK_GATE_PCT}" in cmd + assert "skip" in cmd + + def test_includes_full_prune_chain(self): + cmd = self._cmd() + assert "docker image prune -a -f" in cmd + assert "docker volume prune -f" in cmd + assert "docker builder prune -a -f" in cmd + + def test_reports_disk_before_and_after(self): + cmd = self._cmd() + assert "DISK BEFORE" in cmd + assert "DISK AFTER" in cmd + + def test_no_destructive_outside_docker(self): + """命令只該動 docker 子系統,不該有 rm -rf / 寫 /etc / 改 sudoers""" + cmd = self._cmd() + assert "rm -rf" not in cmd + assert "/etc/" not in cmd + assert "sudoers" not in cmd + assert "authorized_keys" not in cmd + + def test_only_uses_safe_params(self): + """ssh_docker_prune 不接受 user-supplied 字串 params(只有 host/trust_score)""" + provider = SSHProvider() + tools = asyncio.run(provider.list_tools()) + prune = next(t for t in tools if t.name == "ssh_docker_prune") + # schema 只有 host (string, whitelist-checked) + trust_score (number) + assert set(prune.input_schema["properties"].keys()) == {"host", "trust_score"} diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index dd51d879a..562925024 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -6,6 +6,32 @@ --- +## 2026-05-02 | Telegram 告警噴爆事故閉環 + Docker prune 飛輪補完 + +承接昨晚到今早 Telegram 告警量爆增(峰值 53/hr)事故。根因是 `ssh-mcp-key` Secret 的 `known_hosts` 欄位是 0 bytes 空檔,asyncssh 拒絕所有 SSH,導致 110 磁碟告警的 auto_repair 全部走「Host key is not trusted → emergency_channel → Telegram」路徑無限重試。 + +### 完成 +- **P0 止血**:`ssh-keyscan` 取得 110/120/121/188 四台 host keys,patch `awoooi-prod/ssh-mcp-key` Secret 的 `known_hosts` 欄位(4548 bytes),rollout restart awoooi-api。pod 內 `/etc/ssh-mcp/known_hosts` 已含四台主機。 +- **P1 磁碟清理**:手動跑 `docker image prune -a -f && docker volume prune -f && docker builder prune -a -f`,110 磁碟 86% → 82%(回收 35GB),低於 `HostDiskUsageHigh` 80% 閾值的延伸區間,但已停 escalation。 +- **權限治理**:`.claude/settings.local.json` 加 5 條 SSH allow 規則(`Bash(ssh wooo@192.168.0.110:*)` 等),補完 4 台主機 + ollama@188 的全範圍 SSH 授權。 +- **飛輪補完(本 commit)**: + - `ssh_provider.py` 新增 Group B 工具 `ssh_docker_prune`,命令含 ≥75% 磁碟使用率守衛(低於閾值 no-op),執行 image+volume+builder prune 三鏈。 + - `decision_manager._ssh_execute` 新增 action 路由:含 `docker prune` 字樣 → `ssh_docker_prune`,trust_score=0.85(≥0.8 門檻)。 + - 110 上 `/home/wooo/monitoring/alerts.yml` 的 `HostDiskUsageHigh` annotation:`auto_repair: "false"` → `"true"`、加 `mcp_provider: "ssh_host"` / `host_type: "bare_metal"`、annotation 加 `auto_repair_action` 提示 LLM。Prometheus 已 reload `HTTP 200`,新 labels 已生效。 + - 同份 alerts.yml 納入 repo `ops/monitoring/alerts.yml`,避免 config drift(之前只在 110 上)。 + +### 驗證 +- `pytest tests/test_ssh_provider_docker_prune.py tests/test_decision_manager_docker_prune_routing.py` → 12 passed clean。 +- `pytest tests/ -k "ssh or decision or prune or playbook" --ignore=tests/integration` → 203 passed, 2 skipped, 0 failed。 +- `python3 -m py_compile` 通過。 +- Prometheus `/api/v1/rules` 確認 `HostDiskUsageHigh` 新 labels 生效。 +- Telegram 告警量:修復前 53/hr 峰值 → 修復後 ~2/hr(28 分鐘只 1 則)。 + +### 遺留 +- `awoooi-repair-known-hosts` Secret 與 `ssh-mcp-key` Secret 的 known_hosts 欄位重複,後續可整併。 +- 110 仍有 ~55GB 可回收 volumes(被活躍 container 持有,需停 container 才能清)。 +- `ssh_docker_prune` 目前還沒實際被 LLM 提案觸發過,要等下次 110 磁碟超 80% 持續 10 分鐘觀察自治飛輪是否成功。 + ## 2026-05-01 | Emergency intervention 留痕閉環 承接 SSH/backup 自動修復閉環與 Telegram ghost-button 補洞:SecOps 隔離/封鎖按鈕已降級成授權記錄,但若只回文字、不寫 Redis/AOL/timeline,就會形成「看似有人接手,系統沒有記憶」的新斷點;drift auto-adopt 被擋時也需要同樣進 WarRoom timeline。 @@ -61,6 +87,47 @@ - Live service state:110 已使用 user-level `gitea-act-runner-host.service` 接管,狀態 `active (running)`;`e7991b8e` Code Review 成功。 - Remaining host permission item:110 `loginctl show-user wooo -p Linger` 仍為 `Linger=no`。若要讓 user-level fallback 在無登入 session 的 boot 狀態自動啟動,需要 root 執行 `loginctl enable-linger wooo`,或改走 system-level service。 +## 2026-05-01 | AwoooP Implementation Roadmap + +承接 ADR-106/ADR-107 後,補一份給下一個工程 session 使用的實施總綱,避免 ADR 只有決策而沒有分期落地路線。 + +### 完成 +- 新增 `docs/awooop/IMPLEMENTATION-ROADMAP.md`,整理 AwoooP 解決方案總結、目錄策略、八階段實施步驟、驗收條件與 Codex 開工方式。 +- 明定現在只建立 `docs/awooop/` 規劃文件,不建立 `apps/awooop-runtime`、`apps/awooop-worker`、`packages/awooop-contracts`、`packages/awooop-client` 等空 code 目錄。 +- 建議實作新開 Codex 對話,但 cwd 維持 `/Users/ogt/awoooi` monorepo root;第一個 code slice 從 PostgreSQL contract revision schema 開始。 + +### 驗證 +- Docs-only change;執行 `git diff --check`。 + +## 2026-05-01 | ADR-107 AwoooP Control Plane Storage Strategy + +承接 ADR-106 六合約總綱後的控制面實體化決策:AwoooP 需要明確決定六大 contract 的 source of truth,避免在 PostgreSQL、Redis、Kubernetes CRD、Git artifact 之間產生 split-brain。 + +### 完成 +- 新增 `docs/adr/ADR-107-awooop-control-plane-storage.md`,決定 AwoooP v1 採 PostgreSQL-first,不採 CRD-first。 +- 明定 PostgreSQL 擁有 contract drafts/revisions、active revision pointers、tenant/agent/policy/MCP grants、budget、ACL、run state、approval、channel event、audit/trace correlation。 +- 明定 Redis 只能做 cache/watch/counter/coordination,任何 runtime cache 都必須帶 `revision_id` 與 `body_hash`。 +- 明定 prompt、JSON Schema、eval suite、replay fixture 等大型 artifact 以 ref + SHA-256 hash 管理。 +- 明定 Kubernetes CRD 僅作未來 runtime projection,如 `AwoooPRuntime`、`AwoooPWorker`、`MCPServerBinding`、`ChannelIngress`、`TenantRuntimeBinding`,不承擔 budget/run/audit/event source of truth。 + +### 驗證 +- Docs-only change;執行 `git diff --check`。 + +## 2026-05-01 | ADR-106 AwoooP Agent Platform 六合約總綱 + +承接多專案 AI Agent 共用架構討論:AWOOOI、EwoooC/MOMO PRO、岑洋、碧潭與未來產品需要共用 OpenClaw/NemoTron/Hermes/ElephantAlpha 與後續 Agent/MCP/Channel 能力,但不能把 AWOOOI 私有邏輯當成所有專案的大腦。 + +### 完成 +- 新增 `docs/adr/ADR-106-agent-platform-architecture.md`,確立平台產品名為 AwoooP:AWOOOI 是 first tenant / first runtime host,不是平台邊界。 +- 鎖定六份 v1.0 contract baseline:Project/Tenant、Agent、MCP Gateway、Policy/Routing、Runtime/Run State、Communication/Channel Event。 +- 明定 MCP 必須走 Gateway、Context Firewall 必須由底層強制、Runtime 必須用 durable Run state、Channel Adapter 只做收驗轉送。 +- 明定遷移策略為 `shadow -> canary -> read_only -> suggest -> auto_remediate`,禁止大爆改切換。 +- 明定暫不建立空專案目錄;待實作開始時再建立 `packages/awooop-contracts`、`packages/awooop-client`、`apps/awooop-runtime`、`apps/awooop-worker`、`docs/awooop` 等有明確 ownership 的路徑。 +- 記錄 `ADR-106` 編號與舊 `models.json` placeholder 的衝突處理:模型/動態路由債務併入 Policy/Routing contract 後續實作或另開非衝突 ADR。 + +### 驗證 +- Docs-only change;執行 `git diff --check`。 + ## 2026-05-01 | Agent Loop shadow structured metadata guard 承接 P1 canary 上線後的 production 觀測:`ENABLE_OPENCLAW_AGENT_LOOP_SHADOW=True`、max iteration 2 已在 API pod 生效;`mcp_audit_log` 已有 MCP 呼叫,但尚未看到新的 `openclaw_agent_loop_shadow` production incident log。下一步先讓 shadow 一旦觸發就留下可評估、可治理的結構化結果,而不是直接改主決策。 @@ -406,11 +473,11 @@ Codex 接續 AI 自動化 Wave B,先把「告警→AI→安全閘→執行→ - 新增記錄 `project_revert_a2_ollama_primary.md` - ADR-105 完整記錄推翻 A2 決策 -### 已知債(後續 ADR) -- ADR-106 `models.json` 對齊 prod 實載 model -- ADR-107 `complexity_map` 4/5 寫死雲端 改動態 -- ADR-108 OpenClaw 188 服務 500 根因 -- ADR-109 Claude API endpoint 過期升級 +### 已知債(後續追蹤) +- `models.json` 對齊 prod 實載 model;併入 ADR-106 Policy/Routing contract 後續實作或另開非衝突 ADR。 +- `complexity_map` 4/5 寫死雲端改動態;併入 ADR-106 Policy/Routing contract 後續實作或另開非衝突 ADR。 +- OpenClaw 188 服務 500 根因。 +- Claude API endpoint 過期升級。 --- diff --git a/ops/monitoring/alerts.yml b/ops/monitoring/alerts.yml new file mode 100644 index 000000000..bd078ede7 --- /dev/null +++ b/ops/monitoring/alerts.yml @@ -0,0 +1,1138 @@ +# ops/monitoring/alerts-unified.yml +# AWOOOI 統一 Prometheus 告警規則 +# 2026-04-05 Claude Code: 整合所有規則,加入統一 layer 標籤 +# 2026-04-08 Claude Sonnet 4.6: 補 database_detail_alerts 群組 (6條詳細規則) +# 2026-04-12 Claude Sonnet 4.6: 補回 4 個僅存在主機的群組 (backup/flywheel/connectivity/infra-detailed) +# 部署目標: 192.168.0.110:/home/wooo/monitoring/alerts.yml +# 部署方式: scripts/ops/deploy-alerts.sh (CD 自動部署) +# +# 標籤規範: +# layer: k8s | docker-110 | docker-188 | systemd-188 +# component: 服務名稱 +# team: ops | backend | ai | platform +# host: "110" | "188" | "120" | "121" +# auto_repair: "true" | "false" + +groups: + + # ========================================================================= + # 主機層告警 (host_alerts) + # ========================================================================= + - name: host_alerts + rules: + - alert: HostDown + expr: up{job=~"node-exporter.*"} == 0 + for: 1m + labels: + severity: critical + layer: systemd-188 + team: ops + auto_repair: "false" + annotations: + summary: "主機 {{ $labels.host }} 不可達" + description: "Node Exporter 無回應超過 1 分鐘" + + - alert: HostHighCpuLoad + expr: 100 - (avg by(host) (rate(node_cpu_seconds_total{mode="idle"}[5m])) * 100) > 80 + for: 5m + labels: + severity: warning + layer: systemd-188 + team: ops + auto_repair: "true" + # MCP Phase 2a (ADR-071, 2026-04-11 Claude Sonnet 4.6): SSH MCP 路由標籤 + mcp_provider: "ssh_host" + host_type: "bare_metal" + alert_category: "host_resource" + annotations: + summary: "主機 {{ $labels.host }} CPU 高負載" + description: "CPU 使用率超過 80%" + + - alert: HostOutOfMemory + expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) * 100 > 85 + for: 5m + labels: + severity: warning + layer: systemd-188 + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + host_type: "bare_metal" + alert_category: "host_resource" + annotations: + summary: "主機 {{ $labels.host }} 記憶體不足" + description: "記憶體使用率超過 85%" + + - alert: HostOutOfDiskSpace + expr: (1 - (node_filesystem_avail_bytes{fstype!~"tmpfs|overlay"} / node_filesystem_size_bytes{fstype!~"tmpfs|overlay"})) * 100 > 85 + for: 5m + labels: + severity: warning + layer: systemd-188 + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + host_type: "bare_metal" + alert_category: "host_resource" + annotations: + summary: "主機 {{ $labels.host }} 磁碟空間不足" + description: "磁碟使用率超過 85%" + + # ========================================================================= + # K8s 叢集告警 (kubernetes_alerts) + # ========================================================================= + - name: kubernetes_alerts + rules: + - alert: K3sNodeNotReady + expr: kube_node_status_condition{condition="Ready", status="true"} == 0 + for: 2m + labels: + severity: critical + layer: k8s + team: ops + auto_repair: "false" + annotations: + summary: "K3s 節點 {{ $labels.node }} 未就緒" + description: "節點超過 2 分鐘未達到 Ready 狀態" + + - alert: KubePodCrashLooping + expr: rate(kube_pod_container_status_restarts_total{namespace="awoooi-prod"}[15m]) > 0 + for: 5m + labels: + severity: warning + layer: k8s + team: ops + auto_repair: "true" + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 持續重啟" + description: "Pod 在過去 15 分鐘內重啟次數異常" + + - alert: KubePodNotReady + expr: kube_pod_status_ready{condition="true",namespace="awoooi-prod"} == 0 + for: 5m + labels: + severity: warning + layer: k8s + team: ops + auto_repair: "true" + annotations: + summary: "Pod {{ $labels.namespace }}/{{ $labels.pod }} 未就緒" + description: "Running 中的 Pod 超過 5 分鐘未達到 Ready 狀態" + + - alert: KubeDeploymentReplicasMismatch + expr: kube_deployment_spec_replicas{namespace="awoooi-prod"} != kube_deployment_status_replicas_available{namespace="awoooi-prod"} + for: 10m + labels: + severity: warning + layer: k8s + team: ops + auto_repair: "true" + annotations: + summary: "Deployment {{ $labels.namespace }}/{{ $labels.deployment }} 副本數不匹配" + description: "期望副本數與可用副本數不一致超過 10 分鐘" + + - alert: VeleroBackupFailed + expr: increase(velero_backup_failure_total[24h]) > 0 + for: 5m + labels: + severity: warning + layer: k8s + team: ops + component: velero + auto_repair: "false" + annotations: + summary: "Velero 備份失敗" + description: "過去 24 小時有備份失敗" + + - alert: VeleroBackupNotRun + expr: time() - velero_backup_last_successful_timestamp > 86400 + for: 10m + labels: + severity: critical + layer: k8s + team: ops + component: velero + auto_repair: "false" + annotations: + summary: "Velero 超過 24 小時未成功備份" + description: "最後一次成功備份超過 24 小時" + + # Sprint C-2 Host rsync 備份告警 (2026-04-11 Claude Sonnet 4.6) + # backup-from-110.sh 成功時寫入 /var/run/backup-110.last_success + # node-exporter textfile collector 讀取此檔案暴露指標 + - alert: HostBackupFailed + expr: time() - node_textfile_scrape_error{collector="backup_110"} == 0 or absent(node_textfile_scrape_error{collector="backup_110"}) or (time() - backup_110_last_success_timestamp > 90000) + for: 10m + labels: + severity: warning + layer: docker-188 + team: ops + component: backup + host: "188" + auto_repair: "true" + alert_category: host_resource + annotations: + summary: "188 Host 備份超過 25 小時未成功" + description: "backup-from-110.sh 可能失敗,/backup/110 資料可能過舊" + + # ADR-075: CoreDNS 解析失敗 (2026-04-12 ogt) + - alert: CoreDNSResolutionFailed + expr: rate(coredns_dns_responses_total{rcode="SERVFAIL"}[5m]) > 0.05 + for: 5m + labels: + severity: critical + layer: k8s + team: ops + auto_repair: "true" + alert_category: kubernetes + notification_type: TYPE-3 + annotations: + summary: "CoreDNS SERVFAIL 率過高 {{ $value | humanizePercentage }}" + description: "CoreDNS 在 5 分鐘內 SERVFAIL 回應率超過 5%,K8s 服務間 DNS 解析可能失敗" + runbook: "kubectl -n kube-system get pods -l k8s-app=kube-dns && kubectl -n kube-system logs -l k8s-app=kube-dns" + + # ========================================================================= + # 資料庫告警 (database_alerts) + # ========================================================================= + - name: database_alerts + rules: + - alert: PostgreSQLDown + expr: up{job="postgres-exporter"} == 0 or pg_up == 0 + for: 1m + labels: + severity: critical + layer: systemd-188 + component: postgres + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "PostgreSQL 資料庫離線" + description: "PostgreSQL Exporter 無法連接資料庫超過 1 分鐘" + + - alert: RedisDown + expr: up{job="redis-exporter"} == 0 or redis_up == 0 + for: 1m + labels: + severity: critical + layer: systemd-188 + component: redis + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "Redis 快取服務離線" + description: "Redis Exporter 無法連接 Redis 超過 1 分鐘" + + - alert: PostgreSQLHighConnections + expr: pg_stat_activity_count > 80 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: postgres + team: ops + auto_repair: "false" + annotations: + summary: "PostgreSQL 連接數過高" + description: "當前連接數 {{ $value }} 超過 80" + + - alert: RedisMemoryHigh + expr: redis_memory_max_bytes > 0 and redis_memory_used_bytes / redis_memory_max_bytes * 100 > 80 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: redis + team: ops + auto_repair: "false" + annotations: + summary: "Redis 記憶體使用過高" + description: "Redis 記憶體使用率超過 80%" + + # ========================================================================= + # Sprint 5.2 Plan B: 資料庫詳細指標告警 (database_detail_alerts) + # 前置: postgres-exporter:9187 + redis-exporter:9121 on 192.168.0.188 + # 2026-04-08 Claude Sonnet 4.6 Asia/Taipei + # ========================================================================= + - name: database_detail_alerts + rules: + # ---- PostgreSQL 詳細指標 ---- + - alert: PostgreSQLSlowQueries + expr: pg_stat_activity_max_tx_duration{datname="awoooi_prod"} > 60 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: postgres + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "PostgreSQL 有慢查詢 (>60s)" + description: "awoooi_prod 資料庫最長事務超過 60 秒" + + - alert: PostgreSQLDeadlocks + expr: increase(pg_stat_database_deadlocks{datname="awoooi_prod"}[5m]) > 0 + for: 1m + labels: + severity: warning + layer: systemd-188 + component: postgres + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "PostgreSQL 死鎖發生" + description: "過去 5 分鐘 awoooi_prod 資料庫有死鎖" + + - alert: PostgreSQLTooManyConnections + expr: pg_stat_activity_count{datname="awoooi_prod"} > 50 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: postgres + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "PostgreSQL 連接數過高 ({{ $value }})" + description: "awoooi_prod 連接數超過 50" + + # ---- Redis 詳細指標 ---- + - alert: RedisKeyEviction + expr: increase(redis_evicted_keys_total[5m]) > 0 + for: 1m + labels: + severity: warning + layer: systemd-188 + component: redis + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "Redis 發生 Key 驅逐" + description: "過去 5 分鐘有 Key 被驅逐,可能記憶體不足" + + - alert: RedisConnectionsHigh + expr: redis_connected_clients > 100 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: redis + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "Redis 連接數過高 ({{ $value }})" + description: "Redis 連接數超過 100" + + - alert: RedisCommandLatencyHigh + expr: redis_commands_duration_seconds_total / redis_commands_processed_total > 0.01 + for: 5m + labels: + severity: warning + layer: systemd-188 + component: redis + host: "188" + team: ops + auto_repair: "false" + annotations: + summary: "Redis 命令平均延遲過高 (>10ms)" + description: "Redis 命令平均延遲超過 10ms" + + # ========================================================================= + # 服務可用性告警 (service_alerts) — 含 layer 標籤決定修復路徑 + # ========================================================================= + - name: service_alerts + rules: + # ---- 188 Docker 層 ---- + - alert: OpenClawDown + # 2026-04-05 Claude Code: 修正舊命名 ClawBotDown → OpenClawDown + expr: up{job="clawbot"} == 0 + for: 2m + labels: + severity: critical + layer: docker-188 + component: openclaw + host: "188" + team: ops + auto_repair: "true" + annotations: + summary: "OpenClaw 服務離線" + description: "OpenClaw (192.168.0.188:8088) 已離線超過 2 分鐘" + + - alert: SignOzDown + expr: probe_success{job="blackbox-http", instance=~".*3301.*"} == 0 + for: 2m + labels: + severity: warning + layer: docker-188 + component: signoz + host: "188" + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + target_host: "192.168.0.188" + alert_category: "devops_tool" + annotations: + summary: "SignOz 服務離線" + description: "SignOz (192.168.0.188:3301) 已離線超過 2 分鐘" + + # ---- 110 Docker 層 ---- + - alert: SentryDown + expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:9000"} == 0 + for: 2m + labels: + severity: warning + layer: docker-110 + component: sentry + host: "110" + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + target_host: "192.168.0.110" + alert_category: "devops_tool" + annotations: + summary: "Sentry 服務離線" + description: "Sentry (192.168.0.110:9000) 已離線超過 2 分鐘" + + - alert: HarborDown + expr: probe_success{job="blackbox-http", instance=~".*5000.*"} == 0 + for: 2m + labels: + severity: critical + layer: docker-110 + component: harbor + host: "110" + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + target_host: "192.168.0.110" + alert_category: "devops_tool" + annotations: + summary: "Harbor Registry 離線" + description: "Harbor (192.168.0.110:5000) 已離線超過 2 分鐘,CD pipeline 將無法拉取映像" + + - alert: GiteaDown + expr: probe_success{job="blackbox-http", instance="http://192.168.0.110:3001"} == 0 + for: 2m + labels: + severity: critical + layer: docker-110 + component: gitea + host: "110" + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + target_host: "192.168.0.110" + alert_category: "devops_tool" + annotations: + summary: "Gitea Git 服務離線" + description: "Gitea (192.168.0.110:3001) 已離線超過 2 分鐘,CD pipeline 失效" + + - alert: AlertmanagerDown + expr: probe_success{job="blackbox-tcp", instance="192.168.0.110:9093"} == 0 + for: 2m + labels: + severity: critical + layer: docker-110 + component: alertmanager + host: "110" + team: ops + auto_repair: "true" + annotations: + summary: "Alertmanager 離線" + description: "Alertmanager (192.168.0.110:9093) 已離線,所有告警將靜默" + + # ========================================================================= + # 告警鏈路監控 (alert_chain) — 防止 2026-03-26/04-05 事故重演 + # ========================================================================= + - name: alert_chain + rules: + - alert: AlertChainBroken_Alertmanager + expr: | + sum(rate(awoooi_webhook_requests_total{source="alertmanager",status!="success"}[5m])) + / sum(rate(awoooi_webhook_requests_total{source="alertmanager"}[5m])) > 0.1 + for: 10m + labels: + severity: critical + layer: k8s + team: platform + auto_repair: "false" + annotations: + summary: "Alertmanager Webhook 錯誤率 > 10%" + description: "告警鏈路可能斷裂,請執行 E2E 驗證" + + - alert: AlertChainBroken_Sentry + expr: | + sum(rate(awoooi_webhook_requests_total{source="sentry",status!="success"}[5m])) + / sum(rate(awoooi_webhook_requests_total{source="sentry"}[5m])) > 0.1 + for: 10m + labels: + severity: warning + layer: k8s + team: platform + auto_repair: "false" + annotations: + summary: "Sentry Webhook 錯誤率 > 10%" + description: "Sentry 錯誤可能無法正確處理" + + - alert: NoAlertsReceived2Hours + expr: time() - max by (source)(awoooi_alert_chain_last_success_timestamp) > 7200 + for: 5m + labels: + severity: warning + layer: k8s + team: platform + auto_repair: "false" + annotations: + summary: "2 小時內未收到任何告警 ({{ $labels.source }})" + description: "可能是告警鏈路問題,請執行 Smoke Test" + + - alert: AlertChainUnhealthy + expr: awoooi_alert_chain_healthy == 0 + for: 5m + labels: + severity: critical + layer: k8s + team: platform + auto_repair: "false" + annotations: + summary: "告警鏈路不健康 ({{ $labels.source }})" + description: "告警鏈路標記為不健康,最近處理失敗" + + # ========================================================================= + # 自動修復監控 (auto_repair) + # ========================================================================= + - name: auto_repair + rules: + - alert: AutoRepairLowSuccessRate + expr: awoooi_auto_repair_success_rate < 0.3 + for: 30m + labels: + severity: warning + layer: k8s + team: backend + auto_repair: "false" + annotations: + summary: "自動修復成功率過低 ({{ $value | humanizePercentage }})" + description: "動作 {{ $labels.action }} 的成功率低於 30%,建議檢查 Playbook" + + - alert: PermanentFixRequired + expr: sum(rate(awoooi_anomaly_escalation_total{level="PERMANENT_FIX"}[1h])) > 0 + for: 1m + labels: + severity: critical + layer: k8s + team: backend + auto_repair: "false" + annotations: + summary: "需要永久修復的異常升級" + description: "有異常升級到 PERMANENT_FIX 級別,需要根本修復" + + # ========================================================================= + # Sprint 5.1: Docker 容器健康監控(docker-health-monitor 感知層接入) + # 由 docker-health-monitor.sh 送 Alertmanager 格式 webhook, + # 或 Prometheus 自訂 exporter 上報時使用。 + # auto_repair: "true" 代表允許 AWOOOI Guardrail 決策(非直接修復) + # 實際修復動作由 Service Registry 分級決定(ADR-062) + # 2026-04-08 Claude Sonnet 4.6 Asia/Taipei + # ========================================================================= + - name: docker_health_alerts + rules: + - alert: DockerContainerUnhealthy + expr: container_health_status{job="docker-health-monitor"} == 0 + for: 2m + labels: + severity: warning + layer: docker + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + alert_category: "devops_tool" + annotations: + summary: "容器 {{ $labels.container }} 健康檢查失敗" + description: "主機 {{ $labels.host }} 容器 {{ $labels.container }} 健康狀態異常,持續 2 分鐘" + + - alert: DockerContainerExited + expr: container_running_status{job="docker-health-monitor"} == 0 + for: 1m + labels: + severity: critical + layer: docker + team: ops + auto_repair: "true" + mcp_provider: "ssh_host" + alert_category: "devops_tool" + annotations: + summary: "容器 {{ $labels.container }} 已停止" + description: "主機 {{ $labels.host }} 容器 {{ $labels.container }} 已 exited/dead,持續 1 分鐘" + + # ========================================================================= + # MinIO / Kali 告警 + # ========================================================================= + - name: minio_kali_alerts + rules: + - alert: MinIODown + expr: probe_success{job="blackbox-http", instance=~".*9001.*"} == 0 + for: 2m + labels: + severity: warning + layer: docker-188 + component: minio + host: "188" + team: ops + auto_repair: "true" + annotations: + summary: "MinIO (Velero 備份) 離線" + description: "MinIO (192.168.0.188:9001) 已離線超過 2 分鐘,Velero 備份可能失敗" + + - alert: KaliScannerDown + expr: probe_success{job="blackbox-http", instance=~".*192.168.0.112.*"} == 0 + for: 5m + labels: + severity: info + layer: docker-188 + component: kali + host: "112" + team: ops + auto_repair: "false" + annotations: + summary: "Kali Scanner 離線" + description: "Kali (192.168.0.112:8080) 離線,安全掃描功能暫停" + + # ========================================================================= + # Plan C — 外部網站監控 (Sprint 5.2, 2026-04-09 Claude Sonnet 4.6 Asia/Taipei) + # blackbox-http 已涵蓋 4 個外部網站,此群組提供結構化告警 + # auto_repair: "true" — 由 AWOOOI Guardrail 決策(Service Registry 分級) + # ========================================================================= + - name: external_website_alerts + rules: + - alert: MoWoooWorkDown + expr: probe_success{job="blackbox-http", instance="https://mo.wooo.work"} == 0 + for: 3m + labels: + severity: critical + layer: external + component: momo-app + host: "188" + team: ops + auto_repair: "true" + annotations: + summary: "外部網站 mo.wooo.work 離線" + description: "mo.wooo.work 探測失敗超過 3 分鐘,容器 momo-app (188) 可能需要重啟" + + - alert: TsenyangWebsiteDown + expr: probe_success{job="blackbox-http", instance="https://www.tsenyang.com"} == 0 + for: 3m + labels: + severity: critical + layer: external + component: tsenyang-website + host: "188" + team: ops + auto_repair: "true" + annotations: + summary: "外部網站 tsenyang.com 離線" + description: "tsenyang.com 探測失敗超過 3 分鐘,容器 tsenyang-website (188) 可能需要重啟" + + - alert: StockWoooWorkDown + expr: probe_success{job="blackbox-http", instance="http://stock.wooo.work"} == 0 + for: 3m + labels: + severity: critical + layer: external + component: stock-platform + host: "110" + team: ops + auto_repair: "true" + annotations: + summary: "外部網站 stock.wooo.work 離線" + description: "stock.wooo.work 探測失敗超過 3 分鐘,容器 stock-platform (110) 可能需要重啟" + + - alert: BitanWoooWorkDown + expr: probe_success{job="blackbox-http", instance="https://bitan.wooo.work"} == 0 + for: 3m + labels: + severity: critical + layer: external + component: bitan-app + host: "188" + team: ops + auto_repair: "true" + annotations: + summary: "外部網站 bitan.wooo.work 離線" + description: "bitan.wooo.work 探測失敗超過 3 分鐘,容器 bitan-app (188) 可能需要重啟" + + - alert: ExternalSiteSSLExpiringSoon + expr: probe_ssl_earliest_cert_expiry{job="blackbox-http"} - time() < 14 * 24 * 3600 + for: 1h + labels: + severity: warning + layer: external + component: ssl + team: ops + auto_repair: "false" + annotations: + summary: "SSL 憑證即將到期: {{ $labels.instance }}" + description: "{{ $labels.instance }} SSL 憑證將在 14 天內到期,請手動更新" + +# ============================================================================= +# ADR-075 新增規則群組 (2026-04-12 ogt) +# ============================================================================= + + - name: awoooi_secops_alerts + interval: 60s + rules: + - alert: UnauthorizedSSHLogin + expr: increase(node_failed_auth_attempts_total[5m]) > 10 + for: 1m + labels: + severity: critical + layer: systemd-188 + team: security + auto_repair: "false" + alert_category: secops + annotations: + summary: "異常 SSH 登入嘗試: {{ $labels.instance }}" + description: "5 分鐘內失敗登入 {{ $value }} 次,可能遭受暴力破解" + + - name: awoooi_business_alerts + interval: 60s + rules: + - alert: AITokenCostSpike + expr: increase(awoooi_ai_token_cost_usd_total[1h]) > 10 + for: 5m + labels: + severity: warning + layer: k8s + team: finops + auto_repair: "false" + alert_category: business + annotations: + summary: "AI Token 費用 1 小時內暴增 ${{ $value | humanize }}" + description: "AI API 調用費用異常,請檢查是否有迴圈或濫用" + - alert: GeminiAPIErrorRateHigh + expr: rate(awoooi_ai_request_errors_total{provider="gemini"}[5m]) / rate(awoooi_ai_requests_total{provider="gemini"}[5m]) > 0.2 + for: 10m + labels: + severity: warning + layer: k8s + team: finops + auto_repair: "false" + alert_category: business + annotations: + summary: "Gemini API 錯誤率過高: {{ $value | humanizePercentage }}" + description: "Gemini API 5 分鐘錯誤率超過 20%,AI 降級可能失效" + # ADR-075: 業務爬蟲健康 (2026-04-12 ogt) + - alert: MomoScraperSuccessLow + expr: | + rate(momo_scraper_requests_total{status="success"}[5m]) + / rate(momo_scraper_requests_total[5m]) < 0.9 + for: 10m + labels: + severity: warning + layer: docker-110 + auto_repair: "false" + alert_category: business + notification_type: TYPE-3 + annotations: + summary: "Momo 抓取成功率跌至 {{ $value | humanizePercentage }}" + description: "Momo 爬蟲成功率低於 90%,業務資料可能缺失" + + - name: awoooi_flywheel_meta_alerts + interval: 60s + rules: + - alert: FlywheelPlaybookZero + expr: awoooi_flywheel_playbook_count == 0 + for: 1h + labels: + severity: critical + layer: k8s + team: aiops + auto_repair: "false" + alert_category: flywheel_health + notification_type: TYPE-8M + annotations: + summary: "飛輪 Playbook 數量為零,AI 修復完全依賴 LLM" + description: "Redis 中無任何已批准 Playbook,自動修復能力大幅降低" + runbook: "執行 scripts/cold_start_playbooks.py 冷啟動" + - alert: FlywheelExecutionSuccessLow + expr: awoooi_flywheel_execution_success_rate < 0.1 + for: 2h + labels: + severity: warning + layer: k8s + team: aiops + auto_repair: "false" + alert_category: flywheel_health + notification_type: TYPE-8M + annotations: + summary: "飛輪執行成功率 {{ $value | humanizePercentage }} 低於 10%" + description: "連續 2 小時執行成功率不足 10%,Playbook 可能已過時" + runbook: "檢查 decision_manager 日誌,確認 target 解析和 SSH MCP 狀態" + - alert: FlywheelKMVectorizationLow + expr: awoooi_flywheel_km_unvectorized_count > 10 + for: 30m + labels: + severity: warning + layer: k8s + team: aiops + auto_repair: "false" + alert_category: flywheel_health + notification_type: TYPE-8M + annotations: + summary: "{{ $value }} 筆 KM 未向量化,RAG 查詢命中率下降" + description: "knowledge_entries 中 embedding IS NULL 超過 10 筆且持續 30 分鐘" + runbook: "執行 scripts/batch_vectorize_km.py 或檢查每日 CronJob 狀態" + - alert: FlywheelAlertnameNullHigh + expr: awoooi_flywheel_alertname_null_rate > 0.05 + for: 30m + labels: + severity: warning + layer: k8s + team: aiops + auto_repair: "false" + alert_category: flywheel_health + notification_type: TYPE-8M + annotations: + summary: "飛輪 alertname NULL 率超過 5%" + description: "alertname NULL 率 {{ $value | humanizePercentage }},影響路由準確性。" + runbook: "執行 scripts/backfill_alertname.py 回填" + - alert: FlywheelIncidentsStuck + expr: awoooi_flywheel_incidents_stuck > 5 + for: 10m + labels: + severity: warning + layer: k8s + team: aiops + auto_repair: "false" + alert_category: flywheel_health + notification_type: TYPE-8M + annotations: + summary: "{{ $value }} 筆 Incident 卡在 INVESTIGATING 超過 24h" + description: "飛輪推理匹配節點可能堵塞,需人工清理或重新觸發診斷" + + # ========================================================================= + # 備份還原告警 (awoooi_backup_restore) — 從主機補回 2026-04-12 + # ========================================================================= + - name: awoooi_backup_restore + interval: 1h + rules: + - alert: BackupRestoreTestFailed + expr: awoooi_backup_restore_test_success == 0 + for: 5m + labels: + severity: critical + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "備份還原 dry-run 測試失敗" + description: "Velero restore dry-run 失敗,備份可能無法還原。立即人工驗證備份狀態。" + runbook: "執行 velero backup describe awoooi-daily 及 velero restore create --from-backup awoooi-daily --dry-run" + + - alert: BackupRestoreTestStale + expr: (time() - awoooi_backup_restore_test_timestamp_seconds) > 691200 + for: 10m + labels: + severity: warning + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "備份還原測試超過 8 天未執行" + description: "上次備份測試距今 {{ $value | humanizeDuration }},週排程 CronJob 可能失效。" + runbook: "檢查 velero namespace 中的 backup-restore-test CronJob 狀態" + + # ========================================================================= + # 基礎設施詳細告警 (awoooi_infrastructure_detailed) — 從主機補回 2026-04-12 + # ========================================================================= + - name: awoooi_infrastructure_detailed + interval: 60s + rules: + - alert: DockerContainerUnhealthyDetailed + expr: | + count by (name, instance) ( + container_tasks_state{state="running", instance=~"192.168.0.188.*"} + ) == 0 + or + container_last_seen{instance=~"192.168.0.188.*", name!=""} < (time() - 120) + for: 5m + labels: + severity: warning + layer: docker-188 + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "188 主機容器 {{ $labels.name }} 異常" + description: "容器 {{ $labels.name }} 在 {{ $labels.instance }} 已超過 2 分鐘無活動或不在 running 狀態。" + runbook: "SSH 到 192.168.0.188:docker inspect {{ $labels.name }} 確認健康狀態" + + - alert: RedisStreamBacklogHigh + expr: awoooi_redis_stream_len > 500 + for: 10m + labels: + severity: warning + layer: docker-188 + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "Redis Stream {{ $labels.stream }} 積壓 {{ $value }} 筆" + description: "Redis Stream 積壓超過 500 筆,飛輪消費者可能阻塞。" + runbook: "檢查 consumer group lag:XINFO GROUPS " + + # 2026-04-19 Hermes E3 決策: PostgreSQLDiskGrowthRate deprecated + # 真因: 500MB/h 增長是 PG WAL 正常行為 (commits/checkpoints),不該告警 + # 過去 30d 觸發 7 次,全部 AI 判 NO_ACTION 或誤判 kubectl rollout restart 失敗 + # 統帥決策 (2026-04-19 18:xx Taipei): 選項 C 刪除舊規則 + 改用絕對磁碟使用率 + # ----------------------------------------------------------------- + - alert: HostDiskUsageHigh + expr: | + ( + node_filesystem_size_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + ) + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + * 100 > 80 + for: 10m + labels: + severity: warning + layer: systemd-188 + alert_category: host_resource + notification_type: TYPE-3 + # 2026-05-02 ogt + Claude Sonnet 4.6: ADR-068 飛輪 — disk full SOP + # auto_repair: false → true,路由到 ssh_host MCP Group B `ssh_docker_prune` + # 工具內含 ≥75% 磁碟守衛,低於閾值 no-op,避免誤刪 + auto_repair: "true" + mcp_provider: "ssh_host" + host_type: "bare_metal" + supersedes: PostgreSQLDiskGrowthRate + annotations: + summary: "主機 {{ $labels.instance }} 根目錄磁碟使用率 {{ $value | humanize }}% (>80%)" + description: "磁碟使用率持續 10 分鐘超過 80%, 需清理或擴容. 常見原因: PG WAL, 日誌, container images, 舊 build cache." + auto_repair_action: "ssh {{ $labels.instance }} docker prune (image+volume+builder; gated by 75% disk usage)" + runbook: "SSH 該主機: df -h / && du -sh /var/lib/postgresql/*/pg_wal /var/log /var/lib/docker" + + - alert: HostDiskUsageCritical + expr: | + ( + node_filesystem_size_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + - node_filesystem_avail_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + ) + / node_filesystem_size_bytes{fstype!~"tmpfs|overlay", mountpoint="/"} + * 100 > 90 + for: 5m + labels: + severity: critical + layer: systemd-188 + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + supersedes: PostgreSQLDiskGrowthRate + annotations: + summary: "🔴 主機 {{ $labels.instance }} 根目錄磁碟使用率 {{ $value | humanize }}% (>90%, critical)" + description: "磁碟即將滿, 需立即清理. 超過 95% 可能導致服務中斷." + runbook: "立即 SSH 該主機: df -h / && du -sh /* 2>/dev/null | sort -h | tail -10" + + # ========================================================================= + # 主機連通性告警 (awoooi_host_connectivity) — 從主機補回 2026-04-12 + # ========================================================================= + - name: awoooi_host_connectivity + interval: 60s + rules: + - alert: HostNetworkPartition + expr: probe_success{job="host-connectivity"} == 0 + for: 5m + labels: + severity: critical + layer: systemd-188 + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + annotations: + summary: "主機 {{ $labels.instance }} 無法連通" + description: "TCP probe 到 {{ $labels.instance }} 失敗超過 5 分鐘,可能發生網路分區。" + runbook: "SSH 檢查路由和防火牆規則" + + # ========================================================================= + # 監控工具自監控 (infra_self_monitoring) — ADR-090 Phase 7 + # 2026-04-19 Claude Opus 4.7 / 鐵律:監控工具必須被監控 + # 設計:不寫死 CPU% 或 MB 數,改用 (配額佔比) + (throttle 訊號) 動態判斷 + # 配額由 docker-compose 宣告,告警條件 = 使用量 / 配額 > 0.8 + # 比寫死 80% 更智能 — 配額改告警閾值自動跟著變 + # ========================================================================= + - name: infra_self_monitoring + interval: 1m + rules: + + # --- cadvisor 自監控 --- + - alert: CadvisorDown + expr: up{job=~".*cadvisor.*"} == 0 + for: 5m + labels: + severity: critical + layer: docker-110-188 + component: cadvisor + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + annotations: + summary: "cAdvisor ({{ $labels.instance }}) 停擺" + description: "主機 {{ $labels.instance }} 的 cadvisor 已停擺 5 分鐘,容器監控中斷。" + runbook: "SSH 主機 docker compose up -d cadvisor;檢查 OOMKill 訊號" + + - alert: CadvisorMemoryPressure + expr: container_memory_usage_bytes{name="cadvisor"} / container_spec_memory_limit_bytes{name="cadvisor"} > 0.8 + for: 10m + labels: + severity: warning + component: cadvisor + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "cAdvisor 記憶體使用率 > 80% limit" + description: "cadvisor 記憶體用量 / mem_limit = {{ $value | humanizePercentage }},接近 OOMKill。" + runbook: "若頻繁觸發 → 檢查 cardinality 是否持續成長,考慮調整 --disable_metrics" + + - alert: CadvisorCPUThrottled + expr: rate(container_cpu_cfs_throttled_seconds_total{name="cadvisor"}[5m]) > 0.5 + for: 15m + labels: + severity: warning + component: cadvisor + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "cAdvisor CPU 被 throttle(配額不足)" + description: "cadvisor 每秒被 throttle {{ $value }} 秒,表示實際需求超過 cpus 配額。" + runbook: "調高 docker-compose cpus 設定,或檢查 scrape interval / cardinality" + + # --- node-exporter 自監控 --- + - alert: NodeExporterDown + expr: up{job=~"node-exporter.*|node_exporter.*"} == 0 + for: 5m + labels: + severity: critical + component: node-exporter + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "true" + annotations: + summary: "node-exporter ({{ $labels.instance }}) 停擺" + description: "主機 {{ $labels.instance }} node-exporter 已停擺 5 分鐘,主機 metrics 中斷。" + runbook: "SSH 主機檢查 docker ps node-exporter;重啟 docker compose up -d node-exporter" + + - alert: NodeExporterCPUThrottled + expr: rate(container_cpu_cfs_throttled_seconds_total{name="node-exporter"}[5m]) > 0.5 + for: 15m + labels: + severity: warning + component: node-exporter + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "node-exporter CPU 被 throttle(配額不足)" + description: "node-exporter 每秒被 throttle {{ $value }} 秒。可能 collector 未適度 disable。" + runbook: "檢查 node-exporter --collector.* flags 是否該關掉閒置硬體 probe" + + # --- Sentry self-hosted 自監控(110)--- + # 2026-04-25 ogt + Claude Opus 4.7: 修正假告警根因 + # 舊規則用 container_memory_usage_bytes(含 page cache),導致 ClickHouse + # 執行大查詢時 OS 把 SSTable 緩存進 page cache,比例衝到 88.5% 觸發誤報 + # (2026-04-23 23:13 鐵證:usage_bytes=88.5% / working_set=7.8%)。 + # 改用 container_memory_working_set_bytes — 這才是 K8s/Docker OOM killer + # 實際追蹤的「真實工作集」(RSS + active page cache),不含 inactive page cache。 + # 參考: https://github.com/google/cadvisor/blob/master/info/v1/container.go + - alert: SentryClickHouseMemoryPressure + expr: container_memory_working_set_bytes{name=~".*sentry.*clickhouse.*"} / container_spec_memory_limit_bytes{name=~".*sentry.*clickhouse.*"} > 0.85 + for: 10m + labels: + severity: warning + component: sentry-clickhouse + team: platform + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "Sentry ClickHouse 工作集記憶體 > 85% limit" + description: "sentry clickhouse working_set / mem_limit = {{ $value | humanizePercentage }} (排除 page cache)。" + runbook: "檢查 Sentry 查詢壓力;確認非 page cache 假象;必要時調整 /opt/sentry/docker-compose.override.yml clickhouse mem_limit" + + - alert: SentryClickHouseCPUThrottled + expr: rate(container_cpu_cfs_throttled_seconds_total{name=~".*sentry.*clickhouse.*"}[5m]) > 1.0 + for: 15m + labels: + severity: warning + component: sentry-clickhouse + team: platform + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "Sentry ClickHouse CPU 持續被 throttle" + description: "每秒 throttle {{ $value }} 秒,配額 cpus=4.0 可能不足。" + runbook: "檢查 Sentry retention / query pattern;必要時調高 override.yml cpus" + + # --- Gitea 自監控 --- + - alert: GiteaMemoryPressure + # 2026-04-25 ogt + Claude Sonnet 4.6 — 同 ClickHouse 假警報根因: + # container_memory_usage_bytes 含 page cache(OS inactive,OOM killer 不管)→ 虛高假警報 + # 改用 container_memory_working_set_bytes(RSS + active cache,真實壓力,cadvisor 適用 Docker + K8s) + expr: container_memory_working_set_bytes{name="gitea"} / container_spec_memory_limit_bytes{name="gitea"} > 0.85 + for: 10m + labels: + severity: warning + component: gitea + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "Gitea 記憶體工作集 > 85% limit" + description: "gitea working_set / mem_limit = {{ $value | humanizePercentage }}(真實記憶體壓力,非 page cache 干擾)。" + runbook: "檢查 CI/CD 任務堆積;必要時調高 docker-compose mem_limit" + + - alert: GiteaCPUThrottled + expr: rate(container_cpu_cfs_throttled_seconds_total{name=~"gitea|gitea-runner"}[5m]) > 1.0 + for: 15m + labels: + severity: warning + component: gitea + team: ops + alert_category: infrastructure + notification_type: TYPE-1 + auto_repair: "false" + annotations: + summary: "Gitea / Runner CPU 持續被 throttle" + description: "{{ $labels.name }} 每秒 throttle {{ $value }} 秒,CD peak 可能卡關。" + runbook: "檢查 job 並行度;考慮縮減並行或調高 cpus" + + # --- 監控自監控元層(Prometheus 本身)--- + - alert: PrometheusDown + expr: up{job="prometheus"} == 0 + for: 2m + labels: + severity: critical + component: prometheus + team: ops + alert_category: infrastructure + notification_type: TYPE-3 + auto_repair: "false" + annotations: + summary: "Prometheus ({{ $labels.instance }}) 停擺" + description: "Prometheus 自己停擺 → 所有其他告警失效" + runbook: "SSH 110 docker compose -f /home/wooo/monitoring/docker-compose.yml up -d prometheus"