diff --git a/apps/api/src/core/config.py b/apps/api/src/core/config.py index 02457f930..9bad02662 100644 --- a/apps/api/src/core/config.py +++ b/apps/api/src/core/config.py @@ -62,6 +62,21 @@ class Settings(BaseSettings): description="Phase 24: True=新 AIRouter 路由, False=舊 openclaw.py fallback chain", ) + # ========================================================================== + # P1-1: KMWriter 統一契約 (2026-04-28 ogt + Claude Sonnet 4.6) + # KM_WRITE_AWAIT=true → 強制 await asyncio.wait_for(timeout=KM_WRITE_TIMEOUT_SECONDS) + # KM_WRITE_AWAIT=false → 舊 fire-and-forget 行為(回滾用) + # 回滾指令: kubectl set env deployment/awoooi-api KM_WRITE_AWAIT=false + # ========================================================================== + KM_WRITE_AWAIT: bool = Field( + default=True, + description="P1-1: True=強制 await KM 寫入(可靠), False=fire-and-forget(舊行為,回滾用)", + ) + KM_WRITE_TIMEOUT_SECONDS: float = Field( + default=5.0, + description="P1-1: KMWriter await timeout(秒),超時記錄 warning 但不阻塞主流程", + ) + # ========================================================================== # aider-watch v2 integration (2026-04-20 ADR-091) # 整合 Mac aider CLI 監控進 awoooi 飛輪(events → incident → ai_router feedback) @@ -354,11 +369,13 @@ class Settings(BaseSettings): # ========================================================================== # AI Fallback Strategy (ADR-006 v1.3 + ADR-036) - # Order: Ollama (local) -> Gemini (cloud) -> Claude (cloud) + # Order: Ollama (local) -> NVIDIA NIM -> Gemini (cloud) -> Claude (cloud) # Tool Calling: Nemotron (專用) -> Gemini -> Claude + # 2026-04-28 ogt + Claude Opus 4.7: 補 nvidia 對齊 ConfigMap (k8s/awoooi-prod/04-configmap.yaml) + # 之前 default 缺 nvidia 與 ConfigMap drift,feedback_ai_fallback_order.md 鐵律 # ========================================================================== AI_FALLBACK_ORDER: list[str] = Field( - default=["ollama", "gemini", "claude"], + default=["ollama", "nvidia", "gemini", "claude"], description="AI provider fallback order", ) GEMINI_API_KEY: str = Field(default="", description="Google Gemini API key") @@ -663,9 +680,12 @@ class Settings(BaseSettings): # ========================================================================== # MCP Phase 2b: Prometheus MCP Server (ADR-071, 2026-04-11 Claude Sonnet 4.6) # ========================================================================== + # 2026-04-29 ogt + Claude Opus 4.7: drift fix — 188 是 Ollama Hub,Prometheus 實際在 110 + # ConfigMap 04-configmap.yaml 也是 110;governance_agent / SLO check 連 188 會 timeout + # 此 drift 是 SPF-4 (governance_agent silently fail) 根因之一 PROMETHEUS_URL: str = Field( - default="http://192.168.0.188:9090", - description="Prometheus server URL", + default="http://192.168.0.110:9090", + description="Prometheus server URL (DevOps 金庫主機)", ) PROMETHEUS_MCP_ENABLED: bool = Field( default=True, @@ -685,8 +705,10 @@ class Settings(BaseSettings): # MCP Phase 3: ArgoCD MCP Server (2026-04-11 Claude Sonnet 4.6) # ========================================================================== + # 2026-04-29 ogt + Claude Opus 4.7: drift fix — ArgoCD 實際在 121,125 是舊位置 + # ConfigMap 04-configmap.yaml 已寫 121;T0 G3 drift 已知,此次正式對齊 ARGOCD_URL: str = Field( - default="https://192.168.0.125:30443", + default="https://192.168.0.121:30443", description="ArgoCD API Server URL(K3s NodePort HTTPS)", ) ARGOCD_API_TOKEN: str = Field( diff --git a/apps/api/src/services/governance_agent.py b/apps/api/src/services/governance_agent.py index 86bafd0c1..815faefbc 100644 --- a/apps/api/src/services/governance_agent.py +++ b/apps/api/src/services/governance_agent.py @@ -303,7 +303,24 @@ class GovernanceAgent: data = resp.json() if data.get("status") == "success": result_list = data.get("data", {}).get("result", []) - value = float(result_list[0]["value"][1]) if result_list else 0.0 + # 2026-04-28 ogt + Claude Opus 4.7: P0-1 假警報止血 + # 空 result = Prometheus 查無資料(metric 未 emit / rule 未部署),不等於 SLO=0 + # ADR-100 emitter 全部尚未實作(automation_operation_log_total 等 4 個 counter 零定義) + # 不可 fallback 0.0,否則必觸發 violated=True 噴假警報 + if not result_list: + results[name] = { + "error": "no_data", + "skipped": True, + "reason": "prometheus_empty_result_metric_not_emitted", + } + logger.warning( + "governance_slo_no_data", + slo=name, + query=query, + hint="ADR-100 emitter not yet implemented", + ) + continue + value = float(result_list[0]["value"][1]) threshold = hard_red_lines[name] target = slo_targets[name] violated = value < threshold diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index e978291fc..dc815c420 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -1658,7 +1658,19 @@ class TelegramGateway: await redis.setex(key, 3600, value) logger.debug("llm_button_redis_written", count=len(redis_writes)) except Exception as exc: - logger.warning("llm_button_redis_write_failed", error=str(exc)) + # 2026-04-28 ogt + Claude Opus 4.7: P0-4 鬼魂按鈕守門 + # feedback_no_ghost_buttons.md 三缺一鐵律:callback 對應 short_id 找不到 = 鬼魂 + # Redis 寫入失敗 → LLM 動態按鈕的 callback_data 在 Redis 撈不到 payload → 鬼魂風險 + # 對策:清空 LLM 動態按鈕,caller (build_keyboard) 1488 行的 first_row 永遠保留 + # (✅ 批准 / ❌ 拒絕 用 HMAC nonce,無狀態,不依賴 Redis) + # 統帥仍可走核心通道,少了 LLM 推薦的 specific actions(可接受的降級) + logger.error( + "llm_button_redis_write_failed_fallback_to_static", + error=str(exc), + dropped_count=len(btn_list), + hint="user will see only first_row (approve/reject), LLM-recommended actions dropped", + ) + btn_list.clear() # 每排最多 2 個 rows: list[list[dict]] = [btn_list[i:i+2] for i in range(0, len(btn_list), 2)] diff --git a/ops/alertmanager/alertmanager.yml b/ops/alertmanager/alertmanager.yml index efa254c06..826070817 100644 --- a/ops/alertmanager/alertmanager.yml +++ b/ops/alertmanager/alertmanager.yml @@ -7,32 +7,43 @@ # 架構: awoooi-webhook (主路徑) + telegram-direct (fallback,獨立路由) # 當 AWOOOI API 無法回應時,critical 告警直接送 Telegram Bot API # ⚠️ bot_token/chat_id 部署時由 secrets 替換,此檔為模板 +# +# 2026-04-29 ogt + Claude Opus 4.7: P1-4 新版語法升級 + 因果抑制擴展 +# 改動: +# 1. match/match_re → matchers (Alertmanager v0.27+ deprecated 警告) +# 2. source_match/target_match/target_match_re → source_matchers/target_matchers +# 3. group_by 加 team label(防 4 條 SLO 同秒爆,依 web-researcher 文件) +# 4. PostgreSQLDown / RedisDown inhibit 補 equal: ['instance'](防全 ns 爆炸抑制) +# 5. 新增 OllamaInstanceDown / KMConverterDown / SLO FastBurn 三組因果抑制 +# 根因:本次 4 SLO 雪崩證實 Ollama 111 掛 → AI 推理鏈斷 → SLO 級聯爆炸無守門 +# 6. 命名鐵律 feedback_telegram_alert_format.md 對齊(label team=ai/component/auto_repair) global: resolve_timeout: 5m route: receiver: 'awoooi-webhook' - group_by: ['alertname', 'severity'] + # 2026-04-29: 加 team — SLO/AI PrometheusRule 含 team=ai 時可獨立分組合併 + group_by: ['team', 'alertname', 'severity'] group_wait: 30s group_interval: 5m repeat_interval: 4h routes: - - match: - severity: critical + - matchers: + - severity="critical" receiver: 'awoooi-webhook' group_wait: 10s # continue:true 讓 critical 同時送 telegram-direct (fallback) continue: true - - match: - severity: critical + - matchers: + - severity="critical" receiver: 'telegram-direct' group_wait: 10s - - match: - severity: warning + - matchers: + - severity="warning" receiver: 'awoooi-webhook' - - match_re: - alertname: 'Zombie.*|Container.*' + - matchers: + - alertname=~"Zombie.*|Container.*" receiver: 'awoooi-webhook' group_wait: 1m @@ -41,6 +52,8 @@ receivers: # 2026-04-16 ogt + Claude Sonnet 4.6: 改指向 VIP 192.168.0.125 # 根因: 121:32334 Connection Refused,120:32334 也 Refused # 只有 VIP 125:32334 可連通(kube-proxy NodePort 路由正常) + # ⚠️ SPF-1 風險:VIP 125 為單點,VIP host 整機 down → 主鏈斷 + # 緩解計畫見 critic SPF 治理設計(中度方案:webhook_configs 多 url round-robin) - name: 'awoooi-webhook' webhook_configs: - url: 'http://192.168.0.125:32334/api/v1/webhooks/alertmanager' @@ -48,6 +61,8 @@ receivers: # Fallback 路徑: AWOOOI API 掛掉時,critical 告警直接送 Telegram # 只有 critical severity 走此路徑(避免 warning 雙重通知) + # ⚠️ bot_token / chat_id 由 CD pipeline 在 deploy 時用 K8s Secret 注入 + # feedback_telegram_secrets_injection.md 鐵律:禁止 PLACEHOLDER 上線 - name: 'telegram-direct' telegram_configs: - bot_token: 'TELEGRAM_BOT_TOKEN_PLACEHOLDER' @@ -65,26 +80,60 @@ receivers: send_resolved: false inhibit_rules: - - source_match: - severity: critical - target_match: - severity: warning + # === 基礎因果抑制(原有規則,新語法重寫)=== + - source_matchers: + - severity="critical" + target_matchers: + - severity="warning" equal: ['alertname', 'instance'] - - source_match: - alertname: HostDown - target_match_re: - alertname: 'HostHighCpuLoad|HostOutOfMemory|HostOutOfDiskSpace' + + - source_matchers: + - alertname="HostDown" + target_matchers: + - alertname=~"HostHighCpuLoad|HostOutOfMemory|HostOutOfDiskSpace" equal: ['host'] - - source_match: - alertname: KubeNodeNotReady - target_match_re: - alertname: 'KubePodCrashLooping|KubePodNotReady|KubeDeploymentReplicasMismatch' + + - source_matchers: + - alertname="KubeNodeNotReady" + target_matchers: + - alertname=~"KubePodCrashLooping|KubePodNotReady|KubeDeploymentReplicasMismatch" equal: ['node'] - - source_match: - alertname: PostgreSQLDown - target_match: - alertname: PostgreSQLHighConnections - - source_match: - alertname: RedisDown - target_match: - alertname: RedisMemoryHigh + + # 2026-04-29: 補 equal: ['instance'] — 原本缺,PG 在 instance A down + # 不該抑制 instance B 的 HighConnections(爆炸抑制 bug) + - source_matchers: + - alertname="PostgreSQLDown" + target_matchers: + - alertname="PostgreSQLHighConnections" + equal: ['instance'] + + - source_matchers: + - alertname="RedisDown" + target_matchers: + - alertname="RedisMemoryHigh" + equal: ['instance'] + + # === 新增:AI 鏈因果抑制(2026-04-29 ADR-035 因果抑制擴展)=== + # 根因:本次 4 SLO 雪崩證實 Ollama 111 掛 → AI 推理鏈斷 → 4 SLO 同秒爆 + # 無此抑制 → 假警報淹沒真警報(Ollama down 本身才是真信號) + + # Ollama 任一實例掛 → 抑制所有 AI/SLO 告警 30 分鐘 + - source_matchers: + - alertname="OllamaInstanceDown" + target_matchers: + - alertname=~"SLO_.*|AI_.*" + equal: [] + + # KM converter 掛 → 抑制 KM Growth Rate SLO(避免 KM 寫入失敗本身觸發 SLO) + - source_matchers: + - alertname="KMConverterDown" + target_matchers: + - alertname=~"SLO_KMGrowthRate.*" + equal: [] + + # 同 SLO 較嚴重抑制較輕(FastBurn 抑制 Medium/Slow Burn) + - source_matchers: + - alertname=~"SLO_.+_FastBurn" + target_matchers: + - alertname=~"SLO_.+_(Medium|Slow)Burn" + equal: ['alertname'] diff --git a/scripts/check_config_drift.py b/scripts/check_config_drift.py new file mode 100755 index 000000000..9aa29fe70 --- /dev/null +++ b/scripts/check_config_drift.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python3 +# 2026-04-28 ogt + Claude Opus 4.7: P2-1 ConfigMap vs code default drift checker +# 來源:tool-expert 統一治理方案 +# 目的:CI / pre-commit 階段驗證 k8s ConfigMap 與 apps/api/src/core/config.py default 一致 +# 違反「事實驅動」紅線案例:AI_FALLBACK_ORDER、ARGOCD_URL 都曾發生 drift +""" +ConfigMap vs Code Default Drift Checker + +用法: + python3 scripts/check_config_drift.py +退出碼: + 0 = 全部對齊 + 1 = 至少一項 drift(CI 應 fail) + +可加進 .pre-commit-config.yaml: +- repo: local + hooks: + - id: config-drift-check + name: ConfigMap vs code default drift + entry: python3 scripts/check_config_drift.py + language: python + pass_filenames: false + additional_dependencies: [pyyaml] +""" +from __future__ import annotations + +import json +import re +import sys +from pathlib import Path + +import yaml # noqa: F401 pre-commit 會經 additional_dependencies 安裝 + +ROOT = Path(__file__).resolve().parent.parent +CONFIGMAP_PATH = ROOT / "k8s" / "awoooi-prod" / "04-configmap.yaml" +CONFIG_PY_PATH = ROOT / "apps" / "api" / "src" / "core" / "config.py" + +# 需要比對的欄位 +# code_default_pattern: 在 config.py 找 default=... 用的 regex(DOTALL) +CHECK_FIELDS: dict[str, dict[str, str]] = { + "AI_FALLBACK_ORDER": { + "configmap_key": "AI_FALLBACK_ORDER", + "code_pattern": r"AI_FALLBACK_ORDER:\s*list\[str\]\s*=\s*Field\([^)]*?default=(\[[^\]]+\])", + }, + "ARGOCD_URL": { + "configmap_key": "ARGOCD_URL", + "code_pattern": r"ARGOCD_URL[^\n]*?\n[^)]*?default=[\"']([^\"']+)[\"']", + }, + "PROMETHEUS_URL": { + "configmap_key": "PROMETHEUS_URL", + "code_pattern": r"PROMETHEUS_URL[^\n]*?\n[^)]*?default=[\"']([^\"']+)[\"']", + }, + "OLLAMA_URL": { + "configmap_key": "OLLAMA_URL", + "code_pattern": r"OLLAMA_URL[^\n]*?\n[^)]*?default=[\"']([^\"']+)[\"']", + }, +} + + +def _normalize(raw: str) -> object: + """嘗試把字串解析成 list/dict,失敗就回原字串。""" + raw_strip = raw.strip().strip("'\"") + if raw_strip.startswith("["): + try: + return json.loads(raw_strip.replace("'", '"')) + except json.JSONDecodeError: + return raw_strip + return raw_strip + + +def main() -> int: + if not CONFIGMAP_PATH.exists(): + print(f"[ERROR] ConfigMap not found: {CONFIGMAP_PATH}") + return 2 + if not CONFIG_PY_PATH.exists(): + print(f"[ERROR] config.py not found: {CONFIG_PY_PATH}") + return 2 + + with CONFIGMAP_PATH.open() as fh: + cm_data: dict = yaml.safe_load(fh).get("data", {}) or {} + py_src = CONFIG_PY_PATH.read_text() + + exit_code = 0 + print("=== ConfigMap ↔ code.default Drift Check ===") + for field, spec in CHECK_FIELDS.items(): + cm_raw = cm_data.get(spec["configmap_key"], "") + m = re.search(spec["code_pattern"], py_src, re.DOTALL) + py_raw = m.group(1) if m else "" + + cm_val = _normalize(cm_raw) + py_val = _normalize(py_raw) + + if cm_val == py_val: + print(f"[OK] {field}: {cm_val}") + else: + print(f"[DRIFT] {field}:") + print(f" ConfigMap = {cm_val}") + print(f" config.py = {py_val}") + exit_code = 1 + + if exit_code == 0: + print("=== All drift-check fields aligned ===") + else: + print("=== DRIFT detected, fix the inconsistency ===") + return exit_code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/health_check_session.sh b/scripts/health_check_session.sh new file mode 100755 index 000000000..d93924cb2 --- /dev/null +++ b/scripts/health_check_session.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# 2026-04-28 ogt + Claude Opus 4.7: P2-2 Session 啟動健康驗證 +# 來源:tool-expert 統一治理方案 +# 目的:每次 Claude session 啟動時快速確認 5 主機 + 關鍵服務可達 +# 純 read-only(curl + ssh -o BatchMode),不修改任何狀態 +# +# 用法: +# bash scripts/health_check_session.sh +# 或加 alias: alias awoooi-health='bash ~/awoooi/scripts/health_check_session.sh' + +set -uo pipefail # 不要 -e,個別 check 失敗不阻擋全部 + +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[0;33m' +NC='\033[0m' + +ok() { printf "${GREEN}[OK]${NC} %s\n" "$1"; } +fail() { printf "${RED}[FAIL]${NC} %s\n" "$1"; } +warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$1"; } + +check_url() { + local name=$1 url=$2 + local code + code=$(curl -sk --max-time 3 -o /dev/null -w "%{http_code}" "$url" 2>/dev/null || echo "000") + if [[ "$code" =~ ^[23] ]]; then + ok "$name → $url ($code)" + return 0 + elif [[ "$code" =~ ^[45] ]]; then + warn "$name → $url ($code, 服務有回應但非 2xx/3xx)" + return 0 + else + fail "$name → $url (unreachable)" + return 1 + fi +} + +check_ssh() { + local name=$1 host=$2 + if ssh -o ConnectTimeout=3 -o BatchMode=yes -o StrictHostKeyChecking=no \ + "$host" "echo ok" 2>/dev/null | grep -q ok; then + ok "SSH $name ($host)" + return 0 + else + fail "SSH $name ($host) — 無法連線(timeout / 認證失敗 / 主機不可達)" + return 1 + fi +} + +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +TS="$(date '+%Y-%m-%d %H:%M %Z')" + +echo "==========================================" +echo "AWOOOI Session Health Check $TS" +echo "==========================================" + +echo "" +echo "--- K8s 控制平面 ---" +check_url "K3s VIP API" "https://192.168.0.125:6443/healthz" +check_url "ArgoCD (121)" "https://192.168.0.121:30443" + +echo "" +echo "--- AI 推理層 ---" +check_url "Ollama 111 GPU" "http://192.168.0.111:11434/api/tags" +check_url "Ollama 188 Hub" "http://192.168.0.188:11434/api/tags" + +echo "" +echo "--- 觀測層 ---" +check_url "Prometheus 110" "http://192.168.0.110:9090/-/healthy" +check_url "Alertmanager 110" "http://192.168.0.110:9093/-/healthy" +check_url "Gitea 110" "http://192.168.0.110:3001" +check_url "Langfuse 110" "http://192.168.0.110:3100" + +echo "" +echo "--- AWOOOI 核心服務 (prod NodePort) ---" +check_url "AWOOOI API (125)" "http://192.168.0.125:32334/api/v1/health" + +echo "" +echo "--- SSH 連通 ---" +check_ssh "awoooi-devops (110)" "wooo@192.168.0.110" +check_ssh "k3s-1 (120)" "wooo@192.168.0.120" +check_ssh "k3s-2 (121)" "wooo@192.168.0.121" +check_ssh "ollama-111-gpu (ProxyJump 110)" "ollama-111-gpu" + +echo "" +echo "--- Config Drift Check ---" +if [ -x "$ROOT/scripts/check_config_drift.py" ]; then + python3 "$ROOT/scripts/check_config_drift.py" || warn "config drift detected (見上方 [DRIFT] 行)" +else + warn "drift checker 不存在 ($ROOT/scripts/check_config_drift.py)" +fi + +echo "" +echo "--- Git 狀態 ---" +if [ -d "$ROOT/.git" ]; then + cd "$ROOT" || exit + branch=$(git branch --show-current 2>/dev/null || echo "") + upstream_diff=$(git rev-list --count "@{u}..HEAD" 2>/dev/null || echo "?") + echo " 分支: $branch (本地超前上游 $upstream_diff 個 commit)" + if ! git diff --quiet 2>/dev/null; then + warn " 有未 commit 的變更(git status 自查)" + else + ok " 工作目錄 clean" + fi +fi + +echo "" +echo "==========================================" +echo "Session Health Check 結束" +echo "=========================================="