From edba52f401351d287f9c6aacc032cd428fc8ab34 Mon Sep 17 00:00:00 2001 From: AWOOOI CD Date: Wed, 13 May 2026 19:19:24 +0800 Subject: [PATCH 1/3] chore(cd): deploy 596f2f6 [skip ci] --- k8s/awoooi-prod/kustomization.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/awoooi-prod/kustomization.yaml b/k8s/awoooi-prod/kustomization.yaml index 8edb74bac..47c273be3 100644 --- a/k8s/awoooi-prod/kustomization.yaml +++ b/k8s/awoooi-prod/kustomization.yaml @@ -40,7 +40,7 @@ resources: images: - name: 192.168.0.110:5000/library/api:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/api - newTag: 3bad354414edcef35406796b9b9e2cfb90b0740f + newTag: 596f2f682094d0916f6a18a6f50e7667e4ca86ff - name: 192.168.0.110:5000/library/web:IMAGE_TAG_PLACEHOLDER newName: 192.168.0.110:5000/awoooi/web - newTag: 3bad354414edcef35406796b9b9e2cfb90b0740f + newTag: 596f2f682094d0916f6a18a6f50e7667e4ca86ff From 872abea00844c6b1ff6f87a4729aa6a4d870d800 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 13 May 2026 19:24:01 +0800 Subject: [PATCH 2/3] docs(awooop): record t14b auto approved evidence link --- docs/LOGBOOK.md | 84 +++++++++++++++++++ ...-04-15-MASTER-ai-autonomous-flywheel-v2.md | 8 ++ 2 files changed, 92 insertions(+) diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index e7521d473..5ec011f14 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -7751,3 +7751,87 @@ auto_repair_24h=6 - 目前 production smoke 沒有新的 auto-repair 事件可驗證 fallback 寫入，因此仍不能宣稱完整閉環；這是正確保守判讀。 - 下一步 T14b：等下一筆 `auto_repair=true` 事件或設計安全 live-fire，驗證 `auto_repair_executions -> incident_evidence.verification_result -> learning/KM -> truth-chain auto_repaired_verified` 是否全鏈路成立；同時補 auto-approved approval execution 的 incident linkage / durable execution record。 - 目前整體進度更新：約 80%。 + +### 2026-05-13 — AwoooP truth-chain T14b：auto-approved execution 補 incident linkage 與 durable evidence（production deployed） + +**live diagnosis**： + +- CS2 `auto_approve_rule_engine` 與 CS3 `auto_approve_llm_cs3` 的高信心自動執行路徑，是先呼叫 `ApprovalExecutionService.execute_approved_action()`，再建立 incident。 +- executor 執行當下沒有 `incident_id`，因此 post-execution verifier、KM writeback、incident resolve、`auto_repair_executions` 都無法串回同一張告警。 +- CS3 另有一個實際斷點：auto approval 沒有把 DB 內 `approval.id` 帶給 executor，會讓執行狀態回寫到錯的 transient id。 + +**變更**： + +- `ApprovalExecutionService.finalize_auto_approved_execution()` 新增為「不重跑 action，只補證據鏈」的收斂點： + - 寫入 `auto_repair_executions`，`triggered_by=auto_approve_*`。 + - 補 incident-linked timeline event。 + - 以自動修復模式寫 KM。 + - 呼叫 `PostExecutionVerifier`，`action_taken=auto_repair_playbook:*`，讓 fallback evidence 可取得 `matched_playbook_id`。 + - 成功後 resolve incident。 + - `NO_ACTION` / `OBSERVE` / `INVESTIGATE` 不算自動修復，避免 KPI 污染。 +- CS2 / CS3 在 incident 建立與 `update_incident_id()` 後呼叫 finalize。 +- CS3 補 `_cs3_auto_approval.id = approval.id` 與 `service.update_execution_status()`。 +- `requested_by` 判斷從只接受 `auto_approve` 改成接受 `auto_approve*`，避免 `auto_approve_rule_engine` / `auto_approve_llm_cs3` 被 KM 誤標成人工修復。 + +**local verification**： + +```text +python3 -m py_compile apps/api/src/services/approval_execution.py apps/api/src/api/v1/webhooks.py apps/api/tests/test_approval_execution_auto_approved_finalize.py +OK + +ruff check --select F821 apps/api/src/services/approval_execution.py apps/api/src/api/v1/webhooks.py apps/api/tests/test_approval_execution_auto_approved_finalize.py +OK + +pytest tests/test_approval_execution_auto_approved_finalize.py tests/test_approval_execution_no_action.py tests/test_learning_chain_e2e.py tests/test_awooop_truth_chain_service.py -q +26 passed + +pytest tests/test_post_execution_verifier.py tests/test_learning_chain_e2e.py tests/test_awooop_truth_chain_service.py tests/test_platform_router_order.py tests/test_cs1_auto_execute.py tests/test_cs3_auto_execute.py tests/test_approval_execution_auto_approved_finalize.py -q +77 passed + +pytest tests/test_rule_engine_auto_execute.py tests/test_alertmanager_rule_bypass.py tests/test_approval_execution_auto_approved_finalize.py -q +31 passed +``` + +**production deploy / smoke（完成）**： + +```text +Commit: 596f2f68 fix(awooop): link auto approved execution evidence +Gitea: +2066 code-review 596f2f68 -> success +2065 CD Pipeline 596f2f68 -> success + tests -> success + build-and-deploy -> success + post-deploy-checks -> success +Deploy marker: edba52f4 chore(cd): deploy 596f2f6 [skip ci] + +K8s image: +awoooi-api 192.168.0.110:5000/awoooi/api:596f2f682094d0916f6a18a6f50e7667e4ca86ff +awoooi-worker 192.168.0.110:5000/awoooi/api:596f2f682094d0916f6a18a6f50e7667e4ca86ff +awoooi-web 192.168.0.110:5000/awoooi/web:596f2f682094d0916f6a18a6f50e7667e4ca86ff + +health: +https://awoooi.wooo.work/api/v1/health -> 200 + +quality summary, hours=24, limit=30: +verified_auto_repair_total=0 +production_claim.can_claim_full_auto_repair=false +by_verdict: + manual_required_no_action=17 + received_only=12 + approval_required=1 + +DB baseline after deploy time 2026-05-13T11:19:27Z: +auto_repair_since_deploy=0 +auto_approved_since_deploy=0 +verified_evidence_since_deploy=0 +auto_repair_24h=5 +auto_approved_24h=0 +verified_evidence_24h=0 +``` + +判讀： + +- T14b 已完成並推版：下一筆 CS2/CS3 auto-approved real execution 會留下 incident-linked `auto_repair_executions`、timeline、KM、verifier evidence，不再只停留在 Telegram / log。 +- production smoke 尚未出現部署後新的 auto-approved 或 auto-repair live event，因此仍不能宣稱完整閉環已被 production live-fire 證明。 +- 下一步 T14c：用安全 live-fire 或等待自然告警，驗證 `auto_approve_* -> auto_repair_executions -> incident_evidence.verification_result -> learning/KM -> truth-chain auto_repaired_verified` 實際打通；並把 Telegram 卡片改成明確顯示「目前跑到哪個節點 / 是否已自動修復 / 是否轉人工」。 +- 目前整體進度更新：約 82%。 diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index f210d36f6..fba07b08e 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -2040,6 +2040,14 @@ Phase 6 完成後 - Smoke：quality summary 仍為 `verified_auto_repair_total=0`、`production_claim=false`；deploy 後尚無新 auto-repair 事件（`auto_repair_since_deploy=0`），所以不能宣稱完整閉環，只能宣稱「未來 auto-repair verifier 結果會有 durable evidence target」。 - 下一步 T14b：等待下一筆 `auto_repair=true` 事件或設計安全 live-fire，驗證 `auto_repair_executions -> incident_evidence.verification_result -> learning/KM -> truth-chain auto_repaired_verified` 全鏈路；並補 auto-approved approval execution 的 incident linkage / durable execution record。 +**T14b auto-approved execution incident linkage production deployed（2026-05-13 台北）**： +- 觸發：CS2 `auto_approve_rule_engine` 與 CS3 `auto_approve_llm_cs3` 會先執行 action、再建立 incident；executor 當下沒有 `incident_id`，導致 `auto_repair_executions`、timeline、KM、PostExecutionVerifier、incident resolve 無法串回同一事件。CS3 另缺 `_cs3_auto_approval.id = approval.id`，會讓 execution status 回寫到 transient id。 +- 修正：新增 `ApprovalExecutionService.finalize_auto_approved_execution()`，在 incident 建立後補 durable trace，不重新執行 action；內容包含 `auto_repair_executions(triggered_by=auto_approve*)`、incident-linked timeline、KM、`PostExecutionVerifier(action_taken=auto_repair_playbook:*)`、成功後 resolve incident。`NO_ACTION` / `OBSERVE` / `INVESTIGATE` 不算自動修復。 +- Webhook：CS2 / CS3 在 `update_incident_id()` 後呼叫 finalize；CS3 補 DB approval id 與 `update_execution_status()`；`requested_by` 判斷改為 `auto_approve*`，避免 `auto_approve_rule_engine` / `auto_approve_llm_cs3` 被誤標成人工修復。 +- Production：`596f2f68 fix(awooop): link auto approved execution evidence` 已推 Gitea main；Gitea run `2066` code-review success、run `2065` tests/build-and-deploy/post-deploy-checks 全 success；deploy marker `edba52f4`；API/Worker image `192.168.0.110:5000/awoooi/api:596f2f682094d0916f6a18a6f50e7667e4ca86ff`，Web image `192.168.0.110:5000/awoooi/web:596f2f682094d0916f6a18a6f50e7667e4ca86ff`，health 200。 +- Smoke：quality summary 仍為 `verified_auto_repair_total=0`、`production_claim=false`；deploy 後尚無新 auto-approved 或 auto-repair live event（`auto_repair_since_deploy=0`、`auto_approved_since_deploy=0`、`verified_evidence_since_deploy=0`），所以仍不能宣稱完整閉環已 production live-fire verified。 +- 下一步 T14c：用安全 live-fire 或等待自然告警驗證 `auto_approve_* -> auto_repair_executions -> incident_evidence.verification_result -> learning/KM -> truth-chain auto_repaired_verified`；並把 Telegram 卡片改為明確顯示流程節點、是否自動修復、是否轉人工。 + --- ### 2026-04-20 晚 (台北) — C1-C4 全流程串接 — Playbook 鏈路保護（commit de2d34d） From 74c47672da3d61dc649d118e9a95579b597ef848 Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 13 May 2026 19:29:51 +0800 Subject: [PATCH 3/3] feat(telegram): show automation flow progress --- apps/api/src/services/telegram_gateway.py | 115 ++++++++++++++++++ .../test_telegram_ai_automation_block.py | 40 ++++++ 2 files changed, 155 insertions(+) diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index 22aa60f1a..6aa7bc1ba 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -412,6 +412,7 @@ class TelegramMessage: alert_category: str = "" # host/k8s/database/service/external_site/secops 等 playbook_name: str = "" # 匹配到的 Playbook 名稱（空字串=規則匹配） automation_state: str = "" # diagnosis_collected_manual_required / diagnosis_failed_manual_required + automation_quality: dict | None = None # truth-chain automation_quality 摘要 # ========================================================================== # Phase 22: Nemotron 協作欄位 (ADR-044) @@ -471,6 +472,23 @@ class TelegramMessage: action = (self.suggested_action or "").upper() text = f"{self.root_cause} {self.suggested_action}".lower() state = (self.automation_state or "").lower() + quality = self.automation_quality or {} + facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {} + verdict = str(quality.get("verdict") or "") + auto_repair_records = int(facts.get("auto_repair_execution_records") or 0) + operation_records = int(facts.get("automation_operation_records") or 0) + verification = str(facts.get("verification_result") or "missing") + + if verdict == "auto_repaired_verified": + return "✅ 已驗證自動修復完成" + if auto_repair_records > 0 or operation_records > 0: + if verification == "missing": + return "🔄 已自動執行，等待驗證證據" + return f"🔄 已自動執行，驗證結果：{verification}" + if verdict == "approval_required": + return "🟡 需要審批後才會執行" + if verdict.startswith("manual_required"): + return "🟠 未自動修復，需人工判斷" if state == "diagnosis_collected_manual_required": return "🔎 AI 已完成只讀診斷，需人工判斷" @@ -518,6 +536,75 @@ class TelegramMessage: f"└ Flow：{flow}\n" ) + def _format_flow_progress_block(self) -> str: + """Operator-facing state of where the alert is in the automation loop.""" + quality = self.automation_quality or {} + facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {} + verdict = str(quality.get("verdict") or self._automation_mode()) + + action_upper = (self.suggested_action or "").upper() + is_noop = ( + "NO_ACTION" in action_upper + or action_upper.startswith("OBSERVE") + or action_upper.startswith("INVESTIGATE") + or not action_upper.strip() + or action_upper == "待分析" + ) + auto_repair_records = int(facts.get("auto_repair_execution_records") or 0) + operation_records = int(facts.get("automation_operation_records") or 0) + verification = str(facts.get("verification_result") or "missing") + gateway_total = int(facts.get("mcp_gateway_total") or 0) + km_entries = int(facts.get("knowledge_entries") or 0) + + if self.confidence > 0: + diagnose_state = "ai_ready" + elif self.automation_state == "diagnosis_failed_manual_required": + diagnose_state = "failed" + else: + diagnose_state = "rule_or_degraded" + + match_state = self.playbook_name or "rule_catalog" + if auto_repair_records > 0: + execute_state = f"auto_repair_recorded:{auto_repair_records}" + elif operation_records > 0: + execute_state = f"operation_recorded:{operation_records}" + elif is_noop: + execute_state = "no_action_or_observe" + elif "approval" in verdict or self._automation_mode() == "ai_proposal_ready": + execute_state = "awaiting_approval" + else: + execute_state = "not_started" + + if verification != "missing": + verify_state = verification + elif auto_repair_records > 0 or operation_records > 0: + verify_state = "pending_or_missing" + else: + verify_state = "not_started" + + if verdict == "auto_repaired_verified": + conclusion = "已驗證自動修復" + elif auto_repair_records > 0 or operation_records > 0: + conclusion = "已記錄執行，等待或缺少驗證" + elif is_noop: + conclusion = "未自動修復，需人工判斷" + elif "approval" in verdict: + conclusion = "等待審批後才會執行" + elif "manual" in verdict: + conclusion = "轉人工處理" + else: + conclusion = "尚未形成可宣稱自動修復的證據鏈" + + return ( + "🧭 流程進度\n" + f"├ 收件：received | 診斷：{html.escape(diagnose_state)}\n" + f"├ 匹配：{html.escape(str(match_state)[:60])} | " + f"執行：{html.escape(execute_state)}\n" + f"├ 驗證：{html.escape(verify_state)} | " + f"KM：{km_entries} | MCP：{gateway_total}\n" + f"└ 判定：{html.escape(verdict)} — {html.escape(conclusion)}\n" + ) + def format(self) -> str: """ 格式化為 SOUL.md 規範的訊息 (含 AI 仲裁 + SignOz) @@ -660,6 +747,7 @@ class TelegramMessage: playbook_line = "" if self.playbook_name: playbook_line = f"📖 Playbook：{html.escape(self.playbook_name)}\n" + flow_progress_block = self._format_flow_progress_block() automation_block = self._format_automation_block() # ADR-075 TYPE-3 格式組裝 @@ -671,6 +759,7 @@ class TelegramMessage: f"{category_line}" f"🧭 處置狀態：{safe_automation_summary}\n" f"\n" + f"{flow_progress_block}\n" f"{automation_block}" f"\n" f"🧠 AI 深度診斷\n" @@ -816,6 +905,7 @@ class TelegramMessage: playbook_line = "" if self.playbook_name: playbook_line = f"📖 {html.escape(self.playbook_name)}\n" + flow_progress_block = self._format_flow_progress_block() # 組裝訊息 message = ( @@ -823,6 +913,7 @@ class TelegramMessage: f"{safe_resource}\n" f"{category_line}" f"\n" + f"{flow_progress_block}\n" f"{self._format_automation_block()}\n" f"{conf_line}\n" f"👥 {resp_display}\n" @@ -2229,6 +2320,29 @@ class TelegramGateway: trace_url=signoz_trace_url, ) + automation_quality: dict | None = None + if incident_id: + try: + from src.services.awooop_truth_chain_service import fetch_truth_chain + + truth_chain = await asyncio.wait_for( + fetch_truth_chain( + source_id=incident_id, + project_id="awoooi", + ), + timeout=2.5, + ) + quality = truth_chain.get("automation_quality") + if isinstance(quality, dict): + automation_quality = quality + except Exception as truth_exc: + logger.debug( + "telegram_approval_card_truth_chain_fetch_failed", + approval_id=approval_id, + incident_id=incident_id, + error=str(truth_exc), + ) + # 建立訊息結構 (含 AI 仲裁 + SignOz + Token/Cost + 頻率統計) message = TelegramMessage( status_emoji=emoji, @@ -2266,6 +2380,7 @@ class TelegramGateway: alert_category=alert_category, playbook_name=playbook_name, automation_state=automation_state, + automation_quality=automation_quality, ) # 格式化訊息 — Phase 22: 如果 Nemotron 啟用，使用雙軌格式 diff --git a/apps/api/tests/test_telegram_ai_automation_block.py b/apps/api/tests/test_telegram_ai_automation_block.py index 5ac5b24cd..8cb9cd1b8 100644 --- a/apps/api/tests/test_telegram_ai_automation_block.py +++ b/apps/api/tests/test_telegram_ai_automation_block.py @@ -24,6 +24,8 @@ def test_action_required_card_exposes_ai_automation_on_fallback() -> None: assert "NemoTron" in body assert "Hermes" in body assert "ElephantAlpha" in body + assert "流程進度" in body + assert "執行：no_action_or_observe" in body def test_nemotron_card_exposes_same_ai_automation_chain() -> None: @@ -50,3 +52,41 @@ def test_nemotron_card_exposes_same_ai_automation_chain() -> None: assert "OpenClaw Nemo" in body assert "tool_ready" in body assert "restart_deployment" in body + assert "流程進度" in body + + +def test_action_required_card_exposes_truth_chain_progress() -> None: + message = TelegramMessage( + status_emoji="⚠️", + risk_level="LOW", + resource_name="awoooi-api", + root_cause="restart spike", + suggested_action="kubectl rollout restart deployment/awoooi-api", + estimated_downtime="30s", + approval_id="approval-id", + incident_id="INC-20260513-TEST03", + primary_responsibility="INFRA", + confidence=0.91, + playbook_name="restart_deployment", + automation_quality={ + "verdict": "auto_repaired_verified", + "facts": { + "auto_repair_execution_records": 1, + "automation_operation_records": 1, + "verification_result": "success", + "mcp_gateway_total": 5, + "knowledge_entries": 2, + }, + }, + ) + + body = message.format() + + assert "流程進度" in body + assert "執行：auto_repair_recorded:1" in body + assert "驗證：success" in body + assert "KM：2" in body + assert "MCP：5" in body + assert "已驗證自動修復" in body + assert "已驗證自動修復完成" in body + assert "等待人工批准" not in body