diff --git a/apps/api/src/services/awooop_truth_chain_service.py b/apps/api/src/services/awooop_truth_chain_service.py index 4151d6549..6e4a41114 100644 --- a/apps/api/src/services/awooop_truth_chain_service.py +++ b/apps/api/src/services/awooop_truth_chain_service.py @@ -209,9 +209,10 @@ def build_incident_reconciliation( attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows) succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows) repair_rows = auto_repair_executions or [] + effective_ops = _effective_execution_ops(automation_ops) executed_ops = [ row - for row in automation_ops + for row in effective_ops if str(row.get("status") or "").lower() in {"success", "completed", "executed"} ] diff --git a/apps/api/src/services/incident_timeline_service.py b/apps/api/src/services/incident_timeline_service.py index 299e1c797..7218a3dc8 100644 --- a/apps/api/src/services/incident_timeline_service.py +++ b/apps/api/src/services/incident_timeline_service.py @@ -27,6 +27,7 @@ from src.db.models import ( KnowledgeEntryRecord, TimelineEvent, ) +from src.services.approval_action_classifier import is_no_action_approval_action from src.services.awooop_truth_chain_service import build_incident_reconciliation logger = structlog.get_logger(__name__) @@ -555,9 +556,14 @@ async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None: data=approval.blast_radius or {}, )) + execution_truth = _approval_execution_truth(approval) + events.append(_event( stage="safe", - status=_approval_status_to_timeline_status(approval.status), + status=_approval_status_to_timeline_status( + approval.status, + repair_executed=execution_truth["repair_executed"], + ), title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}", timestamp=approval.created_at, description=_dry_run_summary(approval.dry_run_checks), @@ -570,22 +576,44 @@ async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None: "required_signatures": approval.required_signatures, "current_signatures": approval.current_signatures, "dry_run_checks": approval.dry_run_checks or [], + "execution_kind": execution_truth["execution_kind"], + "repair_executed": execution_truth["repair_executed"], }, )) if str(_value(approval.status)).startswith("execution_"): - success = _value(approval.status) == "execution_success" + success = ( + _value(approval.status) == "execution_success" + and execution_truth["repair_executed"] is not False + ) + no_repair_success = ( + _value(approval.status) == "execution_success" + and execution_truth["repair_executed"] is False + ) events.append(_event( stage="executor", - status="success" if success else "error", - title="Approval execution completed", + status="success" if success else ("info" if no_repair_success else "error"), + title=( + "Approval observation recorded" + if no_repair_success + else "Approval execution completed" + ), timestamp=approval.resolved_at or approval.updated_at, - description=approval.rejection_reason, + description=( + approval.rejection_reason + or ( + "Diagnostic or observation was recorded; no repair was executed." + if no_repair_success + else None + ) + ), actor="approval_execution", source_table="approval_records", data={ "approval_id": approval.id, "status": _value(approval.status), + "execution_kind": execution_truth["execution_kind"], + "repair_executed": execution_truth["repair_executed"], }, )) @@ -742,10 +770,32 @@ def _provider_from_description(description: str | None) -> str | None: return None -def _approval_status_to_timeline_status(status: Any) -> str: +def _approval_execution_truth(approval: Any) -> dict[str, Any]: + raw_metadata = getattr(approval, "extra_metadata", None) + metadata = raw_metadata if isinstance(raw_metadata, dict) else {} + execution_kind = str(metadata.get("execution_kind") or "").strip().lower() + repair_executed = metadata.get("repair_executed") + if repair_executed is None: + if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}: + repair_executed = False + elif is_no_action_approval_action(getattr(approval, "action", None)): + repair_executed = False + return { + "execution_kind": execution_kind or None, + "repair_executed": repair_executed, + } + + +def _approval_status_to_timeline_status( + status: Any, + *, + repair_executed: bool | None = None, +) -> str: value = str(_value(status)) if value in {"rejected", "expired"}: return "error" + if value in {"approved", "execution_success"} and repair_executed is False: + return "info" if value in {"approved", "execution_success"}: return "success" if value == "execution_failed": diff --git a/apps/api/src/services/telegram_gateway.py b/apps/api/src/services/telegram_gateway.py index b5516ab36..ce722ff67 100644 --- a/apps/api/src/services/telegram_gateway.py +++ b/apps/api/src/services/telegram_gateway.py @@ -306,6 +306,20 @@ def _safe_int(value: object) -> int: return 0 +def _has_repair_execution_evidence(facts: dict[str, object]) -> bool: + return ( + _safe_int(facts.get("auto_repair_execution_records")) > 0 + or _safe_int(facts.get("effective_execution_records")) > 0 + ) + + +def _has_nonrepair_operation_evidence(facts: dict[str, object]) -> bool: + return ( + _safe_int(facts.get("automation_operation_records")) > 0 + and not _has_repair_execution_evidence(facts) + ) + + def _bool_code(value: object, *, unknown_when_none: bool = False) -> str: if value is None and unknown_when_none: return "unknown" @@ -364,12 +378,18 @@ def _format_awooop_status_chain_lines( km_entries = _safe_int(facts.get("knowledge_entries")) needs_human = bool(truth_status.get("needs_human")) + has_repair_execution = _has_repair_execution_evidence(facts) + has_nonrepair_operation = _has_nonrepair_operation_evidence(facts) + if verdict == "auto_repaired_verified": repair_state = "auto_repaired_verified" next_step = "monitor_for_regression" - elif auto_repair_records > 0 or operation_records > 0: + elif has_repair_execution: repair_state = "executed_pending_verification" if verification == "missing" else "executed" next_step = "verify_execution_result" + elif has_nonrepair_operation: + repair_state = "diagnostic_or_audit_recorded" + next_step = "manual_review_or_collect_repair_evidence" elif remediation_state == "read_only": repair_state = "read_only_dry_run" next_step = "approve_or_escalate_from_awooop" @@ -1430,12 +1450,18 @@ def _callback_reply_awooop_status_chain_snapshot( km_entries = _safe_int(facts.get("knowledge_entries")) needs_human = bool(truth_status.get("needs_human")) + has_repair_execution = _has_repair_execution_evidence(facts) + has_nonrepair_operation = _has_nonrepair_operation_evidence(facts) + if verdict == "auto_repaired_verified": repair_state = "auto_repaired_verified" next_step = "monitor_for_regression" - elif auto_repair_records > 0 or operation_records > 0: + elif has_repair_execution: repair_state = "executed_pending_verification" if verification == "missing" else "executed" next_step = "verify_execution_result" + elif has_nonrepair_operation: + repair_state = "diagnostic_or_audit_recorded" + next_step = "manual_review_or_collect_repair_evidence" elif remediation_state == "read_only": repair_state = "read_only_dry_run" next_step = "approve_or_escalate_from_awooop" @@ -1758,17 +1784,21 @@ class TelegramMessage: quality = self.automation_quality or {} facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {} verdict = str(quality.get("verdict") or "") - auto_repair_records = int(facts.get("auto_repair_execution_records") or 0) - operation_records = int(facts.get("automation_operation_records") or 0) + has_repair_execution = _has_repair_execution_evidence(facts) + has_nonrepair_operation = _has_nonrepair_operation_evidence(facts) verification = str(facts.get("verification_result") or "missing") remediation_state = _remediation_evidence_state(self.remediation_summary) if verdict == "auto_repaired_verified": return "✅ 已驗證自動修復完成" - if auto_repair_records > 0 or operation_records > 0: + if has_repair_execution: if verification == "missing": return "🔄 已自動執行，等待驗證證據" return f"🔄 已自動執行，驗證結果：{verification}" + if has_nonrepair_operation: + if verification == "missing": + return "🔎 已記錄診斷/觀察，尚未證明修復" + return f"🔎 已記錄診斷/觀察，驗證結果：{verification}" if remediation_state == "read_only": return "🔎 AI 已完成只讀補救試跑，等待人工審批" if remediation_state == "write_observed": @@ -1844,6 +1874,8 @@ class TelegramMessage: ) auto_repair_records = int(facts.get("auto_repair_execution_records") or 0) operation_records = int(facts.get("automation_operation_records") or 0) + has_repair_execution = _has_repair_execution_evidence(facts) + has_nonrepair_operation = _has_nonrepair_operation_evidence(facts) verification = str(facts.get("verification_result") or "missing") gateway_total = int(facts.get("mcp_gateway_total") or 0) km_entries = int(facts.get("knowledge_entries") or 0) @@ -1858,8 +1890,10 @@ class TelegramMessage: match_state = self.playbook_name or "rule_catalog" if auto_repair_records > 0: execute_state = f"auto_repair_recorded:{auto_repair_records}" - elif operation_records > 0: + elif has_repair_execution: execute_state = f"operation_recorded:{operation_records}" + elif has_nonrepair_operation: + execute_state = f"diagnostic_recorded:{operation_records}" elif is_noop: execute_state = "no_action_or_observe" elif "approval" in verdict or self._automation_mode() == "ai_proposal_ready": @@ -1869,15 +1903,17 @@ class TelegramMessage: if verification != "missing": verify_state = verification - elif auto_repair_records > 0 or operation_records > 0: + elif has_repair_execution: verify_state = "pending_or_missing" else: verify_state = "not_started" if verdict == "auto_repaired_verified": conclusion = "已驗證自動修復" - elif auto_repair_records > 0 or operation_records > 0: + elif has_repair_execution: conclusion = "已記錄執行，等待或缺少驗證" + elif has_nonrepair_operation: + conclusion = "已記錄診斷/觀察，尚未證明修復" elif is_noop: conclusion = "未自動修復，需人工判斷" elif "approval" in verdict: diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index 2c4c05f16..f5599a510 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -474,6 +474,39 @@ def test_automation_quality_marks_no_action_without_execution() -> None: assert "execution_recorded" in quality["blockers"] +def test_reconciliation_ignores_no_action_audit_rows_as_execution() -> None: + reconciliation = build_incident_reconciliation( + incident={"incident_id": "INC-NOOP", "status": "INVESTIGATING"}, + approvals=[ + { + "id": "approval-noop", + "status": "APPROVED", + "action": "未知操作 | NO_ACTION", + "resolved_at": "2026-05-31T01:00:00+00:00", + } + ], + evidence_rows=[{"sensors_attempted": 8, "sensors_succeeded": 6}], + automation_ops=[ + { + "operation_type": "playbook_executed", + "status": "success", + "actor": "approval_execution", + "output_reason": "NO_ACTION", + "output_action": "未知操作 | NO_ACTION", + } + ], + auto_repair_executions=[], + timeline_events=[{"event_type": "executor", "status": "success"}], + ) + + codes = {row["code"] for row in reconciliation["mismatches"]} + assert reconciliation["facts"]["executed_operation_records"] == 0 + assert reconciliation["facts"]["effective_execution_records"] == 0 + assert "approval_approved_without_execution_record" in codes + assert "approval_no_action_without_execution" in codes + assert "incident_open_after_successful_execution" not in codes + + def test_automation_quality_ignores_no_action_audit_rows_as_execution() -> None: quality = build_automation_quality( incident={"incident_id": "INC-1", "status": "RESOLVED"}, diff --git a/apps/api/tests/test_incident_timeline_service.py b/apps/api/tests/test_incident_timeline_service.py index 7d68bc143..292ba3d82 100644 --- a/apps/api/tests/test_incident_timeline_service.py +++ b/apps/api/tests/test_incident_timeline_service.py @@ -1,5 +1,7 @@ from src.services.incident_timeline_service import ( STAGE_DEFS, + _approval_execution_truth, + _approval_status_to_timeline_status, _reconciliation_event, format_ascii_timeline, ) @@ -54,3 +56,33 @@ def test_reconciliation_event_omits_consistent_state() -> None: "consistency_status": "consistent", "mismatches": [], }) is None + + +def test_approval_execution_success_without_repair_is_timeline_info() -> None: + approval = type("Approval", (), { + "action": "OBSERVE", + "extra_metadata": {"execution_kind": "no_action", "repair_executed": False}, + })() + + truth = _approval_execution_truth(approval) + + assert truth == {"execution_kind": "no_action", "repair_executed": False} + assert _approval_status_to_timeline_status( + "execution_success", + repair_executed=truth["repair_executed"], + ) == "info" + + +def test_approval_execution_success_with_repair_stays_success() -> None: + approval = type("Approval", (), { + "action": "kubectl rollout restart deployment/awoooi-api", + "extra_metadata": {"execution_kind": "kubectl", "repair_executed": True}, + })() + + truth = _approval_execution_truth(approval) + + assert truth == {"execution_kind": "kubectl", "repair_executed": True} + assert _approval_status_to_timeline_status( + "execution_success", + repair_executed=truth["repair_executed"], + ) == "success" diff --git a/apps/api/tests/test_telegram_ai_automation_block.py b/apps/api/tests/test_telegram_ai_automation_block.py index 8cb9cd1b8..30f5296da 100644 --- a/apps/api/tests/test_telegram_ai_automation_block.py +++ b/apps/api/tests/test_telegram_ai_automation_block.py @@ -90,3 +90,37 @@ def test_action_required_card_exposes_truth_chain_progress() -> None: assert "已驗證自動修復" in body assert "已驗證自動修復完成" in body assert "等待人工批准" not in body + + +def test_action_required_card_does_not_call_diagnostic_ops_auto_repair() -> None: + message = TelegramMessage( + status_emoji="🚨", + risk_level="CRITICAL", + resource_name="dirty-reboot-evidence", + root_cause="Expert System: 偵測到高錯誤率", + suggested_action="ssh 192.168.0.110 'df -h /data/minio'", + estimated_downtime="0 min", + approval_id="approval-id", + incident_id="INC-20260530-88D960", + primary_responsibility="INFRA", + confidence=0.0, + playbook_name="rule_catalog", + automation_quality={ + "verdict": "auto_repaired_verification_degraded", + "facts": { + "auto_repair_execution_records": 0, + "automation_operation_records": 1, + "effective_execution_records": 0, + "verification_result": "degraded", + "mcp_gateway_total": 22, + "knowledge_entries": 2, + }, + }, + ) + + body = message.format() + + assert "已記錄診斷/觀察，驗證結果：degraded" in body + assert "執行：diagnostic_recorded:1" in body + assert "已記錄診斷/觀察，尚未證明修復" in body + assert "已自動執行" not in body diff --git a/apps/api/tests/test_telegram_message_templates.py b/apps/api/tests/test_telegram_message_templates.py index 423f49423..388ae0159 100644 --- a/apps/api/tests/test_telegram_message_templates.py +++ b/apps/api/tests/test_telegram_message_templates.py @@ -177,6 +177,37 @@ def test_awooop_status_chain_lines_show_read_only_manual_gate() -> None: assert "pending_human_approval" in joined +def test_awooop_status_chain_lines_do_not_treat_audit_ops_as_repair() -> None: + lines = telegram_gateway_module._format_awooop_status_chain_lines( + truth_chain={ + "truth_status": { + "current_stage": "manual_required", + "stage_status": "blocked", + "needs_human": True, + "blockers": ["approval_resolved_no_action_without_execution"], + }, + "automation_quality": { + "verdict": "auto_repaired_verification_degraded", + "facts": { + "auto_repair_execution_records": 0, + "automation_operation_records": 1, + "effective_execution_records": 0, + "verification_result": "degraded", + "mcp_gateway_total": 2, + "knowledge_entries": 1, + }, + "blockers": ["verification_recorded"], + }, + }, + remediation_history={"total": 0}, + ) + + joined = "\n".join(lines) + assert "diagnostic_or_audit_recorded" in joined + assert "manual_review_or_collect_repair_evidence" in joined + assert "executed_pending_verification" not in joined + + def test_awooop_agent_evidence_lines_show_mcp_source_execution_playbook_km() -> None: """Telegram 詳情/歷史要像前端一樣顯示五段 AI Agent 證據鏈。""" lines = telegram_gateway_module._format_awooop_agent_evidence_lines( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 5b73fae3d..a909f4f46 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,42 @@ +## 2026-05-31｜Telegram 告警前台真相顯示與舊資料補正 + +**背景**： + +- T154 已修新 approval 的執行語意，但舊資料中 `execution_success` 仍可能代表 `OBSERVE` 或 SSH 診斷，不是 repair。 +- Telegram 首屏與 AwoooP 詳情原本只看到 `automation_operation_records > 0` 就顯示「已自動執行 / executed」，會把 diagnostic / audit-only operation 誤讀成 repair execution。 +- `incident_timeline_service` 也會把 `execution_success` 畫成 executor success，舊 incident 即使 backfill 了 `repair_executed=false`，前台仍可能誤導值班者。 + +**本次調整**： + +- `incident_timeline_service` 讀取 `approval_records.extra_metadata.execution_kind / repair_executed`；`execution_success + repair_executed=false` 改成 `info`，標題為 observation recorded，不再顯示 repair success。 +- Telegram 首屏 `_automation_status_summary()` 與 AwoooP status-chain / callback snapshot 改以 `auto_repair_execution_records` 或 `effective_execution_records` 判斷「真的有 repair execution」。 +- 若只有 `automation_operation_records` 但 `effective_execution_records=0`，顯示 `diagnostic_recorded` / `diagnostic_or_audit_recorded`，文案改為「已記錄診斷/觀察，尚未證明修復」。 +- `build_incident_reconciliation()` 也改用 `_effective_execution_ops()` 計算 executed ops，避免 NO_ACTION / audit-only row 觸發 `incident_open_after_successful_execution`。 +- Production backfill 兩筆舊 incident： + - `INC-20260530-88D960`：approval 標記 `execution_kind=diagnostic`、`repair_executed=false`。 + - `INC-20260531-88394F`：approval 標記 `execution_kind=no_action`、`repair_executed=false`。 + - 兩筆都新增 `alert_operation_log.EXECUTION_COMPLETED`、postmortem `knowledge_entries(entry_type=POSTMORTEM,path_type=postmortem)`、`KM_CONVERTED`，`truth_backfill_id=telegram_execution_truth_backfill_20260531_t154b`。 + +**Verification**： + +```text +python3 -m py_compile incident_timeline_service.py telegram_gateway.py awooop_truth_chain_service.py related tests + -> pass +ruff check --select E9,F401,F821,F841 modified services/tests + -> pass +pytest test_incident_timeline_service.py test_awooop_truth_chain_service.py test_telegram_ai_automation_block.py test_telegram_message_templates.py -q + -> 101 passed +production DB readback: + INC-20260530-88D960 approval extra_metadata.execution_kind=diagnostic repair_executed=false + INC-20260531-88394F approval extra_metadata.execution_kind=no_action repair_executed=false + both incidents have EXECUTION_COMPLETED + POSTMORTEM + KM_CONVERTED backfill records +``` + +**判讀 / 下一步**： + +- 本輪沒有重跑任何修復動作；只把舊資料的 truth metadata、稽核與 KM 補齊。 +- 前台/Telegram 後續應以 `effective_execution_records` / `repair_executed` 判斷是否可宣稱 repair，不能用 raw operation log 數量代替。 + ## 2026-05-31｜Telegram 告警執行語意與 DB 稽核完整性修復 **背景**： diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index d5d1397ec..2a06dfc3b 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -2677,6 +2677,13 @@ Phase 6 完成後 - Verification：`py_compile` pass；`test_approval_execution_no_action.py` + `test_telegram_webhook_execution_handoff.py` 6 passed；`test_alert_rule_engine_validation.py` + `test_report_generation_service.py` 67 passed；`test_heartbeat_ollama_endpoints.py` + `test_heartbeat_pod_state_machine.py` + `test_gap_a4_placeholder_resolution.py` 49 passed；`test_decision_manager_bare_metal_kubectl_guard.py` + `test_alert_rule_engine_validation.py` + `test_gap_a4_placeholder_resolution.py` 75 passed。 - 判讀：T154 修的是「Telegram / DB / 前台統計的 truthfulness」，不是補跑舊 incident 的修復。舊資料中 status 已是 `execution_success` 的 OBSERVE 仍需靠新 metadata 才能精確分辨；部署後新 approval 會留下 immutable execution start/end 與 no-action 語意，operator 不應再把 OBSERVE 視為完成修復。 +**T154b Telegram display truth + historical backfill（2026-05-31 台北）**： +- 觸發：T154 補了新流量 execution metadata，但前台/Telegram 顯示層仍以 raw `automation_operation_records > 0` 顯示「已自動執行」，會把 diagnostic / audit-only operation 誤認成 repair。`incident_timeline_service` 也會把 `execution_success` 一律畫成 executor success，舊 incident 即使補 metadata 仍會誤導 operator。 +- 修正：`incident_timeline_service` 讀 `approval_records.extra_metadata.execution_kind / repair_executed`；`execution_success + repair_executed=false` 改成 info / observation recorded。Telegram 首屏、AwoooP status-chain、callback snapshot 改用 `auto_repair_execution_records` 或 `effective_execution_records` 判斷 repair execution；只有 raw operation records 時顯示 `diagnostic_recorded` / `diagnostic_or_audit_recorded`，文案為「已記錄診斷/觀察，尚未證明修復」。`build_incident_reconciliation()` 改用 `_effective_execution_ops()`，NO_ACTION / audit-only 不再觸發 successful execution mismatch。 +- Production backfill：`INC-20260530-88D960` approval 標記 `execution_kind=diagnostic, repair_executed=false`；`INC-20260531-88394F` approval 標記 `execution_kind=no_action, repair_executed=false`。兩筆均新增 `alert_operation_log.EXECUTION_COMPLETED`、postmortem `knowledge_entries(entry_type=POSTMORTEM,path_type=postmortem)`、`KM_CONVERTED`，`truth_backfill_id=telegram_execution_truth_backfill_20260531_t154b`。未重跑任何修復動作。 +- Verification：`py_compile` pass；targeted `ruff --select E9,F401,F821,F841` pass；`test_incident_timeline_service.py` + `test_awooop_truth_chain_service.py` + `test_telegram_ai_automation_block.py` + `test_telegram_message_templates.py` -> 101 passed。Production DB readback confirms both approvals have `repair_executed=false` plus execution/postmortem/KM backfill rows. +- 判讀：可宣稱 repair 的條件是 `effective_execution_records > 0` 或 `auto_repair_execution_records > 0`，不是 operation log 數量。舊資料修正後，operator 看到「已記錄診斷/觀察」時不能再視為自動修復完成。 + **T152 Ansible runtime readiness surfaced（2026-05-24 台北）**： - 觸發：T151 已讓首頁看到 execution backend / Ansible attribution，但 operator 仍看不到 runtime 端缺什麼，容易把「Ansible 有候選」誤解成「Ansible 已能自動修復」。 - 修正：API image 複製 `infra/ansible/` 作 read-only catalog；`truth-chain/quality/summary` 新增 `ansible_runtime`，回報 playbook binary、catalog、inventory、playbook_count、can_run_check_mode、blockers。首頁 execution evidence 同步顯示 runtime 狀態；目前 production 顯示 `runtime 未就緒：ansible_playbook_binary_missing`。未安裝 `ansible-core`、未啟用 check-mode / apply。