From 872d1aa5e407189e111a25b422ff23c1805ea4da Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 31 May 2026 15:02:12 +0800 Subject: [PATCH] fix(awooop): honor approval repair metadata --- .../services/awooop_truth_chain_service.py | 37 ++++++++++++--- .../tests/test_awooop_truth_chain_service.py | 45 +++++++++++++++++++ docs/LOGBOOK.md | 1 + ...-04-15-MASTER-ai-autonomous-flywheel-v2.md | 2 +- 4 files changed, 78 insertions(+), 7 deletions(-) diff --git a/apps/api/src/services/awooop_truth_chain_service.py b/apps/api/src/services/awooop_truth_chain_service.py index 6e4a41114..d92ec39ba 100644 --- a/apps/api/src/services/awooop_truth_chain_service.py +++ b/apps/api/src/services/awooop_truth_chain_service.py @@ -146,6 +146,19 @@ def _approval_has_no_action(approvals: list[dict[str, Any]]) -> bool: return any(_looks_like_no_action(row.get("action")) for row in approvals) +def _approval_suppresses_repair_execution(approvals: list[dict[str, Any]]) -> bool: + for row in approvals: + metadata = row.get("extra_metadata") + if not isinstance(metadata, dict): + continue + execution_kind = str(metadata.get("execution_kind") or "").lower() + if metadata.get("repair_executed") is False: + return True + if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}: + return True + return False + + def _is_no_action_operation(row: dict[str, Any]) -> bool: """Return true for durable audit rows that represent observation, not repair.""" if str(row.get("operation_type") or "") != "playbook_executed": @@ -209,7 +222,8 @@ def build_incident_reconciliation( attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows) succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows) repair_rows = auto_repair_executions or [] - effective_ops = _effective_execution_ops(automation_ops) + approval_suppressed = _approval_suppresses_repair_execution(approvals) + effective_ops = [] if approval_suppressed else _effective_execution_ops(automation_ops) executed_ops = [ row for row in effective_ops @@ -363,7 +377,8 @@ def _truth_status( if incident is not None: incident_status = str(incident.get("status") or "unknown") repair_rows = auto_repair_executions or [] - effective_ops = _effective_execution_ops(automation_ops) + approval_suppressed = _approval_suppresses_repair_execution(approvals) + effective_ops = [] if approval_suppressed else _effective_execution_ops(automation_ops) has_execution_records = bool(effective_ops or repair_rows) stage = "received" stage_status = incident_status.lower() @@ -488,16 +503,21 @@ def build_automation_quality( evidence_succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows) gateway_total = int(gateway_mcp_summary.get("total") or 0) legacy_total = int(legacy_mcp_summary.get("total") or 0) - effective_ops = _effective_execution_ops(automation_ops) + approval_statuses = {str(row.get("status") or "").upper() for row in approvals} + approval_actions = " ".join(str(row.get("action") or "") for row in approvals).upper() + approval_no_action = _approval_has_no_action(approvals) + approval_suppressed = _approval_suppresses_repair_execution(approvals) + effective_ops = ( + [] + if approval_suppressed + else _effective_execution_ops(automation_ops) + ) noop_ops = [row for row in automation_ops if _is_no_action_operation(row)] audit_only_ops = [row for row in automation_ops if _is_audit_only_operation(row)] automation_statuses = {str(row.get("status") or "").lower() for row in effective_ops} auto_repair_successes = {row.get("success") for row in auto_repair_executions} has_execution = bool(effective_ops or auto_repair_executions) verification_result = _latest_verification_result(incident, evidence_rows) - approval_statuses = {str(row.get("status") or "").upper() for row in approvals} - approval_actions = " ".join(str(row.get("action") or "") for row in approvals).upper() - approval_no_action = _approval_has_no_action(approvals) gate("source_persisted", "passed", str(incident.get("incident_id"))) gate("outbound_recorded", "passed" if outbound_rows else "missing", str(len(outbound_rows))) @@ -519,6 +539,8 @@ def build_automation_quality( if any(status in {"PENDING", "WAITING_APPROVAL"} for status in approval_statuses): gate("approval_state", "warning", "waiting_approval") + elif approval_statuses and approval_suppressed and not has_execution: + gate("approval_state", "warning", "approved_diagnostic_or_observe_only") elif approval_statuses and (approval_no_action or "NO_ACTION" in approval_actions) and not has_execution: gate("approval_state", "failed", "approved_no_action_without_execution") elif approvals: @@ -559,6 +581,8 @@ def build_automation_quality( verdict = "auto_repaired_verification_degraded" elif has_execution: verdict = "execution_unverified" + elif approval_suppressed: + verdict = "manual_required_diagnostic_only" elif approval_statuses and (approval_no_action or "NO_ACTION" in approval_actions): verdict = "manual_required_no_action" elif any(status in {"PENDING", "WAITING_APPROVAL"} for status in approval_statuses): @@ -608,6 +632,7 @@ def build_automation_quality( "effective_execution_records": len(effective_ops), "noop_operation_records": len(noop_ops), "audit_only_operation_records": len(audit_only_ops), + "approval_repair_suppressed": approval_suppressed, "auto_repair_execution_records": len(auto_repair_executions), "verification_result": verification_result, "knowledge_entries": len(km_entries), diff --git a/apps/api/tests/test_awooop_truth_chain_service.py b/apps/api/tests/test_awooop_truth_chain_service.py index a4079cdaf..04d0e487d 100644 --- a/apps/api/tests/test_awooop_truth_chain_service.py +++ b/apps/api/tests/test_awooop_truth_chain_service.py @@ -544,6 +544,51 @@ def test_automation_quality_ignores_no_action_audit_rows_as_execution() -> None: assert gates["verification_recorded"] == "not_applicable" +def test_automation_quality_uses_approval_metadata_to_suppress_diagnostic_ops() -> None: + quality = build_automation_quality( + incident={"incident_id": "INC-DIAG", "status": "RESOLVED"}, + approvals=[ + { + "status": "EXECUTION_SUCCESS", + "action": "ssh 192.168.0.110 'df -h /data/minio'", + "extra_metadata": { + "execution_kind": "diagnostic", + "repair_executed": False, + }, + } + ], + evidence_rows=[ + { + "sensors_attempted": 8, + "sensors_succeeded": 6, + "verification_result": "degraded", + } + ], + automation_ops=[ + { + "operation_type": "playbook_executed", + "status": "success", + "actor": "approval_execution", + "output_action": "ssh 192.168.0.110 'df -h /data/minio'", + } + ], + auto_repair_executions=[], + gateway_mcp_summary={"total": 8}, + legacy_mcp_summary={"total": 8}, + outbound_rows=[{"message_id": "m1"}], + km_entries=[{"id": "km-1"}], + timeline_events=[{"id": "tl-1"}], + ) + + gates = {row["name"]: row["status"] for row in quality["gates"]} + assert quality["verdict"] == "manual_required_diagnostic_only" + assert quality["facts"]["automation_operation_records"] == 1 + assert quality["facts"]["effective_execution_records"] == 0 + assert quality["facts"]["approval_repair_suppressed"] is True + assert gates["execution_recorded"] == "missing" + assert gates["verification_recorded"] == "not_applicable" + + def test_automation_quality_marks_verified_auto_repair() -> None: quality = build_automation_quality( incident={ diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 0e1a36f02..84f5d9f9b 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -125,6 +125,7 @@ DB / worker evidence: - `incident_timeline_service` 讀取 `approval_records.extra_metadata.execution_kind / repair_executed`；`execution_success + repair_executed=false` 改成 `info`，標題為 observation recorded，不再顯示 repair success。 - Telegram 首屏 `_automation_status_summary()` 與 AwoooP status-chain / callback snapshot 改以 `auto_repair_execution_records` 或 `effective_execution_records` 判斷「真的有 repair execution」。 - `/api/v1/platform/status-chain` 的前台共用 API 同步改用 `effective_execution_records`；只有 raw operation log 時回 `repair_state=diagnostic_or_audit_recorded`，不再回 `executed`。 +- `awooop_truth_chain_service` 將 approval metadata `repair_executed=false` / `execution_kind=diagnostic` 納入 effective execution 計算，避免 SSH 診斷型 `playbook_executed` 被算成 repair execution。 - 若只有 `automation_operation_records` 但 `effective_execution_records=0`，顯示 `diagnostic_recorded` / `diagnostic_or_audit_recorded`，文案改為「已記錄診斷/觀察，尚未證明修復」。 - `build_incident_reconciliation()` 也改用 `_effective_execution_ops()` 計算 executed ops，避免 NO_ACTION / audit-only row 觸發 `incident_open_after_successful_execution`。 - Production backfill 兩筆舊 incident： diff --git a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md index 35dc277fa..3c829e6f8 100644 --- a/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md +++ b/docs/superpowers/specs/2026-04-15-MASTER-ai-autonomous-flywheel-v2.md @@ -2679,7 +2679,7 @@ Phase 6 完成後 **T154b Telegram display truth + historical backfill（2026-05-31 台北）**： - 觸發：T154 補了新流量 execution metadata，但前台/Telegram 顯示層仍以 raw `automation_operation_records > 0` 顯示「已自動執行」，會把 diagnostic / audit-only operation 誤認成 repair。`incident_timeline_service` 也會把 `execution_success` 一律畫成 executor success，舊 incident 即使補 metadata 仍會誤導 operator。 -- 修正：`incident_timeline_service` 讀 `approval_records.extra_metadata.execution_kind / repair_executed`；`execution_success + repair_executed=false` 改成 info / observation recorded。Telegram 首屏、AwoooP status-chain、callback snapshot、`/api/v1/platform/status-chain` 共用前台 API 改用 `auto_repair_execution_records` 或 `effective_execution_records` 判斷 repair execution；只有 raw operation records 時顯示 `diagnostic_recorded` / `diagnostic_or_audit_recorded`，文案為「已記錄診斷/觀察，尚未證明修復」。`build_incident_reconciliation()` 改用 `_effective_execution_ops()`，NO_ACTION / audit-only 不再觸發 successful execution mismatch。 +- 修正：`incident_timeline_service` 讀 `approval_records.extra_metadata.execution_kind / repair_executed`；`execution_success + repair_executed=false` 改成 info / observation recorded。Telegram 首屏、AwoooP status-chain、callback snapshot、`/api/v1/platform/status-chain` 共用前台 API 改用 `auto_repair_execution_records` 或 `effective_execution_records` 判斷 repair execution；truth-chain 也將 approval metadata `repair_executed=false` / `execution_kind=diagnostic` 納入 effective execution 計算，避免 SSH 診斷型 `playbook_executed` 被算成 repair。只有 raw operation records 時顯示 `diagnostic_recorded` / `diagnostic_or_audit_recorded`，文案為「已記錄診斷/觀察，尚未證明修復」。`build_incident_reconciliation()` 改用 `_effective_execution_ops()`，NO_ACTION / audit-only 不再觸發 successful execution mismatch。 - Production backfill：`INC-20260530-88D960` approval 標記 `execution_kind=diagnostic, repair_executed=false`；`INC-20260531-88394F` approval 標記 `execution_kind=no_action, repair_executed=false`。兩筆均新增 `alert_operation_log.EXECUTION_COMPLETED`、postmortem `knowledge_entries(entry_type=POSTMORTEM,path_type=postmortem)`、`KM_CONVERTED`，`truth_backfill_id=telegram_execution_truth_backfill_20260531_t154b`。未重跑任何修復動作。 - Verification：`py_compile` pass；targeted `ruff --select E9,F401,F821,F841` pass；`test_incident_timeline_service.py` + `test_awooop_truth_chain_service.py` + `test_telegram_ai_automation_block.py` + `test_telegram_message_templates.py` -> 101 passed；`test_awooop_operator_timeline_labels.py` covers status-chain diagnostic/audit-only repair_state。Production DB readback confirms both approvals have `repair_executed=false` plus execution/postmortem/KM backfill rows. - 判讀：可宣稱 repair 的條件是 `effective_execution_records > 0` 或 `auto_repair_execution_records > 0`，不是 operation log 數量。舊資料修正後，operator 看到「已記錄診斷/觀察」時不能再視為自動修復完成。