From 3bf84fdc58a9f5b0dcc917198ff593aeb1108a03 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 09:45:52 +0800 Subject: [PATCH] fix(runs): prioritize failed terminal receipts --- .../src/services/platform_operator_service.py | 204 ++++++++++++++++++ .../test_awooop_operator_timeline_labels.py | 124 +++++++++++ .../src/components/awooop/status-chain.tsx | 29 +++ 3 files changed, 357 insertions(+) diff --git a/apps/api/src/services/platform_operator_service.py b/apps/api/src/services/platform_operator_service.py index e891da881..9f88ac96b 100644 --- a/apps/api/src/services/platform_operator_service.py +++ b/apps/api/src/services/platform_operator_service.py @@ -7434,6 +7434,206 @@ def _build_awooop_status_chain( } +def _run_error_envelope(value: Any) -> dict[str, Any]: + if isinstance(value, dict): + return value + if not value: + return {} + try: + parsed = json.loads(str(value)) + except (TypeError, ValueError, json.JSONDecodeError): + return {} + return parsed if isinstance(parsed, dict) else {} + + +def _terminal_timestamp(value: Any) -> str | None: + if isinstance(value, datetime): + return value.isoformat() + if value is None: + return None + rendered = str(value).strip() + return rendered or None + + +def _apply_run_terminal_truth_precedence( + *, + run: AwoooPRunState, + status_chain: dict[str, Any] | None, +) -> dict[str, Any]: + """Keep a failed run receipt from being painted green by incident history.""" + + chain = dict(status_chain or {}) + if ( + str(getattr(run, "state", "")) != "failed" + or str(getattr(run, "error_code", "")) + != "E-AGENT99-OUTCOME-TIMEOUT" + ): + return chain + + envelope = _run_error_envelope(getattr(run, "error_detail", None)) + if envelope.get("outcome_timeout_no_write_terminal") is not True: + return chain + + verifier = ( + envelope.get("verifier") + if isinstance(envelope.get("verifier"), dict) + else {} + ) + terminal_receipt = ( + verifier.get("receipt") + if isinstance(verifier.get("receipt"), dict) + else {} + ) + safe_next_action = str( + envelope.get("safe_next_action") + or "await_new_source_recurrence_after_agent99_relay_recovery" + ) + historical_incident_context = { + "source": chain.get("source"), + "source_id": chain.get("source_id"), + "incident_ids": chain.get("incident_ids") or [], + "current_stage": chain.get("current_stage"), + "stage_status": chain.get("stage_status"), + "verdict": chain.get("verdict"), + "repair_state": chain.get("repair_state"), + "verification": chain.get("verification"), + "operator_outcome": chain.get("operator_outcome"), + "evidence": chain.get("evidence"), + "writes": chain.get("writes"), + } + run_terminal_precedence = { + "active": True, + "reason": "higher_priority_run_terminal_receipt", + "run_id": str(getattr(run, "run_id", "")), + "run_state": "failed", + "error_code": "E-AGENT99-OUTCOME-TIMEOUT", + "outcome_timeout_no_write_terminal": True, + "timeout_at": ( + terminal_receipt.get("timeout_at") + or _terminal_timestamp(getattr(run, "timeout_at", None)) + ), + "terminal_at": ( + terminal_receipt.get("terminal_at") + or _terminal_timestamp(getattr(run, "completed_at", None)) + ), + "terminalizer_runtime_write_performed": bool( + envelope.get("terminalizer_runtime_write_performed") is True + ), + "outcome_runtime_write_status": str( + envelope.get("outcome_runtime_write_status") + or "unknown_without_authenticated_outcome" + ), + "transport_replayed": bool(envelope.get("transport_replayed") is True), + "incident_resolution_authorized": bool( + envelope.get("incident_resolution_authorized") is True + ), + "runtime_closure_verified": bool( + envelope.get("runtime_closure_verified") is True + ), + "safe_next_action": safe_next_action, + } + + return { + **chain, + "source": "run_terminal_receipt>historical_incident_context", + "current_stage": "outcome_verifier_failed", + "stage_status": "failed", + "verdict": "no_runtime_closure_authenticated_outcome_timeout", + "repair_state": "failed_no_write_terminal", + "verification": "failed_authenticated_outcome_timeout", + "needs_human": False, + "next_step": safe_next_action, + "operator_outcome": { + "schema_version": "operator_outcome_run_terminal_v1", + "state": "failed_no_write_terminal", + "severity": "degraded", + "summary_zh": ( + "Agent99 驗證結果逾時；本 Run 已失敗收斂，" + "未宣稱 runtime 修復或事件結案。" + ), + "needs_human": False, + "human_action_required": False, + "human_action_reason": None, + "next_action": safe_next_action, + "execution_result": { + "approval_status": "not_applicable", + "completion_status": "failed_no_write_terminal", + "command_status": "not_replayed", + "repair_status": "not_verified", + "failure_status": "authenticated_outcome_timeout", + "terminal": True, + "summary_zh": ( + "未取得 authenticated outcome；terminalizer 未執行" + " target runtime write、未重送 transport、未關閉 incident。" + ), + }, + "notification": { + "mode": "no_notification", + "channels": [], + "telegram": "not_sent", + "awooop": "readback_only", + }, + }, + "automation_handoff": None, + "blockers": ["authenticated_outcome_timeout_no_write"], + "evidence": { + "auto_repair_records": 0, + "operation_records": 0, + "mcp_gateway_total": 0, + "knowledge_entries": 0, + "remediation_total": 0, + "remediation_state": "run_terminal_receipt", + "latest_route": "--", + "latest_mode": None, + "latest_at": run_terminal_precedence["terminal_at"], + "latest_preview": "authenticated_outcome_timeout_no_write", + "ansible_dry_run_only": False, + }, + "writes": {"incident": False, "auto_repair": False}, + "mcp": { + "gateway": { + "total": 0, + "success": 0, + "failed": 0, + "blocked": 0, + "first_class_total": 0, + "legacy_bridge_total": 0, + "policy_enforced_total": 0, + "stage": "not_applicable_run_terminal", + "stage_status": "failed", + }, + "legacy": {"total": 0, "success": 0, "failed": 0}, + "top_tools": [], + }, + "execution": { + "operation_total": 0, + "latest_operation_type": "outcome_timeout_no_write_terminal", + "latest_status": "failed", + "latest_actor": "agent99_controlled_dispatch_reconciler", + "latest_action": "record_no_write_terminal", + "latest_executor": "none", + "playbook_ids": [], + "playbook_paths": [], + "ansible": { + "considered": False, + "record_total": 0, + "candidate_count": 0, + "not_used_reason": "run_terminal_no_runtime_write", + "check_mode_total": 0, + "apply_total": 0, + "rollback_total": 0, + "applied": False, + "controlled_apply": False, + }, + }, + "repair_candidate_promotion": ( + _empty_repair_candidate_promotion_projection() + ), + "run_terminal_precedence": run_terminal_precedence, + "historical_incident_context": historical_incident_context, + } + + def _empty_repair_candidate_promotion_projection() -> dict[str, Any]: return { "schema_version": "repair_candidate_promotion_projection_v1", @@ -8254,6 +8454,10 @@ async def get_run_detail( project_id=run.project_id, remediation_history=remediation_history, ) + awooop_status_chain = _apply_run_terminal_truth_precedence( + run=run, + status_chain=awooop_status_chain, + ) timeline: list[dict[str, Any]] = [ _timeline_item( diff --git a/apps/api/tests/test_awooop_operator_timeline_labels.py b/apps/api/tests/test_awooop_operator_timeline_labels.py index 33f6a1bc9..cf4bbab07 100644 --- a/apps/api/tests/test_awooop_operator_timeline_labels.py +++ b/apps/api/tests/test_awooop_operator_timeline_labels.py @@ -1,5 +1,6 @@ import asyncio import inspect +import json from datetime import datetime from decimal import Decimal from types import SimpleNamespace @@ -32,6 +33,7 @@ from src.services.platform_operator_service import ( _ai_route_lane_state, _ai_route_policy_order, _ai_route_repair_evidence_item, + _apply_run_terminal_truth_precedence, _build_awooop_status_chain, _callback_reply_audit_summary_from_row, _callback_reply_event_item, @@ -2940,6 +2942,128 @@ def test_awooop_status_chain_marks_verified_repair() -> None: assert chain["source_refs"]["refs"]["signoz_alerts"] == ["signoz:abc"] +def test_run_terminal_receipt_overrides_historical_incident_green_state() -> None: + historical_chain = { + "schema_version": "awooop_status_chain_v1", + "source": "truth_chain+adr100_history", + "source_id": "INC-BRR-35994B66C2807E74", + "incident_ids": ["INC-BRR-35994B66C2807E74"], + "current_stage": "execution_succeeded", + "stage_status": "success", + "verdict": "auto_repaired_verified", + "repair_state": "auto_repaired_verified", + "verification": "success", + "needs_human": False, + "next_step": "monitor_for_regression", + "operator_outcome": { + "state": "completed_verified", + "execution_result": { + "completion_status": "completed_verified", + "terminal": True, + }, + }, + "evidence": { + "auto_repair_records": 1, + "operation_records": 4, + "mcp_gateway_total": 8, + "knowledge_entries": 1, + }, + "writes": {"incident": True, "auto_repair": True}, + "mcp": {"gateway": {"total": 8, "success": 4}}, + "execution": {"operation_total": 4, "latest_status": "success"}, + } + run = SimpleNamespace( + run_id=UUID("f53d95ac-04d3-5228-ae74-9ab0ad95c013"), + state="failed", + error_code="E-AGENT99-OUTCOME-TIMEOUT", + timeout_at=datetime.fromisoformat("2026-07-15T00:22:32.527349"), + completed_at=datetime.fromisoformat("2026-07-15T01:23:15.118099"), + error_detail=json.dumps({ + "outcome_timeout_no_write_terminal": True, + "terminalizer_runtime_write_performed": False, + "outcome_runtime_write_status": ( + "unknown_without_authenticated_outcome" + ), + "transport_replayed": False, + "incident_resolution_authorized": False, + "runtime_closure_verified": False, + "safe_next_action": ( + "await_new_source_recurrence_after_agent99_relay_recovery" + ), + "verifier": { + "receipt": { + "timeout_at": "2026-07-15T00:22:32.527349", + "terminal_at": "2026-07-15T01:23:15.118099", + } + }, + }), + ) + + chain = _apply_run_terminal_truth_precedence( + run=run, + status_chain=historical_chain, + ) + + assert chain["source"] == ( + "run_terminal_receipt>historical_incident_context" + ) + assert chain["stage_status"] == "failed" + assert chain["repair_state"] == "failed_no_write_terminal" + assert chain["verification"] == "failed_authenticated_outcome_timeout" + assert chain["needs_human"] is False + assert chain["operator_outcome"]["state"] == "failed_no_write_terminal" + assert chain["operator_outcome"]["execution_result"][ + "completion_status" + ] == "failed_no_write_terminal" + assert chain["writes"] == {"incident": False, "auto_repair": False} + assert chain["evidence"]["auto_repair_records"] == 0 + assert chain["execution"]["operation_total"] == 0 + assert chain["run_terminal_precedence"] == { + "active": True, + "reason": "higher_priority_run_terminal_receipt", + "run_id": "f53d95ac-04d3-5228-ae74-9ab0ad95c013", + "run_state": "failed", + "error_code": "E-AGENT99-OUTCOME-TIMEOUT", + "outcome_timeout_no_write_terminal": True, + "timeout_at": "2026-07-15T00:22:32.527349", + "terminal_at": "2026-07-15T01:23:15.118099", + "terminalizer_runtime_write_performed": False, + "outcome_runtime_write_status": ( + "unknown_without_authenticated_outcome" + ), + "transport_replayed": False, + "incident_resolution_authorized": False, + "runtime_closure_verified": False, + "safe_next_action": ( + "await_new_source_recurrence_after_agent99_relay_recovery" + ), + } + assert chain["historical_incident_context"]["repair_state"] == ( + "auto_repaired_verified" + ) + assert chain["historical_incident_context"]["writes"] == { + "incident": True, + "auto_repair": True, + } + assert "_apply_run_terminal_truth_precedence" in inspect.getsource( + platform_operator_service.get_run_detail + ) + + +def test_run_terminal_precedence_leaves_unrelated_failed_run_unchanged() -> None: + chain = {"repair_state": "auto_repaired_verified"} + run = SimpleNamespace( + state="failed", + error_code="E-OTHER", + error_detail="{}", + ) + + assert _apply_run_terminal_truth_precedence( + run=run, + status_chain=chain, + ) == chain + + def test_awooop_status_chain_surfaces_controlled_ansible_apply_proof() -> None: chain = _build_awooop_status_chain( incident_ids=["INC-20260531-D6A3C4"], diff --git a/apps/web/src/components/awooop/status-chain.tsx b/apps/web/src/components/awooop/status-chain.tsx index 448ee8cba..4f358eae1 100644 --- a/apps/web/src/components/awooop/status-chain.tsx +++ b/apps/web/src/components/awooop/status-chain.tsx @@ -18,6 +18,22 @@ export interface AwoooPStatusChain { verification?: string | null; needs_human?: boolean | null; next_step?: string | null; + run_terminal_precedence?: { + active?: boolean | null; + reason?: string | null; + run_id?: string | null; + run_state?: string | null; + error_code?: string | null; + outcome_timeout_no_write_terminal?: boolean | null; + timeout_at?: string | null; + terminal_at?: string | null; + terminalizer_runtime_write_performed?: boolean | null; + outcome_runtime_write_status?: string | null; + transport_replayed?: boolean | null; + incident_resolution_authorized?: boolean | null; + runtime_closure_verified?: boolean | null; + safe_next_action?: string | null; + } | null; operator_outcome?: { schema_version?: string; state?: string | null; @@ -466,6 +482,13 @@ export interface AwoooPStatusChain { function toneClass(chain?: AwoooPStatusChain | null) { if (!chain) return "border-[#d8d3c7] bg-[#faf9f3] text-[#5f5b52]"; + if ( + chain.run_terminal_precedence?.active === true + || chain.stage_status === "failed" + || String(chain.repair_state ?? "").startsWith("failed_") + ) { + return "border-[#e2a29b] bg-[#fff0ef] text-[#9f2f25]"; + } if (chain.repair_state === "auto_repaired_verified") { return "border-[#9bc7a4] bg-[#f0faf2] text-[#17602a]"; } @@ -477,6 +500,11 @@ function toneClass(chain?: AwoooPStatusChain | null) { function chainIcon(chain?: AwoooPStatusChain | null) { if (!chain) return Route; + if ( + chain.run_terminal_precedence?.active === true + || chain.stage_status === "failed" + || String(chain.repair_state ?? "").startsWith("failed_") + ) return TriangleAlert; if (chain.repair_state === "auto_repaired_verified") return CheckCircle2; if (chain.needs_human || chain.repair_state === "blocked_manual_required") return ShieldCheck; return Activity; @@ -557,6 +585,7 @@ export function AwoooPStatusChainPanel({ approve_post_apply_verifier_plan: t("nextActions.reviewPostApplyVerifier"), assign_km_writeback_owner_after_verified_execution: t("nextActions.prepareKmPlaybookWriteback"), assign_playbook_trust_owner_after_verified_execution: t("nextActions.prepareKmPlaybookWriteback"), + await_new_source_recurrence_after_agent99_relay_recovery: t("nextActions.collectEvidenceOrWait"), }; return labels[key] ?? valueOrEmpty(value, emptyLabel); };