fix(alerts): distinguish diagnostic ops from repair
Some checks failed
CD Pipeline / tests (push) Successful in 1m22s
Code Review / ai-code-review (push) Successful in 12s
CD Pipeline / build-and-deploy (push) Successful in 4m14s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
Your Name
2026-05-31 14:31:07 +08:00
parent 273071b654
commit 8c40621d42
9 changed files with 278 additions and 15 deletions

View File

@@ -209,9 +209,10 @@ def build_incident_reconciliation(
attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows)
succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows)
repair_rows = auto_repair_executions or []
effective_ops = _effective_execution_ops(automation_ops)
executed_ops = [
row
for row in automation_ops
for row in effective_ops
if str(row.get("status") or "").lower()
in {"success", "completed", "executed"}
]

View File

@@ -27,6 +27,7 @@ from src.db.models import (
KnowledgeEntryRecord,
TimelineEvent,
)
from src.services.approval_action_classifier import is_no_action_approval_action
from src.services.awooop_truth_chain_service import build_incident_reconciliation
logger = structlog.get_logger(__name__)
@@ -555,9 +556,14 @@ async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None:
data=approval.blast_radius or {},
))
execution_truth = _approval_execution_truth(approval)
events.append(_event(
stage="safe",
status=_approval_status_to_timeline_status(approval.status),
status=_approval_status_to_timeline_status(
approval.status,
repair_executed=execution_truth["repair_executed"],
),
title=f"Safety gate: {_value(approval.risk_level)} / {_value(approval.status)}",
timestamp=approval.created_at,
description=_dry_run_summary(approval.dry_run_checks),
@@ -570,22 +576,44 @@ async def fetch_incident_timeline(incident_id: str) -> dict[str, Any] | None:
"required_signatures": approval.required_signatures,
"current_signatures": approval.current_signatures,
"dry_run_checks": approval.dry_run_checks or [],
"execution_kind": execution_truth["execution_kind"],
"repair_executed": execution_truth["repair_executed"],
},
))
if str(_value(approval.status)).startswith("execution_"):
success = _value(approval.status) == "execution_success"
success = (
_value(approval.status) == "execution_success"
and execution_truth["repair_executed"] is not False
)
no_repair_success = (
_value(approval.status) == "execution_success"
and execution_truth["repair_executed"] is False
)
events.append(_event(
stage="executor",
status="success" if success else "error",
title="Approval execution completed",
status="success" if success else ("info" if no_repair_success else "error"),
title=(
"Approval observation recorded"
if no_repair_success
else "Approval execution completed"
),
timestamp=approval.resolved_at or approval.updated_at,
description=approval.rejection_reason,
description=(
approval.rejection_reason
or (
"Diagnostic or observation was recorded; no repair was executed."
if no_repair_success
else None
)
),
actor="approval_execution",
source_table="approval_records",
data={
"approval_id": approval.id,
"status": _value(approval.status),
"execution_kind": execution_truth["execution_kind"],
"repair_executed": execution_truth["repair_executed"],
},
))
@@ -742,10 +770,32 @@ def _provider_from_description(description: str | None) -> str | None:
return None
def _approval_status_to_timeline_status(status: Any) -> str:
def _approval_execution_truth(approval: Any) -> dict[str, Any]:
raw_metadata = getattr(approval, "extra_metadata", None)
metadata = raw_metadata if isinstance(raw_metadata, dict) else {}
execution_kind = str(metadata.get("execution_kind") or "").strip().lower()
repair_executed = metadata.get("repair_executed")
if repair_executed is None:
if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}:
repair_executed = False
elif is_no_action_approval_action(getattr(approval, "action", None)):
repair_executed = False
return {
"execution_kind": execution_kind or None,
"repair_executed": repair_executed,
}
def _approval_status_to_timeline_status(
status: Any,
*,
repair_executed: bool | None = None,
) -> str:
value = str(_value(status))
if value in {"rejected", "expired"}:
return "error"
if value in {"approved", "execution_success"} and repair_executed is False:
return "info"
if value in {"approved", "execution_success"}:
return "success"
if value == "execution_failed":

View File

@@ -306,6 +306,20 @@ def _safe_int(value: object) -> int:
return 0
def _has_repair_execution_evidence(facts: dict[str, object]) -> bool:
return (
_safe_int(facts.get("auto_repair_execution_records")) > 0
or _safe_int(facts.get("effective_execution_records")) > 0
)
def _has_nonrepair_operation_evidence(facts: dict[str, object]) -> bool:
return (
_safe_int(facts.get("automation_operation_records")) > 0
and not _has_repair_execution_evidence(facts)
)
def _bool_code(value: object, *, unknown_when_none: bool = False) -> str:
if value is None and unknown_when_none:
return "unknown"
@@ -364,12 +378,18 @@ def _format_awooop_status_chain_lines(
km_entries = _safe_int(facts.get("knowledge_entries"))
needs_human = bool(truth_status.get("needs_human"))
has_repair_execution = _has_repair_execution_evidence(facts)
has_nonrepair_operation = _has_nonrepair_operation_evidence(facts)
if verdict == "auto_repaired_verified":
repair_state = "auto_repaired_verified"
next_step = "monitor_for_regression"
elif auto_repair_records > 0 or operation_records > 0:
elif has_repair_execution:
repair_state = "executed_pending_verification" if verification == "missing" else "executed"
next_step = "verify_execution_result"
elif has_nonrepair_operation:
repair_state = "diagnostic_or_audit_recorded"
next_step = "manual_review_or_collect_repair_evidence"
elif remediation_state == "read_only":
repair_state = "read_only_dry_run"
next_step = "approve_or_escalate_from_awooop"
@@ -1430,12 +1450,18 @@ def _callback_reply_awooop_status_chain_snapshot(
km_entries = _safe_int(facts.get("knowledge_entries"))
needs_human = bool(truth_status.get("needs_human"))
has_repair_execution = _has_repair_execution_evidence(facts)
has_nonrepair_operation = _has_nonrepair_operation_evidence(facts)
if verdict == "auto_repaired_verified":
repair_state = "auto_repaired_verified"
next_step = "monitor_for_regression"
elif auto_repair_records > 0 or operation_records > 0:
elif has_repair_execution:
repair_state = "executed_pending_verification" if verification == "missing" else "executed"
next_step = "verify_execution_result"
elif has_nonrepair_operation:
repair_state = "diagnostic_or_audit_recorded"
next_step = "manual_review_or_collect_repair_evidence"
elif remediation_state == "read_only":
repair_state = "read_only_dry_run"
next_step = "approve_or_escalate_from_awooop"
@@ -1758,17 +1784,21 @@ class TelegramMessage:
quality = self.automation_quality or {}
facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {}
verdict = str(quality.get("verdict") or "")
auto_repair_records = int(facts.get("auto_repair_execution_records") or 0)
operation_records = int(facts.get("automation_operation_records") or 0)
has_repair_execution = _has_repair_execution_evidence(facts)
has_nonrepair_operation = _has_nonrepair_operation_evidence(facts)
verification = str(facts.get("verification_result") or "missing")
remediation_state = _remediation_evidence_state(self.remediation_summary)
if verdict == "auto_repaired_verified":
return "✅ 已驗證自動修復完成"
if auto_repair_records > 0 or operation_records > 0:
if has_repair_execution:
if verification == "missing":
return "🔄 已自動執行,等待驗證證據"
return f"🔄 已自動執行,驗證結果:{verification}"
if has_nonrepair_operation:
if verification == "missing":
return "🔎 已記錄診斷/觀察,尚未證明修復"
return f"🔎 已記錄診斷/觀察,驗證結果:{verification}"
if remediation_state == "read_only":
return "🔎 AI 已完成只讀補救試跑,等待人工審批"
if remediation_state == "write_observed":
@@ -1844,6 +1874,8 @@ class TelegramMessage:
)
auto_repair_records = int(facts.get("auto_repair_execution_records") or 0)
operation_records = int(facts.get("automation_operation_records") or 0)
has_repair_execution = _has_repair_execution_evidence(facts)
has_nonrepair_operation = _has_nonrepair_operation_evidence(facts)
verification = str(facts.get("verification_result") or "missing")
gateway_total = int(facts.get("mcp_gateway_total") or 0)
km_entries = int(facts.get("knowledge_entries") or 0)
@@ -1858,8 +1890,10 @@ class TelegramMessage:
match_state = self.playbook_name or "rule_catalog"
if auto_repair_records > 0:
execute_state = f"auto_repair_recorded:{auto_repair_records}"
elif operation_records > 0:
elif has_repair_execution:
execute_state = f"operation_recorded:{operation_records}"
elif has_nonrepair_operation:
execute_state = f"diagnostic_recorded:{operation_records}"
elif is_noop:
execute_state = "no_action_or_observe"
elif "approval" in verdict or self._automation_mode() == "ai_proposal_ready":
@@ -1869,15 +1903,17 @@ class TelegramMessage:
if verification != "missing":
verify_state = verification
elif auto_repair_records > 0 or operation_records > 0:
elif has_repair_execution:
verify_state = "pending_or_missing"
else:
verify_state = "not_started"
if verdict == "auto_repaired_verified":
conclusion = "已驗證自動修復"
elif auto_repair_records > 0 or operation_records > 0:
elif has_repair_execution:
conclusion = "已記錄執行,等待或缺少驗證"
elif has_nonrepair_operation:
conclusion = "已記錄診斷/觀察,尚未證明修復"
elif is_noop:
conclusion = "未自動修復,需人工判斷"
elif "approval" in verdict: