Files
awoooi/apps/api/src/services/operator_outcome.py
Your Name 551227f3bb
Some checks failed
CD Pipeline / tests (push) Successful in 1m45s
CD Pipeline / build-and-deploy (push) Successful in 4m55s
CD Pipeline / post-deploy-checks (push) Successful in 1m58s
Code Review / ai-code-review (push) Has been cancelled
Ansible / Reboot Recovery Contract / validate (push) Has been cancelled
fix(ai): convert legacy manual gates to controlled automation
2026-06-27 19:35:41 +08:00

647 lines
27 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Operator-facing alert outcome contract.
This module is intentionally generic: it converts the existing truth-chain
stage, automation quality verdict, and remediation evidence state into one
small contract that Telegram, AwoooP, and result notifications can all share.
"""
from __future__ import annotations
from typing import Any
_ACTION_REQUIRED_CHANNELS = ("telegram_sre_war_room", "awooop_operator_console")
_RESULT_ONLY_CHANNELS = ("telegram_result_reply", "awooop_operator_console")
_LEGACY_AI_CONTROLLED_STATE_REWRITE = {
"diagnostic_only_manual_review": (
"diagnostic_only_ai_repair_required",
"auto_generate_repair_candidate_from_diagnostic_evidence",
"只完成診斷/觀察AI 已排入 PlayBook / transport / verifier 修復候選",
),
"verification_degraded_manual_required": (
"verification_degraded_ai_verifier_required",
"run_verifier_or_rollback_candidate",
"驗證退化AI 已排入 verifier / rollback 判定",
),
"execution_unverified_manual_required": (
"execution_unverified_controlled_verifier_required",
"run_post_execution_verifier_or_rollback",
"已執行但缺少驗證結果AI 進入 verifier / rollback 判定",
),
"no_action_manual_review": (
"no_action_ai_candidate_required",
"collect_evidence_or_generate_playbook_candidate",
"AI 未找到可執行修復;已排入 evidence / PlayBook 補強與 controlled policy 判定",
),
"approval_expired_manual_review": (
"approval_expired_ai_retry",
"ai_retry_or_rebuild_controlled_packet",
"舊審批已逾期AI 重新建立受控處置包",
),
"write_observed_manual_review": (
"write_observed_ai_verifier_or_rollback",
"auto_verify_or_rollback_observed_write",
"補救證據出現寫入旗標AI 進入 verifier / rollback 判定",
),
"blocked_manual_required": (
"blocked_ai_repair_required",
"auto_repair_blocker_or_connector_then_retry",
"自動化流程受阻AI 已排入 blocker / connector 修復",
),
}
_BREAK_GLASS_BLOCKER_TOKENS = (
"secret",
"token",
"private_key",
"authorization_header",
"drop",
"truncate",
"restore",
"prune",
"remote_delete",
"reboot",
"node_drain",
"firewall_cutover",
"credentialed_exploit",
"external_active_scan",
"paid_provider",
"cost_limit",
"force_push",
"repo_delete",
"ref_delete",
"raw_runtime_secret_volume",
"break_glass",
)
def _safe_int(value: Any) -> int:
try:
return int(value or 0)
except (TypeError, ValueError):
return 0
def _first_text(values: list[Any]) -> str | None:
for value in values:
if value:
return str(value)
return None
def _requires_break_glass(blockers: list[str]) -> bool:
haystack = " ".join(str(item).lower() for item in blockers)
return any(token in haystack for token in _BREAK_GLASS_BLOCKER_TOKENS)
def _canonical_operator_state(state: str) -> str:
rewritten = _LEGACY_AI_CONTROLLED_STATE_REWRITE.get(state)
return rewritten[0] if rewritten else state
def normalize_operator_outcome(outcome: dict[str, Any] | None) -> dict[str, Any] | None:
"""Rewrite legacy human-gate outcomes to the current controlled-AI policy.
Older truth-chain rows can contain states such as ``*_manual_required``.
They remain useful as history, but the operator-facing contract must not
present them as the live product path unless a break-glass blocker exists.
"""
if not isinstance(outcome, dict):
return outcome
state = str(outcome.get("state") or "")
rewrite = _LEGACY_AI_CONTROLLED_STATE_REWRITE.get(state)
if not rewrite:
return outcome
normalized = dict(outcome)
new_state, next_action, summary = rewrite
blockers = [
str(item)
for item in normalized.get("blockers", [])
if item
] if isinstance(normalized.get("blockers"), list) else []
break_glass = _requires_break_glass(blockers)
normalized["legacy_state"] = state
normalized["state"] = "break_glass_required" if break_glass else new_state
normalized["summary_zh"] = (
"命中硬阻擋,需 break-glass 或外部授權"
if break_glass
else summary
)
normalized["needs_human"] = break_glass
normalized["human_action_required"] = break_glass
normalized["next_action"] = (
"break_glass_or_external_authorization"
if break_glass
else next_action
)
normalized["human_action_reason"] = (
normalized.get("human_action_reason")
if break_glass
else "legacy_manual_gate_converted_to_controlled_ai_policy"
)
notification = (
dict(normalized.get("notification"))
if isinstance(normalized.get("notification"), dict)
else {}
)
notification["mode"] = "action_required"
notification["channels"] = list(_ACTION_REQUIRED_CHANNELS)
notification["reason"] = str(normalized["human_action_reason"])
normalized["notification"] = notification
return normalized
def _verification_blocker_code(value: Any) -> str:
status = str(value or "missing").strip().lower()
if status in {"", "none", "null", "missing", "--"}:
return "verification_missing"
if status in {"success", "passed", "healthy", "verified", "repaired", "ok"}:
return "verification_recorded"
return f"verification_{status}"
def normalize_operator_blockers(
blockers: list[Any],
facts: dict[str, Any] | None = None,
) -> list[str]:
"""Translate gate names into operator-facing missing/degraded blockers.
Automation quality gates are named after the desired evidence
(for example ``verification_recorded``). When that gate is the blocker,
the operator must see what is missing, not the name of the target state.
"""
facts = facts or {}
normalized: list[str] = []
for raw in blockers:
if not raw:
continue
item = str(raw)
if item == "auto_repair_recorded" and _safe_int(
facts.get("auto_repair_execution_records")
) <= 0:
item = "auto_repair_missing"
elif item == "verification_recorded":
item = _verification_blocker_code(facts.get("verification_result"))
elif item == "learning_recorded" and _safe_int(facts.get("knowledge_entries")) <= 0:
item = "learning_missing"
if item not in normalized:
normalized.append(item)
return normalized
def _build_notification(
*,
mode: str,
channels: tuple[str, ...],
reason: str,
source_id: str | None,
) -> dict[str, Any]:
return {
"mode": mode,
"channels": list(channels),
"reason": reason,
"source_id": source_id,
"telegram": (
"reply_to_original_or_standalone_action_required"
if mode == "action_required"
else "reply_to_original_or_standalone_result"
),
"awooop": "status_chain_panel",
}
def _build_execution_result(
*,
state: str,
verdict: str,
stage: str,
has_repair_execution: bool,
has_nonrepair_operation: bool,
verification: str,
) -> dict[str, Any]:
"""Describe execution completion separately from remediation outcome."""
state = _canonical_operator_state(state)
approval_status = "unknown"
completion_status = "unknown"
command_status = "unknown"
repair_status = "unknown"
failure_status = "unknown"
summary = "尚未能判定執行是否完成或失敗"
terminal = False
if state == "completed_verified":
approval_status = "completed"
completion_status = "completed_verified"
command_status = "succeeded"
repair_status = "verified_repaired"
failure_status = "no_failure"
summary = "已完成:修復指令成功,且驗證通過"
terminal = True
elif state in {"execution_failed_manual_required", "execution_failed_ai_recovery_required"}:
approval_status = "auto_authorized"
completion_status = "failed"
command_status = "failed"
repair_status = "ai_rollback_or_repair_queued"
failure_status = "command_failed"
summary = "已失敗修復指令執行失敗AI 已排入 rollback / repair 候選"
terminal = False
elif state == "diagnostic_only_ai_repair_required":
approval_status = "auto_authorized"
completion_status = "diagnostic_completed_ai_repair_queued"
command_status = "diagnostic_completed" if has_nonrepair_operation else "skipped_no_action"
repair_status = "playbook_or_transport_repair_required"
failure_status = "no_command_failed"
summary = "只完成診斷/觀察AI 已排入修復候選補齊"
terminal = False
elif state == "verification_degraded_ai_verifier_required":
approval_status = "auto_authorized"
completion_status = "completed_verification_degraded_ai_recovery"
command_status = "succeeded" if has_repair_execution else "diagnostic_completed"
repair_status = "ai_verifier_or_rollback_required"
failure_status = "no_command_failed"
summary = "已執行但驗證退化AI 已排入 verifier / rollback 判定"
terminal = False
elif state == "execution_unverified_controlled_verifier_required":
approval_status = "auto_authorized"
completion_status = "completed_unverified"
command_status = "succeeded"
repair_status = "ai_verifier_or_rollback_required"
failure_status = "no_command_failed"
summary = "已執行成功但缺少修復驗證結果AI 進入 verifier / rollback 判定"
terminal = False
elif state == "apply_candidate_owner_review_ready":
approval_status = "auto_authorized_preflight"
completion_status = "dry_run_passed_apply_candidate_ready"
command_status = "check_mode_succeeded"
repair_status = "not_executed"
failure_status = "not_applicable"
summary = "AI 已完成安全乾跑並產生 apply candidate排入受控 preflight / verifier 後才可執行"
terminal = False
elif state == "controlled_apply_queued":
approval_status = "auto_authorized"
completion_status = "dry_run_passed_controlled_apply_queued"
command_status = "check_mode_succeeded"
repair_status = "controlled_apply_pending"
failure_status = "not_applicable"
summary = "AI 已完成 check-mode已排入受控自動 apply"
terminal = False
elif state == "ai_playbook_repair_required":
approval_status = "auto_authorized"
completion_status = "dry_run_failed_ai_repairing_playbook_or_transport"
command_status = "check_mode_failed"
repair_status = "playbook_or_transport_repair_required"
failure_status = "check_mode_failed"
summary = "AI check-mode 失敗,改由 AI 修正 PlayBook / transport / KM 後再重試"
terminal = False
elif state == "dry_run_only_owner_review_required":
approval_status = "owner_review_required"
completion_status = "dry_run_completed_no_apply"
command_status = "check_mode_succeeded"
repair_status = "not_executed"
failure_status = "not_applicable"
summary = "只完成 Ansible check-mode 乾跑,尚未執行修復"
terminal = False
elif state == "blocked_ai_repair_required":
approval_status = "auto_authorized"
completion_status = "blocked_ai_repairing"
command_status = "blocked_before_success"
repair_status = "blocker_or_connector_repair_required"
failure_status = "blocked"
summary = "自動化受阻AI 已排入 blocker / connector 修復候選"
terminal = False
elif state == "write_observed_manual_review":
approval_status = "auto_authorized"
completion_status = "write_observed_verifier_or_rollback"
command_status = "write_observed"
repair_status = "verifier_or_rollback_required"
failure_status = "write_flag_observed"
summary = "補救證據出現寫入旗標AI 已排入 verifier / rollback 判定"
terminal = False
elif state == "truth_chain_ai_recovery_required":
approval_status = "auto_authorized"
completion_status = "legacy_human_gate_converted_to_ai_recovery"
command_status = "pending_ai_action"
repair_status = "ai_recovery_required"
failure_status = "not_applicable"
summary = "真相鏈舊人工閘已轉入 AI 受控處理"
terminal = False
elif state == "no_action_ai_candidate_required":
approval_status = "controlled_policy_check"
completion_status = "not_started_no_action"
command_status = "not_started"
repair_status = "controlled_apply_evaluation"
failure_status = "not_applicable"
summary = "尚未執行AI 建議不修復,已排入 evidence / PlayBook 補強與 controlled policy 判定"
terminal = False
elif state == "approval_rejected_no_execution":
approval_status = "rejected"
completion_status = "closed_no_execution"
command_status = "not_run"
repair_status = "not_executed"
failure_status = "not_applicable"
summary = "已拒絕:審批結案,未執行任何修復指令"
terminal = True
elif state == "approval_expired_ai_retry":
approval_status = "controlled_policy_retry"
completion_status = "expired_route_requeued"
command_status = "not_run"
repair_status = "controlled_apply_evaluation"
failure_status = "not_applicable"
summary = "舊審批逾期:已改排 AI retry / rollback / evidence 補強"
terminal = False
elif state == "approval_required":
approval_status = "auto_policy_check"
completion_status = "current_policy_auto_authorized"
command_status = "not_started"
repair_status = "controlled_apply_evaluation"
failure_status = "not_applicable"
summary = "尚未執行AI 正在套用目前 owner policy / break-glass 判定"
terminal = False
elif state in {"read_only_dry_run_manual_gate", "read_only_dry_run_controlled_apply_gate"}:
approval_status = "auto_policy_check"
completion_status = "dry_run_completed"
command_status = "dry_run_succeeded"
repair_status = "controlled_apply_evaluation"
failure_status = "not_applicable"
summary = "只讀試跑完成AI 正在判定受控 apply"
terminal = False
elif state == "observed_not_executed":
approval_status = "not_required"
completion_status = "observed_not_executed"
command_status = "not_run"
repair_status = "not_executed"
failure_status = "not_applicable"
summary = "只完成收件/觀測,尚未進入執行"
terminal = False
elif has_repair_execution:
approval_status = "completed"
completion_status = "completed_needs_review"
command_status = "succeeded" if verification != "missing" else "completed"
repair_status = "ai_verifier_or_rollback_required"
failure_status = "no_command_failed"
summary = "已有執行紀錄AI 進入 verifier / rollback 判定"
terminal = False
elif verdict in {"received_only", "observed_not_executed"} or stage == "received":
approval_status = "not_started"
completion_status = "not_started"
command_status = "not_started"
repair_status = "not_executed"
failure_status = "not_applicable"
summary = "尚未執行修復指令AI 應補 evidence / PlayBook / controlled policy"
terminal = False
return {
"approval_status": approval_status,
"completion_status": completion_status,
"command_status": command_status,
"repair_status": repair_status,
"failure_status": failure_status,
"terminal": terminal,
"summary_zh": summary,
}
def build_operator_outcome(
*,
truth_status: dict[str, Any] | None = None,
automation_quality: dict[str, Any] | None = None,
remediation_state: str | None = None,
fetch_error: str | None = None,
source_id: str | None = None,
) -> dict[str, Any]:
"""Build a normalized operator outcome for an alert/incident.
The output deliberately answers three questions:
1. What happened?
2. Does a human need to intervene?
3. How will that human be notified / where should they act?
"""
truth_status = truth_status or {}
automation_quality = automation_quality or {}
facts = automation_quality.get("facts")
if not isinstance(facts, dict):
facts = {}
verdict = str(automation_quality.get("verdict") or "unknown")
stage = str(truth_status.get("current_stage") or "unknown")
stage_status = str(truth_status.get("stage_status") or "unknown")
blockers = [
str(item)
for item in [
*(truth_status.get("blockers") if isinstance(truth_status.get("blockers"), list) else []),
*(automation_quality.get("blockers") if isinstance(automation_quality.get("blockers"), list) else []),
]
if item
]
blockers = normalize_operator_blockers(blockers, facts)
verification = str(facts.get("verification_result") or "missing")
has_repair_execution = _safe_int(facts.get("effective_execution_records")) > 0 or _safe_int(
facts.get("auto_repair_execution_records")
) > 0
ansible_dry_run_only = (
_safe_int(facts.get("ansible_check_mode_total")) > 0
and _safe_int(facts.get("ansible_apply_total")) == 0
and _safe_int(facts.get("auto_repair_execution_records")) == 0
)
has_nonrepair_operation = (
_safe_int(facts.get("automation_operation_records")) > 0
and not has_repair_execution
)
needs_human_from_truth = bool(truth_status.get("needs_human"))
first_blocker = _first_text(blockers)
break_glass = _requires_break_glass(blockers)
if fetch_error:
state = "truth_chain_connector_repair_required"
severity = "warning"
needs_human = False
next_action = "auto_repair_truth_chain_connector_then_retry"
summary = "真相鏈查詢失敗AI 已排入 connector / evidence 修復"
reason = str(fetch_error)[:240]
elif verdict == "auto_repaired_verified":
state = "completed_verified"
severity = "success"
needs_human = False
next_action = "monitor_for_regression"
summary = "已驗證自動修復完成"
reason = "execution_and_verification_succeeded"
elif (verdict == "ansible_check_mode_only" or ansible_dry_run_only) and stage == "execution_failed":
state = "ai_playbook_repair_required"
severity = "warning"
needs_human = False
next_action = "auto_generate_playbook_or_transport_fix_candidate"
summary = "AI check-mode 失敗,正在轉為 PlayBook / transport 修復候選,不應只丟回人工"
reason = first_blocker or "check_mode_failed_needs_ai_repair_candidate"
elif verdict == "execution_failed" or stage == "execution_failed":
state = "execution_failed_ai_recovery_required"
severity = "critical"
needs_human = False
next_action = "auto_rollback_or_generate_repair_candidate"
summary = "執行失敗AI 已排入 rollback / PlayBook / transport 修復候選"
reason = first_blocker or "execution_failed"
elif verdict == "manual_required_diagnostic_only" or has_nonrepair_operation:
state = "diagnostic_only_ai_repair_required"
severity = "warning"
needs_human = False
next_action = "auto_generate_repair_candidate_from_diagnostic_evidence"
summary = "只完成診斷/觀察AI 已排入 PlayBook / transport / verifier 修復候選"
reason = first_blocker or "diagnostic_or_audit_only"
elif verdict == "auto_repaired_verification_degraded":
state = "verification_degraded_ai_verifier_required"
severity = "warning"
needs_human = False
next_action = "run_verifier_or_rollback_candidate"
summary = "已執行但驗證退化AI 進入 verifier / rollback 判定"
reason = first_blocker or f"verification={verification}"
elif verdict == "ansible_check_mode_only" or ansible_dry_run_only:
state = "controlled_apply_queued"
severity = "info"
needs_human = False
next_action = "wait_for_controlled_apply_and_post_apply_verifier"
summary = "AI 已完成 Ansible check-mode符合受控自動 apply 條件"
reason = first_blocker or "controlled_apply_auto_authorized"
elif verdict == "execution_unverified" or (
has_repair_execution and verification == "missing"
):
state = "execution_unverified_controlled_verifier_required"
severity = "warning"
needs_human = False
next_action = "run_post_execution_verifier_or_rollback"
summary = "已執行但缺少驗證結果AI 進入 verifier / rollback 判定"
reason = first_blocker or "execution_without_verification_result"
elif verdict == "manual_required_no_action":
state = "no_action_ai_candidate_required"
severity = "warning"
needs_human = False
next_action = "collect_evidence_or_generate_playbook_candidate"
summary = "AI 選擇不執行修復,已排入 evidence / PlayBook 補強"
reason = first_blocker or "no_action_or_observe"
elif verdict == "approval_rejected_no_execution" or stage == "approval_rejected":
state = "approval_rejected_no_execution"
severity = "info"
needs_human = False
next_action = "monitor_or_reopen_if_alert_recurs"
summary = "已拒絕處置,未執行修復"
reason = "approval_rejected"
elif verdict == "approval_expired_manual_review" or stage == "approval_expired":
state = "approval_expired_ai_retry"
severity = "warning"
needs_human = False
next_action = "ai_retry_or_rebuild_controlled_packet"
summary = "舊審批已逾期AI 重新建立受控處置包"
reason = first_blocker or "approval_expired"
elif remediation_state == "read_only":
state = "read_only_dry_run_controlled_apply_gate"
severity = "warning"
needs_human = False
next_action = "evaluate_controlled_apply_from_read_only_evidence"
summary = "只讀試跑完成AI 進入受控 apply 判定"
reason = first_blocker or "read_only_dry_run"
elif remediation_state == "write_observed":
state = "write_observed_ai_verifier_or_rollback"
severity = "critical"
needs_human = False
next_action = "auto_verify_or_rollback_observed_write"
summary = "補救證據出現寫入旗標AI 進入 verifier / rollback 判定"
reason = first_blocker or "write_observed"
elif remediation_state in {"blocked", "fetch_failed"}:
state = "blocked_ai_repair_required"
severity = "critical" if remediation_state == "blocked" else "warning"
needs_human = False
next_action = "auto_repair_blocker_or_connector_then_retry"
summary = "自動化流程受阻AI 已排入 blocker / connector 修復"
reason = first_blocker or remediation_state
elif verdict == "approval_required" or stage == "approval_required":
state = "approval_required"
severity = "warning"
needs_human = False
next_action = "apply_current_owner_policy_or_break_glass_gate"
summary = "已進入目前 owner policy 判定;低/中/高風險由 AI 受控執行"
reason = first_blocker or "current_owner_policy_auto_authorized"
elif needs_human_from_truth:
state = "break_glass_required" if break_glass else "truth_chain_ai_recovery_required"
severity = "critical" if break_glass else "warning"
needs_human = break_glass
next_action = (
"break_glass_or_external_authorization"
if break_glass
else "auto_generate_repair_or_break_glass_packet"
)
summary = (
"命中硬阻擋,需 break-glass 或外部授權"
if break_glass
else "真相鏈舊判定需人工;已由目前政策轉入 AI 受控處理"
)
reason = first_blocker or f"{stage}/{stage_status}"
elif verdict in {"observed_not_executed", "received_only"}:
state = "observed_not_executed"
severity = "info"
needs_human = False
next_action = "collect_evidence_or_wait"
summary = "已收到/觀測,尚未進入修復執行"
reason = first_blocker or verdict
else:
state = "unknown_pending_observation"
severity = "warning"
needs_human = break_glass
next_action = (
"break_glass_or_external_authorization"
if break_glass
else "auto_collect_evidence_or_generate_controlled_candidate"
)
summary = (
"命中硬阻擋,需 break-glass 或外部授權"
if break_glass
else "處置結果尚未形成明確結論AI 應補 evidence / PlayBook / verifier"
)
reason = first_blocker or f"{verdict}:{stage}/{stage_status}"
execution_result = _build_execution_result(
state=state,
verdict=verdict,
stage=stage,
has_repair_execution=has_repair_execution,
has_nonrepair_operation=has_nonrepair_operation,
verification=verification,
)
ai_action_required = severity in {"warning", "critical"} and next_action not in {
"monitor_for_regression",
"monitor_or_reopen_if_alert_recurs",
}
mode = "action_required" if needs_human or ai_action_required else "result_only"
channels = _ACTION_REQUIRED_CHANNELS if mode == "action_required" else _RESULT_ONLY_CHANNELS
return {
"schema_version": "operator_outcome_v1",
"state": state,
"severity": severity,
"summary_zh": summary,
"needs_human": needs_human,
"human_action_required": needs_human,
"human_action_reason": reason,
"next_action": next_action,
"execution_result": execution_result,
"notification": _build_notification(
mode=mode,
channels=channels,
reason=reason,
source_id=source_id,
),
"evidence": {
"verdict": verdict,
"current_stage": stage,
"stage_status": stage_status,
"verification": verification,
"auto_repair_execution_records": _safe_int(
facts.get("auto_repair_execution_records")
),
"effective_execution_records": _safe_int(
facts.get("effective_execution_records")
),
"automation_operation_records": _safe_int(
facts.get("automation_operation_records")
),
"mcp_gateway_total": _safe_int(facts.get("mcp_gateway_total")),
"knowledge_entries": _safe_int(facts.get("knowledge_entries")),
},
"blockers": blockers[:8],
}