feat(governance): normalize AI治理告警輸出與元告警解析度
This commit is contained in:
@@ -102,16 +102,46 @@ class FailoverAlerter:
|
||||
logger.debug("governance_alert_dedup_skipped", event_type=event_type)
|
||||
return
|
||||
|
||||
# 格式化 payload 為可讀字串(key=value,換行分隔)
|
||||
detail_lines = "\n".join(
|
||||
f"{_escape_md(str(k))}:{_escape_md(str(v))}"
|
||||
for k, v in payload.items()
|
||||
)
|
||||
msg = (
|
||||
f"*AI 治理警報*\n\n"
|
||||
f"類型:{_escape_md(event_type)}\n\n"
|
||||
f"{detail_lines}"
|
||||
)
|
||||
status = _escape_md(str(payload.get("status", "warning")))
|
||||
impact = _as_dict(payload.get("impact"))
|
||||
remediation = _as_dict(payload.get("remediation"))
|
||||
actionable = _as_dict(payload.get("actionable"))
|
||||
|
||||
impact_lines = _lines_from_dict(impact, max_items=12, compact=True)
|
||||
remediation_lines = _lines_from_list(remediation.get("items"))
|
||||
remediation_next_action = remediation.get("next_action")
|
||||
remediation_hint = remediation.get("hint")
|
||||
actionable_lines = _lines_from_list(actionable.get("items"))
|
||||
|
||||
next_action_line = ""
|
||||
if remediation_next_action:
|
||||
next_action_line = f"\n 下一步:{_escape_md(str(remediation_next_action))}"
|
||||
if remediation_hint:
|
||||
next_action_line += f"\n 提示:{_escape_md(str(remediation_hint))}"
|
||||
|
||||
sections: list[str] = [
|
||||
"⚠️ *AI 治理警報*",
|
||||
f"\n類型:{_escape_md(event_type)}",
|
||||
f"狀態:{status}",
|
||||
]
|
||||
if impact_lines:
|
||||
sections.append(f"\n*影響*\n{impact_lines}")
|
||||
if remediation_lines or next_action_line:
|
||||
sections.append(f"\n*修復方向*")
|
||||
if remediation_lines:
|
||||
sections.append(remediation_lines)
|
||||
if next_action_line:
|
||||
sections.append(next_action_line)
|
||||
if actionable_lines:
|
||||
sections.append(f"\n*可直接自動化*\n{actionable_lines}")
|
||||
|
||||
fallback_items = _fallback_pairs(payload, keep={"status", "impact", "remediation", "actionable"})
|
||||
if fallback_items:
|
||||
sections.append(
|
||||
"\n*欄位快覽(備援)*\n" + "\n".join(fallback_items)
|
||||
)
|
||||
|
||||
msg = "\n".join(sections)
|
||||
await self._send(msg)
|
||||
logger.info("governance_alert_sent", event_type=event_type)
|
||||
|
||||
@@ -259,6 +289,46 @@ def _escape_md(text: str) -> str:
|
||||
return text
|
||||
|
||||
|
||||
def _as_dict(value: Any) -> dict[str, Any]:
|
||||
return value if isinstance(value, dict) else {}
|
||||
|
||||
|
||||
def _lines_from_dict(data: dict[str, Any], max_items: int = 20, compact: bool = False) -> str:
|
||||
if not data:
|
||||
return ""
|
||||
rows = []
|
||||
idx = 0
|
||||
for k in sorted(data.keys()) if isinstance(data, dict) else []:
|
||||
if idx >= max_items:
|
||||
break
|
||||
rows.append(f"{_escape_md(str(k))}:{_escape_md(str(data.get(k)))}")
|
||||
idx += 1
|
||||
if compact and len(rows) >= max_items:
|
||||
rows.append("...(更多欄位略)")
|
||||
return "\n".join(f" {line}" for line in rows)
|
||||
|
||||
|
||||
def _lines_from_list(value: Any) -> str:
|
||||
if not isinstance(value, list):
|
||||
return ""
|
||||
return "\n".join(
|
||||
f" {idx + 1}. {_escape_md(str(item))}"
|
||||
for idx, item in enumerate(value)
|
||||
)
|
||||
|
||||
|
||||
def _fallback_pairs(payload: dict[str, Any], keep: set[str] | None = None) -> list[str]:
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
keep = set(keep or set())
|
||||
rows = []
|
||||
for key in sorted(payload.keys()):
|
||||
if key in keep:
|
||||
continue
|
||||
rows.append(f"{_escape_md(str(key))}:{_escape_md(str(payload.get(key)))}")
|
||||
return rows
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Singleton
|
||||
# =============================================================================
|
||||
|
||||
@@ -142,6 +142,13 @@ class GovernanceAgent:
|
||||
],
|
||||
"sample_playbook_ids": kept_ids[:10],
|
||||
},
|
||||
"drifted_count": len(drifted),
|
||||
"auto_deprecated_count": len(auto_deprecated_ids),
|
||||
"auto_deprecated_ids": auto_deprecated_ids[:10],
|
||||
"playbook_ids": kept_ids[:10],
|
||||
"total_playbooks": total,
|
||||
"threshold": TRUST_DRIFT_THRESHOLD,
|
||||
"auto_deprecate_after_days": TRUST_DRIFT_AUTO_DEPRECATE_AFTER_DAYS,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -215,6 +222,11 @@ class GovernanceAgent:
|
||||
"安排至少 2 位 owner 對 stale條目做快速人工審核",
|
||||
],
|
||||
},
|
||||
"stale_count": stale,
|
||||
"total_count": total,
|
||||
"stale_ratio": round(ratio, 3),
|
||||
"threshold": KM_STALE_RATIO,
|
||||
"stale_days": KM_STALE_DAYS,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -260,6 +272,27 @@ class GovernanceAgent:
|
||||
await self._alert(
|
||||
"llm_hallucination",
|
||||
{
|
||||
"status": "warning",
|
||||
"impact": {
|
||||
"failed_count": failed,
|
||||
"total_checked": total,
|
||||
"hallucination_rate": round(rate, 3),
|
||||
"threshold": HALLUCINATION_RATE_THRESHOLD,
|
||||
},
|
||||
"remediation": {
|
||||
"items": [
|
||||
"檢核 AI 建議來源與 evidence snapshot 一致性",
|
||||
"檢視最近 incident 的 verifier 輸入欄位是否缺失關鍵上下文",
|
||||
],
|
||||
"next_action": "run_knowledge_gap_audit",
|
||||
"hint": "高失敗率通常表示 evidence 收斂流程退化或資料欄位解讀偏差",
|
||||
},
|
||||
"actionable": {
|
||||
"items": [
|
||||
"啟動 `playbook_evidence` 對齊補償流程",
|
||||
"調整 verify timeout 與降級策略,避免過度信任低品質證據",
|
||||
],
|
||||
},
|
||||
"failed_count": failed,
|
||||
"total_checked": total,
|
||||
"hallucination_rate": round(rate, 3),
|
||||
@@ -304,6 +337,27 @@ class GovernanceAgent:
|
||||
await self._alert(
|
||||
"execution_blast_radius",
|
||||
{
|
||||
"status": "warning",
|
||||
"impact": {
|
||||
"failed_count": failed,
|
||||
"total_executions": total,
|
||||
"failure_rate": round(rate, 3),
|
||||
"threshold": EXECUTION_FAIL_RATE_THRESHOLD,
|
||||
},
|
||||
"remediation": {
|
||||
"items": [
|
||||
"鎖定失敗 playbook 清單,關閉高風險自動執行",
|
||||
"比對 incident evidence 與 post_execution_verification 失敗原因",
|
||||
],
|
||||
"next_action": "pause_auto_repair_for_top_failing_playbooks",
|
||||
"hint": "可能是 auto_repair_playbook 與 runtime 版本/環境脫節",
|
||||
},
|
||||
"actionable": {
|
||||
"items": [
|
||||
"跑 `run_self_check` 快照與失敗 playbook 熱點報表",
|
||||
"必要時啟用 emergency fallback 路由進人工審核",
|
||||
],
|
||||
},
|
||||
"failed_count": failed,
|
||||
"total_executions": total,
|
||||
"failure_rate": round(rate, 3),
|
||||
@@ -548,9 +602,25 @@ class GovernanceAgent:
|
||||
await self._alert(
|
||||
"governance_self_failure",
|
||||
{
|
||||
"failed_checks": failed_checks,
|
||||
"total_checks": 5, # 2026-04-27 P3.4 by Claude — 加入 slo_compliance 後共 5 項
|
||||
"errors": {k: results[k].get("error") for k in failed_checks},
|
||||
"status": "critical",
|
||||
"impact": {
|
||||
"failed_checks": failed_checks,
|
||||
"total_checks": 5, # 2026-04-27 P3.4 by Claude — 加入 slo_compliance 後共 5 項
|
||||
"errors": {k: results[k].get("error") for k in failed_checks},
|
||||
},
|
||||
"remediation": {
|
||||
"items": [
|
||||
"暫停非關鍵治理自動化接收鏈路",
|
||||
"聚焦治理執行路徑錯誤並補齊 fallback",
|
||||
],
|
||||
"next_action": "investigate_governance_pipeline_health",
|
||||
},
|
||||
"actionable": {
|
||||
"items": [
|
||||
"檢查 GovernanceAgent run loop 是否完整執行 5 個項目",
|
||||
"確認 DB 寫入與 Prometheus fetch 未被上游干擾",
|
||||
],
|
||||
},
|
||||
},
|
||||
)
|
||||
except Exception:
|
||||
|
||||
@@ -2603,12 +2603,12 @@ class TelegramGateway:
|
||||
f"━━━━━━━━━━━━━━━━━━━\n"
|
||||
f"📋 <code>{html.escape(incident_id)}</code>\n"
|
||||
f"🚨 異常元件:<b>{html.escape(alertname)}</b>\n"
|
||||
f"🎯 診斷結果:{html.escape(diagnosis[:100])}\n"
|
||||
f"🎯 診斷結果:{html.escape(_smart_truncate(diagnosis, 320))}\n"
|
||||
)
|
||||
if system_impact:
|
||||
text += f"\n🧠 <b>系統影響</b>\n{html.escape(system_impact[:150])}\n"
|
||||
text += f"\n🧠 <b>系統影響</b>\n{html.escape(_smart_truncate(system_impact, 320))}\n"
|
||||
if probable_cause:
|
||||
text += f"└─ 可能根因:{html.escape(probable_cause[:100])}\n"
|
||||
text += f"└─ 可能根因:{html.escape(_smart_truncate(probable_cause, 320))}\n"
|
||||
|
||||
# 2026-04-16 ogt: 移除 flywheel_diag / flywheel_dashboard (3-part ghost button,無 handler)
|
||||
# 鐵律: 寧可沒按鈕,不可有死按鈕 (feedback_no_ghost_buttons.md)
|
||||
|
||||
Reference in New Issue
Block a user