feat(governance): normalize AI治理告警輸出與元告警解析度
Some checks failed
CD Pipeline / tests (push) Failing after 25s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped
Code Review / ai-code-review (push) Successful in 46s

This commit is contained in:
Your Name
2026-05-02 23:49:59 +08:00
parent a38d911213
commit b710f3f38f
7 changed files with 325 additions and 31 deletions

View File

@@ -102,16 +102,46 @@ class FailoverAlerter:
logger.debug("governance_alert_dedup_skipped", event_type=event_type)
return
# 格式化 payload 為可讀字串key=value換行分隔
detail_lines = "\n".join(
f"{_escape_md(str(k))}{_escape_md(str(v))}"
for k, v in payload.items()
)
msg = (
f"*AI 治理警報*\n\n"
f"類型:{_escape_md(event_type)}\n\n"
f"{detail_lines}"
)
status = _escape_md(str(payload.get("status", "warning")))
impact = _as_dict(payload.get("impact"))
remediation = _as_dict(payload.get("remediation"))
actionable = _as_dict(payload.get("actionable"))
impact_lines = _lines_from_dict(impact, max_items=12, compact=True)
remediation_lines = _lines_from_list(remediation.get("items"))
remediation_next_action = remediation.get("next_action")
remediation_hint = remediation.get("hint")
actionable_lines = _lines_from_list(actionable.get("items"))
next_action_line = ""
if remediation_next_action:
next_action_line = f"\n 下一步:{_escape_md(str(remediation_next_action))}"
if remediation_hint:
next_action_line += f"\n 提示:{_escape_md(str(remediation_hint))}"
sections: list[str] = [
"⚠️ *AI 治理警報*",
f"\n類型:{_escape_md(event_type)}",
f"狀態:{status}",
]
if impact_lines:
sections.append(f"\n*影響*\n{impact_lines}")
if remediation_lines or next_action_line:
sections.append(f"\n*修復方向*")
if remediation_lines:
sections.append(remediation_lines)
if next_action_line:
sections.append(next_action_line)
if actionable_lines:
sections.append(f"\n*可直接自動化*\n{actionable_lines}")
fallback_items = _fallback_pairs(payload, keep={"status", "impact", "remediation", "actionable"})
if fallback_items:
sections.append(
"\n*欄位快覽(備援)*\n" + "\n".join(fallback_items)
)
msg = "\n".join(sections)
await self._send(msg)
logger.info("governance_alert_sent", event_type=event_type)
@@ -259,6 +289,46 @@ def _escape_md(text: str) -> str:
return text
def _as_dict(value: Any) -> dict[str, Any]:
return value if isinstance(value, dict) else {}
def _lines_from_dict(data: dict[str, Any], max_items: int = 20, compact: bool = False) -> str:
if not data:
return ""
rows = []
idx = 0
for k in sorted(data.keys()) if isinstance(data, dict) else []:
if idx >= max_items:
break
rows.append(f"{_escape_md(str(k))}{_escape_md(str(data.get(k)))}")
idx += 1
if compact and len(rows) >= max_items:
rows.append("...(更多欄位略)")
return "\n".join(f" {line}" for line in rows)
def _lines_from_list(value: Any) -> str:
if not isinstance(value, list):
return ""
return "\n".join(
f" {idx + 1}. {_escape_md(str(item))}"
for idx, item in enumerate(value)
)
def _fallback_pairs(payload: dict[str, Any], keep: set[str] | None = None) -> list[str]:
if not isinstance(payload, dict):
return []
keep = set(keep or set())
rows = []
for key in sorted(payload.keys()):
if key in keep:
continue
rows.append(f"{_escape_md(str(key))}{_escape_md(str(payload.get(key)))}")
return rows
# =============================================================================
# Singleton
# =============================================================================

View File

@@ -142,6 +142,13 @@ class GovernanceAgent:
],
"sample_playbook_ids": kept_ids[:10],
},
"drifted_count": len(drifted),
"auto_deprecated_count": len(auto_deprecated_ids),
"auto_deprecated_ids": auto_deprecated_ids[:10],
"playbook_ids": kept_ids[:10],
"total_playbooks": total,
"threshold": TRUST_DRIFT_THRESHOLD,
"auto_deprecate_after_days": TRUST_DRIFT_AUTO_DEPRECATE_AFTER_DAYS,
},
)
@@ -215,6 +222,11 @@ class GovernanceAgent:
"安排至少 2 位 owner 對 stale條目做快速人工審核",
],
},
"stale_count": stale,
"total_count": total,
"stale_ratio": round(ratio, 3),
"threshold": KM_STALE_RATIO,
"stale_days": KM_STALE_DAYS,
},
)
@@ -260,6 +272,27 @@ class GovernanceAgent:
await self._alert(
"llm_hallucination",
{
"status": "warning",
"impact": {
"failed_count": failed,
"total_checked": total,
"hallucination_rate": round(rate, 3),
"threshold": HALLUCINATION_RATE_THRESHOLD,
},
"remediation": {
"items": [
"檢核 AI 建議來源與 evidence snapshot 一致性",
"檢視最近 incident 的 verifier 輸入欄位是否缺失關鍵上下文",
],
"next_action": "run_knowledge_gap_audit",
"hint": "高失敗率通常表示 evidence 收斂流程退化或資料欄位解讀偏差",
},
"actionable": {
"items": [
"啟動 `playbook_evidence` 對齊補償流程",
"調整 verify timeout 與降級策略,避免過度信任低品質證據",
],
},
"failed_count": failed,
"total_checked": total,
"hallucination_rate": round(rate, 3),
@@ -304,6 +337,27 @@ class GovernanceAgent:
await self._alert(
"execution_blast_radius",
{
"status": "warning",
"impact": {
"failed_count": failed,
"total_executions": total,
"failure_rate": round(rate, 3),
"threshold": EXECUTION_FAIL_RATE_THRESHOLD,
},
"remediation": {
"items": [
"鎖定失敗 playbook 清單,關閉高風險自動執行",
"比對 incident evidence 與 post_execution_verification 失敗原因",
],
"next_action": "pause_auto_repair_for_top_failing_playbooks",
"hint": "可能是 auto_repair_playbook 與 runtime 版本/環境脫節",
},
"actionable": {
"items": [
"跑 `run_self_check` 快照與失敗 playbook 熱點報表",
"必要時啟用 emergency fallback 路由進人工審核",
],
},
"failed_count": failed,
"total_executions": total,
"failure_rate": round(rate, 3),
@@ -548,9 +602,25 @@ class GovernanceAgent:
await self._alert(
"governance_self_failure",
{
"failed_checks": failed_checks,
"total_checks": 5, # 2026-04-27 P3.4 by Claude — 加入 slo_compliance 後共 5 項
"errors": {k: results[k].get("error") for k in failed_checks},
"status": "critical",
"impact": {
"failed_checks": failed_checks,
"total_checks": 5, # 2026-04-27 P3.4 by Claude — 加入 slo_compliance 後共 5 項
"errors": {k: results[k].get("error") for k in failed_checks},
},
"remediation": {
"items": [
"暫停非關鍵治理自動化接收鏈路",
"聚焦治理執行路徑錯誤並補齊 fallback",
],
"next_action": "investigate_governance_pipeline_health",
},
"actionable": {
"items": [
"檢查 GovernanceAgent run loop 是否完整執行 5 個項目",
"確認 DB 寫入與 Prometheus fetch 未被上游干擾",
],
},
},
)
except Exception:

View File

@@ -2603,12 +2603,12 @@ class TelegramGateway:
f"━━━━━━━━━━━━━━━━━━━\n"
f"📋 <code>{html.escape(incident_id)}</code>\n"
f"🚨 異常元件:<b>{html.escape(alertname)}</b>\n"
f"🎯 診斷結果:{html.escape(diagnosis[:100])}\n"
f"🎯 診斷結果:{html.escape(_smart_truncate(diagnosis, 320))}\n"
)
if system_impact:
text += f"\n🧠 <b>系統影響</b>\n{html.escape(system_impact[:150])}\n"
text += f"\n🧠 <b>系統影響</b>\n{html.escape(_smart_truncate(system_impact, 320))}\n"
if probable_cause:
text += f"└─ 可能根因:{html.escape(probable_cause[:100])}\n"
text += f"└─ 可能根因:{html.escape(_smart_truncate(probable_cause, 320))}\n"
# 2026-04-16 ogt: 移除 flywheel_diag / flywheel_dashboard (3-part ghost button無 handler)
# 鐵律: 寧可沒按鈕,不可有死按鈕 (feedback_no_ghost_buttons.md)