wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

chore(governance,watchdog): enrich alerts and enable prometheus multiproc
Some checks failed
CD Pipeline / tests (push) Failing after 1m22s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details
Code Review / ai-code-review (push) Successful in 43s
Details
Deploy Alert Rules / Deploy Prometheus Alert Rules (push) Successful in 57s
Details

Browse Source
This commit is contained in:
Your Name
2026-05-02 23:44:03 +08:00
parent b371edb70c
commit dedb12085b
7 changed files with 191 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

156
apps/api/src/services/governance_agent.py
View File

@@ -106,25 +106,42 @@ class GovernanceAgent:
else:
kept_ids.append(r.playbook_id)
if auto_deprecated_ids:
await db.commit()
logger.info(
"governance_trust_drift_auto_deprecated",
count=len(auto_deprecated_ids),
ids=auto_deprecated_ids[:10],
)
if auto_deprecated_ids:
await db.commit()
logger.info(
"governance_trust_drift_auto_deprecated",
count=len(auto_deprecated_ids),
ids=auto_deprecated_ids[:10],
)
if drifted:
drift_ratio = len(drifted) / total if total > 0 else 0.0
await self._alert(
"trust_drift",
{
"drifted_count": len(drifted),
"total_playbooks": total,
"playbook_ids": kept_ids[:10],
"auto_deprecated_count": len(auto_deprecated_ids),
"auto_deprecated_ids": auto_deprecated_ids[:10],
"threshold": TRUST_DRIFT_THRESHOLD,
"auto_deprecate_after_days": TRUST_DRIFT_AUTO_DEPRECATE_AFTER_DAYS,
"status": "warning",
"impact": {
"drifted_count": len(drifted),
"total_playbooks": total,
"drift_ratio": round(drift_ratio, 3),
"threshold": TRUST_DRIFT_THRESHOLD,
"auto_deprecate_after_days": TRUST_DRIFT_AUTO_DEPRECATE_AFTER_DAYS,
},
"remediation": {
"items": [
"Auto-deprecate low-trust stale playbooks",
"Review candidate playbooks by impact scope and rollback if needed",
],
"auto_deprecated_count": len(auto_deprecated_ids),
"auto_deprecated_ids": auto_deprecated_ids[:10],
},
"actionable": {
"items": [
"立即補齊 PLAYBOOK_SOURCE 與 playbook_metadata",
"必要時人工覆核 kept_ids 中的高風險 Playbook",
],
"sample_playbook_ids": kept_ids[:10],
},
},
)
@@ -177,11 +194,27 @@ class GovernanceAgent:
await self._alert(
"knowledge_degradation",
{
"stale_count": stale,
"total_count": total,
"stale_ratio": round(ratio, 3),
"threshold": KM_STALE_RATIO,
"stale_days": KM_STALE_DAYS,
"status": "warning",
"impact": {
"stale_count": stale,
"total_count": total,
"stale_ratio": round(ratio, 3),
"threshold": KM_STALE_RATIO,
"stale_days": KM_STALE_DAYS,
},
"remediation": {
"items": [
"啟動 KM 反查與自動補齊流程",
"關鍵服務告警自動同步到 KM 任務，補齊缺失條目",
],
"next_action": "run_kb_growth_healthcheck",
},
"actionable": {
"items": [
"每日檢查 ANTI_PATTERN 更新結果",
"安排至少 2 位 owner 對 stale條目做快速人工審核",
],
},
},
)
@@ -348,9 +381,11 @@ class GovernanceAgent:
# 不可 fallback 0.0，否則必觸發 violated=True 噴假警報
if not result_list:
results[name] = {
"name": name,
"status": "skipped",
"error": "no_data",
"skipped": True,
"reason": "prometheus_empty_result_metric_not_emitted",
"hint": "ADR-100 emitter 未實作或 PROMETHEUS_MULTIPROC_DIR 未設",
}
logger.warning(
"governance_slo_no_data",
@@ -365,9 +400,12 @@ class GovernanceAgent:
violated = value < threshold
results[name] = {
"name": name,
"status": "violated" if violated else "ok",
"value": round(value, 4),
"slo_target": target,
"hard_red_line": threshold,
"gap": round(threshold - value, 4) if violated else round(value - target, 4),
"violated": violated,
}
@@ -375,11 +413,27 @@ class GovernanceAgent:
await self._alert(
f"slo_{name}_violation",
{
"slo_name": name,
"current_value": round(value, 4),
"hard_red_line": threshold,
"slo_target": target,
"gap": round(threshold - value, 4),
"status": "violation",
"impact": {
"name": name,
"value": round(value, 4),
"target": target,
"threshold": threshold,
"gap": round(threshold - value, 4),
},
"remediation": {
"items": [
"Pause auto-scaling or risky auto-fix tasks",
"Review evidence/decision traces and adjust policy thresholds",
],
"next_action": "trigger_flywheel_safeguard",
},
"actionable": {
"items": [
"Check verifier lag and post-exec learning health",
"Run emergency incident audit on failed approvals",
],
},
},
)
logger.warning(
@@ -388,6 +442,12 @@ class GovernanceAgent:
value=round(value, 4),
hard_red_line=threshold,
)
elif value == 0 and threshold <= 0:
logger.warning(
"governance_slo_unexpected_zero",
slo=name,
value=round(value, 4),
)
else:
logger.info(
"governance_slo_ok",
@@ -396,7 +456,12 @@ class GovernanceAgent:
target=target,
)
else:
results[name] = {"error": "prometheus_query_failed", "status": data.get("status")}
results[name] = {
"name": name,
"status": "error",
"error": "prometheus_query_failed",
"response_status": data.get("status"),
}
logger.warning(
"governance_slo_prometheus_error",
slo=name,
@@ -404,22 +469,30 @@ class GovernanceAgent:
response_status=data.get("status"),
)
except Exception as e:
results[name] = {"error": str(e)}
results[name] = {
"name": name,
"status": "error",
"error": str(e),
}
logger.warning("governance_slo_check_error", slo=name, error=str(e))
# 2026-04-29 ogt + Claude Opus 4.7: critic M6 修
# 加聚合 _meta 區分「全 skipped」(metric 未 emit) vs「全 ok」(SLO 健康)
# 防止 dashboard 把 no_data 當 pass 顯示
violated_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("violated"))
skipped_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("skipped"))
violated_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("status") == "violated")
skipped_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("status") == "skipped")
ok_count = sum(
1 for v in results.values()
if isinstance(v, dict) and not v.get("violated") and not v.get("skipped") and "error" not in v
if isinstance(v, dict)
and v.get("status") == "ok"
)
error_count = sum(1 for v in results.values() if isinstance(v, dict) and v.get("status") == "error")
results["_meta"] = {
"violated_count": violated_count,
"skipped_count": skipped_count,
"ok_count": ok_count,
"error_count": error_count,
"all_status": sorted({v.get("status") for v in results.values() if isinstance(v, dict)}),
"all_skipped": skipped_count > 0 and ok_count == 0 and violated_count == 0,
"status": (
"no_data" if (skipped_count > 0 and ok_count == 0 and violated_count == 0)
@@ -496,9 +569,26 @@ class GovernanceAgent:
await self._alert(
"governance_slo_data_gap",
{
"reason": "all_slo_metrics_not_emitted",
"skipped_count": slo_meta.get("skipped_count", 0),
"hint": "ADR-100 emitter 未實作或 PROMETHEUS_MULTIPROC_DIR 未設",
"status": "warning",
"impact": {
"reason": "all_slo_metrics_not_emitted",
"skipped_count": slo_meta.get("skipped_count", 0),
"all_slo_metrics_not_emitted": True,
},
"remediation": {
"items": [
"補齊 ADR-100 SLO emitter（automation_operation_log_total / post_execution_verification_total / km_entries_total）",
"設置 PROMETHEUS_MULTIPROC_DIR 並掛載可寫目錄（如 emptyDir）",
],
"next_action": "run_adr100_slo_emit_playbook",
"hint": "ADR-100 emitter 未實作或 PROMETHEUS_MULTIPROC_DIR 未設",
},
"actionable": {
"items": [
"先確認所有 API Pod 是否有 PROMETHEUS_MULTIPROC_DIR 掛載",
"檢查 Prometheus rule 是否已載入 sli:autonomy_rate:5m 等 4 項告警規則",
],
},
},
)
except Exception: