fix(governance): dedupe km degradation owner review
All checks were successful
CD Pipeline / tests (push) Successful in 5m4s
Code Review / ai-code-review (push) Successful in 10s
CD Pipeline / build-and-deploy (push) Successful in 4m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m38s

This commit is contained in:
Your Name
2026-05-24 16:14:51 +08:00
parent 7fd52d26b5
commit de68514283
3 changed files with 275 additions and 12 deletions

View File

@@ -20,12 +20,13 @@ from datetime import timedelta
from typing import Any
import structlog
from sqlalchemy import func, select
from sqlalchemy import func, select, update
from src.db.base import get_db_context
from src.db.models import (
AiGovernanceEvent,
AutoRepairExecution,
GovernanceRemediationDispatch,
IncidentEvidence,
KnowledgeEntryRecord,
PlaybookRecord,
@@ -53,6 +54,7 @@ KM_STALE_RATIO = 0.20 # 陳舊比例超過此值 → 告警
HALLUCINATION_RATE_THRESHOLD = 0.10 # LLM verification failed 比例超過此值 → 告警
EXECUTION_FAIL_RATE_THRESHOLD = 0.15 # 執行失敗比例超過此值 → 告警
RECENT_LIMIT = 100 # 最近幾筆做統計
GOVERNANCE_SELF_CHECK_LEASE_KEY = "governance:self_check:cycle_lease"
# =============================================================================
@@ -211,6 +213,21 @@ class GovernanceAgent:
ratio = stale / total if total > 0 else 0.0
if total > 0 and ratio > KM_STALE_RATIO:
if await _has_open_knowledge_degradation_review():
logger.info(
"governance_knowledge_degradation_alert_suppressed",
reason="open_owner_review_exists",
total=total,
stale=stale,
ratio=round(ratio, 3),
)
return {
"total": total,
"stale": stale,
"ratio": round(ratio, 3),
"alert_suppressed": True,
"suppress_reason": "open_owner_review_exists",
}
await self._alert(
"knowledge_degradation",
{
@@ -259,7 +276,10 @@ class GovernanceAgent:
stale=stale,
ratio=round(ratio, 3),
)
return {"total": total, "stale": stale, "ratio": round(ratio, 3)}
result = {"total": total, "stale": stale, "ratio": round(ratio, 3)}
if total > 0 and ratio <= KM_STALE_RATIO:
result["resolved_open_events"] = await _resolve_open_knowledge_degradation_events()
return result
# =========================================================================
# 3. LLM 幻覺率
@@ -413,9 +433,10 @@ class GovernanceAgent:
2026-04-27 P3.4 by Claude — AI SLOADR-100
"""
import httpx
import math
import httpx
from src.core.config import settings
prom_url = getattr(settings, "PROMETHEUS_URL", "http://prometheus.observability.svc:9090")
@@ -757,6 +778,70 @@ class GovernanceAgent:
logger.warning("governance_telegram_alert_failed", error=str(e))
async def _has_open_knowledge_degradation_review() -> bool:
"""已有 Hermes owner-review 工單時,不再重複建立 KM stale 告警。
多個 API Pod 會同時啟動 governance loop同一個 stale ratio 若已經
進入 Hermes review draft就應視為「同一個未結治理工作」避免
Telegram / Work Items 每輪產生新的治理事件與 REVIEW 草稿。
"""
try:
async with get_db_context() as db:
result = await db.execute(
select(GovernanceRemediationDispatch.id)
.join(
AiGovernanceEvent,
GovernanceRemediationDispatch.governance_event_id == AiGovernanceEvent.id,
)
.where(AiGovernanceEvent.event_type == "knowledge_degradation")
.where(AiGovernanceEvent.resolved.is_(False))
.where(GovernanceRemediationDispatch.event_type == "knowledge_degradation")
.where(GovernanceRemediationDispatch.executor_type == "hermes_kb_growth_healthcheck")
.where(
GovernanceRemediationDispatch.dispatch_status.in_(
["pending", "dispatched", "executing", "succeeded"]
)
)
.order_by(GovernanceRemediationDispatch.dispatched_at.desc())
.limit(1)
)
return result.scalar_one_or_none() is not None
except Exception as exc:
logger.warning(
"governance_knowledge_degradation_review_lookup_failed_fail_open",
error=str(exc),
)
return False
async def _resolve_open_knowledge_degradation_events() -> int:
"""KM stale ratio 回到門檻內時,收斂未解治理事件。"""
try:
async with get_db_context() as db:
result = await db.execute(
update(AiGovernanceEvent)
.where(AiGovernanceEvent.event_type == "knowledge_degradation")
.where(AiGovernanceEvent.resolved.is_(False))
.values(resolved=True, resolved_at=now_taipei())
.execution_options(synchronize_session=False)
)
resolved_count = int(result.rowcount or 0)
if resolved_count:
await db.commit()
logger.info(
"governance_knowledge_degradation_resolved",
resolved_count=resolved_count,
reason="stale_ratio_recovered",
)
return resolved_count
except Exception as exc:
logger.warning(
"governance_knowledge_degradation_resolve_failed",
error=str(exc),
)
return 0
async def _maybe_create_intake_dispatch(
event_id: str,
event_type: str,
@@ -891,7 +976,40 @@ async def run_governance_loop(interval_seconds: int = 3600) -> None:
agent = get_governance_agent()
while True:
try:
await agent.run_self_check()
if await _try_acquire_governance_self_check_lease(interval_seconds):
await agent.run_self_check()
else:
logger.debug(
"governance_self_check_cycle_skipped",
reason="cycle_lease_held",
)
except Exception as e:
logger.warning("governance_loop_error", error=str(e))
await asyncio.sleep(interval_seconds)
async def _try_acquire_governance_self_check_lease(interval_seconds: int) -> bool:
"""跨 API Pod 的 self-check 週期租約。
這是週期 cooldown不是 critical-section lock取得後不主動 release。
TTL 到期前其他 replica 只略過本輪,避免同一治理狀態被多個 Pod 寫成
多筆事件、多張 Hermes KM 草稿。
"""
ttl = max(60, int(interval_seconds))
try:
from src.core.redis_client import get_redis
redis = get_redis()
acquired = await redis.set(
GOVERNANCE_SELF_CHECK_LEASE_KEY,
"1",
ex=ttl,
nx=True,
)
return bool(acquired)
except Exception as exc:
logger.warning(
"governance_self_check_lease_unavailable_fail_open",
error=str(exc),
)
return True