fix(governance): dedupe km degradation owner review
This commit is contained in:
@@ -20,12 +20,13 @@ from datetime import timedelta
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, update
|
||||
|
||||
from src.db.base import get_db_context
|
||||
from src.db.models import (
|
||||
AiGovernanceEvent,
|
||||
AutoRepairExecution,
|
||||
GovernanceRemediationDispatch,
|
||||
IncidentEvidence,
|
||||
KnowledgeEntryRecord,
|
||||
PlaybookRecord,
|
||||
@@ -53,6 +54,7 @@ KM_STALE_RATIO = 0.20 # 陳舊比例超過此值 → 告警
|
||||
HALLUCINATION_RATE_THRESHOLD = 0.10 # LLM verification failed 比例超過此值 → 告警
|
||||
EXECUTION_FAIL_RATE_THRESHOLD = 0.15 # 執行失敗比例超過此值 → 告警
|
||||
RECENT_LIMIT = 100 # 最近幾筆做統計
|
||||
GOVERNANCE_SELF_CHECK_LEASE_KEY = "governance:self_check:cycle_lease"
|
||||
|
||||
|
||||
# =============================================================================
|
||||
@@ -211,6 +213,21 @@ class GovernanceAgent:
|
||||
ratio = stale / total if total > 0 else 0.0
|
||||
|
||||
if total > 0 and ratio > KM_STALE_RATIO:
|
||||
if await _has_open_knowledge_degradation_review():
|
||||
logger.info(
|
||||
"governance_knowledge_degradation_alert_suppressed",
|
||||
reason="open_owner_review_exists",
|
||||
total=total,
|
||||
stale=stale,
|
||||
ratio=round(ratio, 3),
|
||||
)
|
||||
return {
|
||||
"total": total,
|
||||
"stale": stale,
|
||||
"ratio": round(ratio, 3),
|
||||
"alert_suppressed": True,
|
||||
"suppress_reason": "open_owner_review_exists",
|
||||
}
|
||||
await self._alert(
|
||||
"knowledge_degradation",
|
||||
{
|
||||
@@ -259,7 +276,10 @@ class GovernanceAgent:
|
||||
stale=stale,
|
||||
ratio=round(ratio, 3),
|
||||
)
|
||||
return {"total": total, "stale": stale, "ratio": round(ratio, 3)}
|
||||
result = {"total": total, "stale": stale, "ratio": round(ratio, 3)}
|
||||
if total > 0 and ratio <= KM_STALE_RATIO:
|
||||
result["resolved_open_events"] = await _resolve_open_knowledge_degradation_events()
|
||||
return result
|
||||
|
||||
# =========================================================================
|
||||
# 3. LLM 幻覺率
|
||||
@@ -413,9 +433,10 @@ class GovernanceAgent:
|
||||
|
||||
2026-04-27 P3.4 by Claude — AI SLO(ADR-100)
|
||||
"""
|
||||
import httpx
|
||||
import math
|
||||
|
||||
import httpx
|
||||
|
||||
from src.core.config import settings
|
||||
|
||||
prom_url = getattr(settings, "PROMETHEUS_URL", "http://prometheus.observability.svc:9090")
|
||||
@@ -757,6 +778,70 @@ class GovernanceAgent:
|
||||
logger.warning("governance_telegram_alert_failed", error=str(e))
|
||||
|
||||
|
||||
async def _has_open_knowledge_degradation_review() -> bool:
|
||||
"""已有 Hermes owner-review 工單時,不再重複建立 KM stale 告警。
|
||||
|
||||
多個 API Pod 會同時啟動 governance loop;同一個 stale ratio 若已經
|
||||
進入 Hermes review draft,就應視為「同一個未結治理工作」,避免
|
||||
Telegram / Work Items 每輪產生新的治理事件與 REVIEW 草稿。
|
||||
"""
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
select(GovernanceRemediationDispatch.id)
|
||||
.join(
|
||||
AiGovernanceEvent,
|
||||
GovernanceRemediationDispatch.governance_event_id == AiGovernanceEvent.id,
|
||||
)
|
||||
.where(AiGovernanceEvent.event_type == "knowledge_degradation")
|
||||
.where(AiGovernanceEvent.resolved.is_(False))
|
||||
.where(GovernanceRemediationDispatch.event_type == "knowledge_degradation")
|
||||
.where(GovernanceRemediationDispatch.executor_type == "hermes_kb_growth_healthcheck")
|
||||
.where(
|
||||
GovernanceRemediationDispatch.dispatch_status.in_(
|
||||
["pending", "dispatched", "executing", "succeeded"]
|
||||
)
|
||||
)
|
||||
.order_by(GovernanceRemediationDispatch.dispatched_at.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return result.scalar_one_or_none() is not None
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"governance_knowledge_degradation_review_lookup_failed_fail_open",
|
||||
error=str(exc),
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
async def _resolve_open_knowledge_degradation_events() -> int:
|
||||
"""KM stale ratio 回到門檻內時,收斂未解治理事件。"""
|
||||
try:
|
||||
async with get_db_context() as db:
|
||||
result = await db.execute(
|
||||
update(AiGovernanceEvent)
|
||||
.where(AiGovernanceEvent.event_type == "knowledge_degradation")
|
||||
.where(AiGovernanceEvent.resolved.is_(False))
|
||||
.values(resolved=True, resolved_at=now_taipei())
|
||||
.execution_options(synchronize_session=False)
|
||||
)
|
||||
resolved_count = int(result.rowcount or 0)
|
||||
if resolved_count:
|
||||
await db.commit()
|
||||
logger.info(
|
||||
"governance_knowledge_degradation_resolved",
|
||||
resolved_count=resolved_count,
|
||||
reason="stale_ratio_recovered",
|
||||
)
|
||||
return resolved_count
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"governance_knowledge_degradation_resolve_failed",
|
||||
error=str(exc),
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
async def _maybe_create_intake_dispatch(
|
||||
event_id: str,
|
||||
event_type: str,
|
||||
@@ -891,7 +976,40 @@ async def run_governance_loop(interval_seconds: int = 3600) -> None:
|
||||
agent = get_governance_agent()
|
||||
while True:
|
||||
try:
|
||||
await agent.run_self_check()
|
||||
if await _try_acquire_governance_self_check_lease(interval_seconds):
|
||||
await agent.run_self_check()
|
||||
else:
|
||||
logger.debug(
|
||||
"governance_self_check_cycle_skipped",
|
||||
reason="cycle_lease_held",
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning("governance_loop_error", error=str(e))
|
||||
await asyncio.sleep(interval_seconds)
|
||||
|
||||
|
||||
async def _try_acquire_governance_self_check_lease(interval_seconds: int) -> bool:
|
||||
"""跨 API Pod 的 self-check 週期租約。
|
||||
|
||||
這是週期 cooldown,不是 critical-section lock;取得後不主動 release。
|
||||
TTL 到期前其他 replica 只略過本輪,避免同一治理狀態被多個 Pod 寫成
|
||||
多筆事件、多張 Hermes KM 草稿。
|
||||
"""
|
||||
ttl = max(60, int(interval_seconds))
|
||||
try:
|
||||
from src.core.redis_client import get_redis
|
||||
|
||||
redis = get_redis()
|
||||
acquired = await redis.set(
|
||||
GOVERNANCE_SELF_CHECK_LEASE_KEY,
|
||||
"1",
|
||||
ex=ttl,
|
||||
nx=True,
|
||||
)
|
||||
return bool(acquired)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"governance_self_check_lease_unavailable_fail_open",
|
||||
error=str(exc),
|
||||
)
|
||||
return True
|
||||
|
||||
Reference in New Issue
Block a user