feat(governance): surface remediation dry run history
All checks were successful
Code Review / ai-code-review (push) Successful in 10s
CD Pipeline / tests (push) Successful in 1m1s
CD Pipeline / build-and-deploy (push) Successful in 3m45s
CD Pipeline / post-deploy-checks (push) Successful in 1m39s

This commit is contained in:
Your Name
2026-05-14 22:56:51 +08:00
parent 53cd7f9d66
commit 392cfb9025
8 changed files with 406 additions and 3 deletions

View File

@@ -118,3 +118,18 @@ async def dry_run_ai_slo_remediation(request: RemediationDryRunRequest) -> dict:
)
except RemediationNotFoundError as exc:
raise HTTPException(status_code=404, detail="remediation_work_item_not_found") from exc
@router.get("/ai/slo/remediation/history")
async def list_ai_slo_remediation_history(
limit: int = Query(50, ge=1, le=200),
incident_id: str | None = Query(default=None, min_length=1),
work_item_id: str | None = Query(default=None, min_length=1),
) -> dict:
"""List durable ADR-100 remediation dry-run history from alert_operation_log."""
return await get_adr100_remediation_service().history(
limit=limit,
incident_id=incident_id,
work_item_id=work_item_id,
)

View File

@@ -112,6 +112,67 @@ class Adr100RemediationService:
return await self._dry_run_replay(item, incident, checks)
return await self._dry_run_reverify(item, incident, checks)
async def history(
self,
*,
limit: int = 50,
incident_id: str | None = None,
work_item_id: str | None = None,
) -> dict[str, Any]:
"""Return durable dry-run history written by this remediation service."""
safe_limit = max(1, min(limit, 200))
fetch_limit = min(max(safe_limit * 4, 50), 200)
rows: list[Any] = []
repo = self._alert_operation_log_repository
if repo is None:
from src.repositories.alert_operation_log_repository import (
get_alert_operation_log_repository,
)
repo = get_alert_operation_log_repository()
for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED"):
try:
batch, _total = await repo.list_recent(
limit=fetch_limit,
event_type=event_type,
incident_id=incident_id,
)
rows.extend(batch)
except Exception as exc:
logger.warning(
"adr100_remediation_history_fetch_failed",
event_type=event_type,
incident_id=incident_id,
error=str(exc),
)
rows.sort(key=_record_created_at, reverse=True)
items: list[dict[str, Any]] = []
for row in rows:
context = getattr(row, "context", None) or {}
if context.get("schema_version") != "adr100_remediation_dry_run_history_v1":
continue
if work_item_id and context.get("work_item_id") != work_item_id:
continue
items.append(_history_item(row, context))
if len(items) >= safe_limit:
break
return {
"schema_version": "adr100_remediation_history_v1",
"total": len(items),
"limit": safe_limit,
"filters": {
"incident_id": incident_id,
"work_item_id": work_item_id,
},
"items": items,
"by_work_item": _summarize_history_by_work_item(items),
}
async def _find_work_item(self, work_item_id: str) -> dict[str, Any]:
report = await self._slo_service.fetch_report()
coverage = report.get("verification_coverage") or {}
@@ -442,6 +503,66 @@ def _history_description(context: dict[str, Any]) -> str:
)[:500]
def _record_created_at(record: Any) -> str:
value = getattr(record, "created_at", None)
if hasattr(value, "isoformat"):
return value.isoformat()
return str(value or "")
def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]:
route = context.get("mcp_route") or {}
post_state = context.get("post_state_summary") or {}
return {
"id": str(getattr(record, "id", "")),
"incident_id": getattr(record, "incident_id", None),
"auto_repair_id": getattr(record, "auto_repair_id", None)
or context.get("auto_repair_id"),
"event_type": str(getattr(record, "event_type", "")),
"actor": getattr(record, "actor", None),
"success": getattr(record, "success", None),
"created_at": _record_created_at(record),
"work_item_id": context.get("work_item_id"),
"playbook_id": context.get("playbook_id"),
"alertname": context.get("alertname"),
"mode": context.get("mode"),
"allowed": context.get("allowed"),
"executed": context.get("executed"),
"safety_level": context.get("safety_level"),
"verification_result_preview": context.get("verification_result_preview"),
"tool_count": post_state.get("tool_count", 0),
"tools": post_state.get("tools") or [],
"agent_id": route.get("agent_id"),
"tool_name": route.get("tool_name") or "current_state",
"required_scope": route.get("required_scope"),
"writes_incident_state": context.get("writes_incident_state"),
"writes_auto_repair_result": context.get("writes_auto_repair_result"),
"checks": context.get("checks") or [],
}
def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
summary: dict[str, dict[str, Any]] = {}
for item in items:
key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id"))
if key not in summary:
summary[key] = {
"work_item_id": item.get("work_item_id"),
"incident_id": item.get("incident_id"),
"count": 0,
"latest_at": item.get("created_at"),
"latest_event_type": item.get("event_type"),
"latest_success": item.get("success"),
"latest_preview": item.get("verification_result_preview"),
"latest_mode": item.get("mode"),
"latest_agent_id": item.get("agent_id"),
"latest_tool_name": item.get("tool_name"),
"required_scope": item.get("required_scope"),
}
summary[key]["count"] += 1
return list(summary.values())
def _diagnostic_command_for_incident(incident: Incident) -> str:
labels = _labels_for_incident(incident)
host = str(labels.get("host") or labels.get("instance") or "{host}")

View File

@@ -161,6 +161,40 @@ def _format_automation_quality_lines(quality: dict[str, object] | None) -> list[
return lines
def _format_remediation_history_lines(history: dict[str, object] | None) -> list[str]:
if not history or int(history.get("total") or 0) <= 0:
return []
items = history.get("items") if isinstance(history.get("items"), list) else []
latest = items[0] if items and isinstance(items[0], dict) else {}
agent = latest.get("agent_id") or "unknown_agent"
tool = latest.get("tool_name") or "current_state"
scope = latest.get("required_scope") or "unknown"
writes_incident = latest.get("writes_incident_state")
writes_auto_repair = latest.get("writes_auto_repair_result")
return [
"",
"🧪 <b>ADR-100 補救試跑</b>",
f"歷史: <code>{int(history.get('total') or 0)}</code> 次",
(
"上次: "
f"<code>{html.escape(str(latest.get('mode') or 'unknown'))}</code> / "
f"<code>{html.escape(str(latest.get('verification_result_preview') or 'unknown'))}</code>"
),
(
"MCP: "
f"<code>{html.escape(str(agent))}/{html.escape(str(tool))}</code> / "
f"<code>{html.escape(str(scope))}</code>"
),
(
"寫入: "
f"incident <code>{html.escape(str(writes_incident))}</code> / "
f"auto-repair <code>{html.escape(str(writes_auto_repair))}</code>"
),
]
def _sanitize_telegram_error(text: str) -> str:
"""遮蔽 Telegram Bot URL 中的 token避免例外字串污染 log / trace。"""
return _TELEGRAM_BOT_URL_RE.sub(r"\1<redacted>", text)
@@ -5276,6 +5310,23 @@ class TelegramGateway:
+ html.escape(", ".join(mismatch_codes[:4]))
)
try:
from src.services.adr100_remediation_service import (
get_adr100_remediation_service,
)
remediation_history = await get_adr100_remediation_service().history(
limit=5,
incident_id=incident_id,
)
lines += _format_remediation_history_lines(remediation_history)
except Exception as remediation_exc:
logger.warning(
"incident_detail_remediation_history_summary_failed",
incident_id=incident_id,
error=str(remediation_exc),
)
try:
from src.services.awooop_truth_chain_service import fetch_truth_chain
@@ -5298,6 +5349,23 @@ class TelegramGateway:
error=str(truth_exc),
)
try:
from src.services.adr100_remediation_service import (
get_adr100_remediation_service,
)
remediation_history = await get_adr100_remediation_service().history(
limit=5,
incident_id=incident_id,
)
lines += _format_remediation_history_lines(remediation_history)
except Exception as remediation_exc:
logger.warning(
"incident_history_remediation_summary_failed",
incident_id=incident_id,
error=str(remediation_exc),
)
await self.send_notification("\n".join(lines))
except Exception as e: