feat(governance): surface remediation dry run history
This commit is contained in:
@@ -118,3 +118,18 @@ async def dry_run_ai_slo_remediation(request: RemediationDryRunRequest) -> dict:
|
||||
)
|
||||
except RemediationNotFoundError as exc:
|
||||
raise HTTPException(status_code=404, detail="remediation_work_item_not_found") from exc
|
||||
|
||||
|
||||
@router.get("/ai/slo/remediation/history")
|
||||
async def list_ai_slo_remediation_history(
|
||||
limit: int = Query(50, ge=1, le=200),
|
||||
incident_id: str | None = Query(default=None, min_length=1),
|
||||
work_item_id: str | None = Query(default=None, min_length=1),
|
||||
) -> dict:
|
||||
"""List durable ADR-100 remediation dry-run history from alert_operation_log."""
|
||||
|
||||
return await get_adr100_remediation_service().history(
|
||||
limit=limit,
|
||||
incident_id=incident_id,
|
||||
work_item_id=work_item_id,
|
||||
)
|
||||
|
||||
@@ -112,6 +112,67 @@ class Adr100RemediationService:
|
||||
return await self._dry_run_replay(item, incident, checks)
|
||||
return await self._dry_run_reverify(item, incident, checks)
|
||||
|
||||
async def history(
|
||||
self,
|
||||
*,
|
||||
limit: int = 50,
|
||||
incident_id: str | None = None,
|
||||
work_item_id: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""Return durable dry-run history written by this remediation service."""
|
||||
|
||||
safe_limit = max(1, min(limit, 200))
|
||||
fetch_limit = min(max(safe_limit * 4, 50), 200)
|
||||
rows: list[Any] = []
|
||||
repo = self._alert_operation_log_repository
|
||||
if repo is None:
|
||||
from src.repositories.alert_operation_log_repository import (
|
||||
get_alert_operation_log_repository,
|
||||
)
|
||||
|
||||
repo = get_alert_operation_log_repository()
|
||||
|
||||
for event_type in ("PRE_FLIGHT_PASSED", "PRE_FLIGHT_FAILED"):
|
||||
try:
|
||||
batch, _total = await repo.list_recent(
|
||||
limit=fetch_limit,
|
||||
event_type=event_type,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
rows.extend(batch)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"adr100_remediation_history_fetch_failed",
|
||||
event_type=event_type,
|
||||
incident_id=incident_id,
|
||||
error=str(exc),
|
||||
)
|
||||
|
||||
rows.sort(key=_record_created_at, reverse=True)
|
||||
|
||||
items: list[dict[str, Any]] = []
|
||||
for row in rows:
|
||||
context = getattr(row, "context", None) or {}
|
||||
if context.get("schema_version") != "adr100_remediation_dry_run_history_v1":
|
||||
continue
|
||||
if work_item_id and context.get("work_item_id") != work_item_id:
|
||||
continue
|
||||
items.append(_history_item(row, context))
|
||||
if len(items) >= safe_limit:
|
||||
break
|
||||
|
||||
return {
|
||||
"schema_version": "adr100_remediation_history_v1",
|
||||
"total": len(items),
|
||||
"limit": safe_limit,
|
||||
"filters": {
|
||||
"incident_id": incident_id,
|
||||
"work_item_id": work_item_id,
|
||||
},
|
||||
"items": items,
|
||||
"by_work_item": _summarize_history_by_work_item(items),
|
||||
}
|
||||
|
||||
async def _find_work_item(self, work_item_id: str) -> dict[str, Any]:
|
||||
report = await self._slo_service.fetch_report()
|
||||
coverage = report.get("verification_coverage") or {}
|
||||
@@ -442,6 +503,66 @@ def _history_description(context: dict[str, Any]) -> str:
|
||||
)[:500]
|
||||
|
||||
|
||||
def _record_created_at(record: Any) -> str:
|
||||
value = getattr(record, "created_at", None)
|
||||
if hasattr(value, "isoformat"):
|
||||
return value.isoformat()
|
||||
return str(value or "")
|
||||
|
||||
|
||||
def _history_item(record: Any, context: dict[str, Any]) -> dict[str, Any]:
|
||||
route = context.get("mcp_route") or {}
|
||||
post_state = context.get("post_state_summary") or {}
|
||||
return {
|
||||
"id": str(getattr(record, "id", "")),
|
||||
"incident_id": getattr(record, "incident_id", None),
|
||||
"auto_repair_id": getattr(record, "auto_repair_id", None)
|
||||
or context.get("auto_repair_id"),
|
||||
"event_type": str(getattr(record, "event_type", "")),
|
||||
"actor": getattr(record, "actor", None),
|
||||
"success": getattr(record, "success", None),
|
||||
"created_at": _record_created_at(record),
|
||||
"work_item_id": context.get("work_item_id"),
|
||||
"playbook_id": context.get("playbook_id"),
|
||||
"alertname": context.get("alertname"),
|
||||
"mode": context.get("mode"),
|
||||
"allowed": context.get("allowed"),
|
||||
"executed": context.get("executed"),
|
||||
"safety_level": context.get("safety_level"),
|
||||
"verification_result_preview": context.get("verification_result_preview"),
|
||||
"tool_count": post_state.get("tool_count", 0),
|
||||
"tools": post_state.get("tools") or [],
|
||||
"agent_id": route.get("agent_id"),
|
||||
"tool_name": route.get("tool_name") or "current_state",
|
||||
"required_scope": route.get("required_scope"),
|
||||
"writes_incident_state": context.get("writes_incident_state"),
|
||||
"writes_auto_repair_result": context.get("writes_auto_repair_result"),
|
||||
"checks": context.get("checks") or [],
|
||||
}
|
||||
|
||||
|
||||
def _summarize_history_by_work_item(items: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
summary: dict[str, dict[str, Any]] = {}
|
||||
for item in items:
|
||||
key = str(item.get("work_item_id") or item.get("incident_id") or item.get("id"))
|
||||
if key not in summary:
|
||||
summary[key] = {
|
||||
"work_item_id": item.get("work_item_id"),
|
||||
"incident_id": item.get("incident_id"),
|
||||
"count": 0,
|
||||
"latest_at": item.get("created_at"),
|
||||
"latest_event_type": item.get("event_type"),
|
||||
"latest_success": item.get("success"),
|
||||
"latest_preview": item.get("verification_result_preview"),
|
||||
"latest_mode": item.get("mode"),
|
||||
"latest_agent_id": item.get("agent_id"),
|
||||
"latest_tool_name": item.get("tool_name"),
|
||||
"required_scope": item.get("required_scope"),
|
||||
}
|
||||
summary[key]["count"] += 1
|
||||
return list(summary.values())
|
||||
|
||||
|
||||
def _diagnostic_command_for_incident(incident: Incident) -> str:
|
||||
labels = _labels_for_incident(incident)
|
||||
host = str(labels.get("host") or labels.get("instance") or "{host}")
|
||||
|
||||
@@ -161,6 +161,40 @@ def _format_automation_quality_lines(quality: dict[str, object] | None) -> list[
|
||||
return lines
|
||||
|
||||
|
||||
def _format_remediation_history_lines(history: dict[str, object] | None) -> list[str]:
|
||||
if not history or int(history.get("total") or 0) <= 0:
|
||||
return []
|
||||
|
||||
items = history.get("items") if isinstance(history.get("items"), list) else []
|
||||
latest = items[0] if items and isinstance(items[0], dict) else {}
|
||||
agent = latest.get("agent_id") or "unknown_agent"
|
||||
tool = latest.get("tool_name") or "current_state"
|
||||
scope = latest.get("required_scope") or "unknown"
|
||||
writes_incident = latest.get("writes_incident_state")
|
||||
writes_auto_repair = latest.get("writes_auto_repair_result")
|
||||
|
||||
return [
|
||||
"",
|
||||
"🧪 <b>ADR-100 補救試跑</b>",
|
||||
f"歷史: <code>{int(history.get('total') or 0)}</code> 次",
|
||||
(
|
||||
"上次: "
|
||||
f"<code>{html.escape(str(latest.get('mode') or 'unknown'))}</code> / "
|
||||
f"<code>{html.escape(str(latest.get('verification_result_preview') or 'unknown'))}</code>"
|
||||
),
|
||||
(
|
||||
"MCP: "
|
||||
f"<code>{html.escape(str(agent))}/{html.escape(str(tool))}</code> / "
|
||||
f"<code>{html.escape(str(scope))}</code>"
|
||||
),
|
||||
(
|
||||
"寫入: "
|
||||
f"incident <code>{html.escape(str(writes_incident))}</code> / "
|
||||
f"auto-repair <code>{html.escape(str(writes_auto_repair))}</code>"
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _sanitize_telegram_error(text: str) -> str:
|
||||
"""遮蔽 Telegram Bot URL 中的 token,避免例外字串污染 log / trace。"""
|
||||
return _TELEGRAM_BOT_URL_RE.sub(r"\1<redacted>", text)
|
||||
@@ -5276,6 +5310,23 @@ class TelegramGateway:
|
||||
+ html.escape(", ".join(mismatch_codes[:4]))
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.adr100_remediation_service import (
|
||||
get_adr100_remediation_service,
|
||||
)
|
||||
|
||||
remediation_history = await get_adr100_remediation_service().history(
|
||||
limit=5,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
lines += _format_remediation_history_lines(remediation_history)
|
||||
except Exception as remediation_exc:
|
||||
logger.warning(
|
||||
"incident_detail_remediation_history_summary_failed",
|
||||
incident_id=incident_id,
|
||||
error=str(remediation_exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.awooop_truth_chain_service import fetch_truth_chain
|
||||
|
||||
@@ -5298,6 +5349,23 @@ class TelegramGateway:
|
||||
error=str(truth_exc),
|
||||
)
|
||||
|
||||
try:
|
||||
from src.services.adr100_remediation_service import (
|
||||
get_adr100_remediation_service,
|
||||
)
|
||||
|
||||
remediation_history = await get_adr100_remediation_service().history(
|
||||
limit=5,
|
||||
incident_id=incident_id,
|
||||
)
|
||||
lines += _format_remediation_history_lines(remediation_history)
|
||||
except Exception as remediation_exc:
|
||||
logger.warning(
|
||||
"incident_history_remediation_summary_failed",
|
||||
incident_id=incident_id,
|
||||
error=str(remediation_exc),
|
||||
)
|
||||
|
||||
await self.send_notification("\n".join(lines))
|
||||
|
||||
except Exception as e:
|
||||
|
||||
Reference in New Issue
Block a user