feat(governance): surface verification remediation queue
All checks were successful
Code Review / ai-code-review (push) Successful in 10s
CD Pipeline / tests (push) Successful in 1m0s
CD Pipeline / build-and-deploy (push) Successful in 3m25s
CD Pipeline / post-deploy-checks (push) Successful in 1m16s

This commit is contained in:
Your Name
2026-05-14 21:45:33 +08:00
parent f97127f704
commit aa63ae5eca
6 changed files with 326 additions and 3 deletions

View File

@@ -221,6 +221,7 @@ class Adr100SloStatusService:
"by_verification_result": [],
"by_failure_class": [],
},
"remediation_queue": _remediation_queue_payload([]),
}
return _build_verification_coverage_payload(
@@ -474,6 +475,7 @@ def _build_verification_coverage_payload(
_non_success_finding_payload(dict(raw))
for raw in recent_non_success_rows
]
remediation_queue = _remediation_queue_payload(recent_non_success)
return {
"schema_version": "adr100_verification_coverage_v1",
@@ -512,12 +514,17 @@ def _build_verification_coverage_payload(
"by_failure_class": _count_breakdown(
item["failure_class"] for item in recent_non_success
),
"by_remediation_status": _count_breakdown(
item["remediation_status"] for item in remediation_queue["items"]
),
},
"remediation_queue": remediation_queue,
}
def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]:
failure_class = _classify_non_success_failure(row)
remediation = _remediation_for_failure_class(failure_class)
return {
"auto_repair_id": str(row.get("auto_repair_id")),
"incident_id": str(row.get("incident_id")),
@@ -533,6 +540,10 @@ def _non_success_finding_payload(row: dict[str, Any]) -> dict[str, Any]:
"verification_result": str(row.get("verification_result") or "unknown"),
"failure_class": failure_class,
"next_step": _next_step_for_failure_class(failure_class),
"remediation_status": remediation["status"],
"remediation_action": remediation["action"],
"remediation_owner": remediation["owner"],
"remediation_reason": remediation["reason"],
"auto_error_excerpt": _short_text(row.get("auto_error"), 180),
"evidence_excerpt": _short_text(row.get("evidence_summary"), 180),
"auto_created_at": _iso(row.get("auto_created_at")),
@@ -560,6 +571,55 @@ def _classify_non_success_failure(row: dict[str, Any]) -> str:
return "verification_degraded"
def _remediation_for_failure_class(failure_class: str) -> dict[str, str]:
"""Map a non-success verification class to a read-only remediation work item.
This is dashboard triage metadata only. It does not auto-close incidents,
replay repairs, or approve write actions.
"""
if failure_class == "unsupported_action_scheme":
return {
"status": "ready_for_replay",
"action": "replay_with_supported_executor",
"owner": "auto_repair_executor",
"reason": "executor_gateway_available_after_t23",
}
if failure_class == "verifier_missing_promql":
return {
"status": "ready_for_reverify",
"action": "reverify_with_promql_template",
"owner": "post_execution_verifier",
"reason": "promql_template_available_after_t23",
}
if failure_class == "verifier_target_missing_pod":
return {
"status": "needs_target_mapping",
"action": "map_target_and_reverify",
"owner": "post_execution_verifier",
"reason": "verifier_target_missing",
}
if failure_class == "auto_repair_execution_failed":
return {
"status": "needs_playbook_ticket",
"action": "create_playbook_ticket",
"owner": "solver_or_operator",
"reason": "execution_failed_after_route_normalization",
}
if failure_class in {"verification_failed", "verification_timeout"}:
return {
"status": "manual_review",
"action": "escalate_verification_failure",
"owner": "sre_operator",
"reason": "verifier_returned_hard_failure",
}
return {
"status": "manual_review",
"action": "inspect_degraded_evidence",
"owner": "sre_operator",
"reason": "degraded_evidence_requires_human_context",
}
def _next_step_for_failure_class(failure_class: str) -> str:
if failure_class == "unsupported_action_scheme":
return "normalize_playbook_executor"
@@ -574,6 +634,57 @@ def _next_step_for_failure_class(failure_class: str) -> str:
return "review_degraded_verification"
def _remediation_queue_payload(recent_non_success: list[dict[str, Any]]) -> dict[str, Any]:
items: list[dict[str, Any]] = []
for item in recent_non_success:
items.append({
"work_item_id": (
f"verification:{item.get('incident_id')}:{item.get('auto_repair_id')}"
),
"incident_id": item.get("incident_id"),
"auto_repair_id": item.get("auto_repair_id"),
"alertname": item.get("alertname"),
"playbook_id": item.get("playbook_id"),
"failure_class": item.get("failure_class"),
"verification_result": item.get("verification_result"),
"remediation_status": item.get("remediation_status"),
"remediation_action": item.get("remediation_action"),
"remediation_owner": item.get("remediation_owner"),
"remediation_reason": item.get("remediation_reason"),
"source": "adr100_verification_coverage",
"auto_created_at": item.get("auto_created_at"),
"verification_collected_at": item.get("verification_collected_at"),
})
ready_for_ai = sum(
1 for item in items
if item.get("remediation_status") in {"ready_for_replay", "ready_for_reverify"}
)
needs_human = sum(
1 for item in items
if item.get("remediation_status") in {
"needs_target_mapping",
"needs_playbook_ticket",
"manual_review",
}
)
return {
"schema_version": "adr100_remediation_queue_v1",
"source": "recent_non_success_read_model",
"total": len(items),
"ready_for_ai": ready_for_ai,
"needs_human": needs_human,
"items": items,
"by_status": _count_breakdown(
item.get("remediation_status") for item in items
),
"by_action": _count_breakdown(
item.get("remediation_action") for item in items
),
}
def _count_breakdown(values: Any) -> list[dict[str, Any]]:
counts: dict[str, int] = {}
for value in values: