abc512a7b3
Some checks failed
CD Pipeline / workflow-shape (push) Has been cancelled
CD Pipeline / cancel-stale-cd (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
2672 lines
101 KiB
Python
2672 lines
101 KiB
Python
"""AwoooP read-only truth chain aggregation.
|
|
|
|
T0 only: this service does not mutate incident, approval, execution, or channel
|
|
state. It stitches existing durable records into one operator-facing status so
|
|
Telegram cards can be audited without guessing which subsystem owns the truth.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
from datetime import UTC, date, datetime, timedelta
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
from time import perf_counter, time
|
|
from typing import Any
|
|
from uuid import UUID
|
|
|
|
import structlog
|
|
from sqlalchemy import text
|
|
|
|
from src.core.config import settings
|
|
from src.db.base import get_db_context
|
|
from src.services.operator_outcome import build_operator_outcome
|
|
from src.services.awooop_ansible_check_mode_service import detect_ansible_transport_blockers
|
|
from src.services.awooop_ansible_audit_service import build_ansible_truth
|
|
from src.services.drift_repeat_state import build_drift_repeat_state
|
|
from src.services.operator_summary_cache import (
|
|
get_cached_operator_summary_async,
|
|
store_operator_summary_async,
|
|
)
|
|
|
|
logger = structlog.get_logger(__name__)
|
|
|
|
_MAX_ROWS = 100
|
|
_INBOUND_LOOKUP_BRANCH_LIMIT = int(
|
|
os.getenv("AWOOOP_TRUTH_CHAIN_INBOUND_BRANCH_LIMIT", "25")
|
|
)
|
|
_JSON_TEXT_FIELDS = {"gate_result", "source_envelope"}
|
|
_QUALITY_SUMMARY_CACHE_TTL_SECONDS = int(
|
|
os.getenv("AWOOOP_QUALITY_SUMMARY_CACHE_TTL_SECONDS", "30")
|
|
)
|
|
_QUALITY_SUMMARY_OBSERVATIONS: dict[str, dict[str, Any]] = {}
|
|
|
|
|
|
def record_quality_summary_observation(
|
|
*,
|
|
project_id: str,
|
|
hours: int,
|
|
limit: int,
|
|
cache_status: str,
|
|
success: bool,
|
|
duration_seconds: float,
|
|
error: str | None = None,
|
|
) -> None:
|
|
normalized_project_id = project_id or "awoooi"
|
|
normalized_cache_status = cache_status or "unknown"
|
|
key = "|".join([
|
|
normalized_project_id,
|
|
str(int(hours)),
|
|
str(int(limit)),
|
|
normalized_cache_status,
|
|
"success" if success else "failed",
|
|
])
|
|
_QUALITY_SUMMARY_OBSERVATIONS[key] = {
|
|
"project_id": normalized_project_id,
|
|
"hours": int(hours),
|
|
"limit": int(limit),
|
|
"cache_status": normalized_cache_status,
|
|
"success": bool(success),
|
|
"duration_seconds": max(0.0, float(duration_seconds)),
|
|
"observed_at": time(),
|
|
"error": str(error)[:160] if error else None,
|
|
}
|
|
|
|
|
|
def get_quality_summary_observations() -> list[dict[str, Any]]:
|
|
return [
|
|
dict(observation)
|
|
for observation in sorted(
|
|
_QUALITY_SUMMARY_OBSERVATIONS.values(),
|
|
key=lambda item: (
|
|
str(item.get("project_id") or ""),
|
|
int(item.get("hours") or 0),
|
|
int(item.get("limit") or 0),
|
|
str(item.get("cache_status") or ""),
|
|
bool(item.get("success")),
|
|
),
|
|
)
|
|
]
|
|
|
|
|
|
def _clean(value: Any) -> Any:
|
|
"""Convert DB values into JSON-friendly primitives."""
|
|
if isinstance(value, UUID):
|
|
return str(value)
|
|
if isinstance(value, (datetime, date)):
|
|
return value.isoformat()
|
|
if isinstance(value, Decimal):
|
|
return float(value)
|
|
if isinstance(value, dict):
|
|
return {str(k): _clean(v) for k, v in value.items()}
|
|
if isinstance(value, list):
|
|
return [_clean(v) for v in value]
|
|
return value
|
|
|
|
|
|
def _clean_row(row: Any) -> dict[str, Any]:
|
|
cleaned: dict[str, Any] = {}
|
|
for key, value in dict(row).items():
|
|
if key in _JSON_TEXT_FIELDS and isinstance(value, str):
|
|
try:
|
|
value = json.loads(value)
|
|
except json.JSONDecodeError:
|
|
pass
|
|
cleaned[key] = _clean(value)
|
|
return cleaned
|
|
|
|
|
|
async def _fetch_all(db: Any, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]:
|
|
result = await db.execute(text(sql), params)
|
|
return [_clean_row(row) for row in result.mappings().all()]
|
|
|
|
|
|
async def _fetch_one(
|
|
db: Any,
|
|
sql: str,
|
|
params: dict[str, Any],
|
|
) -> dict[str, Any] | None:
|
|
result = await db.execute(text(sql), params)
|
|
row = result.mappings().first()
|
|
return _clean_row(row) if row else None
|
|
|
|
|
|
def _inbound_lookup_branch_limit(limit: int) -> int:
|
|
return max(1, min(int(limit), _INBOUND_LOOKUP_BRANCH_LIMIT))
|
|
|
|
|
|
async def _fetch_inbound_conversation_event_rows(
|
|
db: Any,
|
|
*,
|
|
project_id: str,
|
|
source_id: str,
|
|
fingerprint_needle: str,
|
|
fingerprint_value: str,
|
|
limit: int = _MAX_ROWS,
|
|
) -> list[dict[str, Any]]:
|
|
columns = """
|
|
event_id,
|
|
project_id,
|
|
channel_type,
|
|
provider_event_id,
|
|
platform_subject_id,
|
|
channel_user_id,
|
|
channel_chat_id,
|
|
run_id,
|
|
content_type,
|
|
content_hash,
|
|
content_preview,
|
|
content_redacted,
|
|
redaction_version,
|
|
source_envelope,
|
|
attachment_sha256,
|
|
is_duplicate,
|
|
provider_ts,
|
|
received_at
|
|
"""
|
|
branch_limit = _inbound_lookup_branch_limit(limit)
|
|
rows_by_id: dict[str, dict[str, Any]] = {}
|
|
source_ref_paths = (
|
|
"{source_refs,event_ids}",
|
|
"{source_refs,incident_ids}",
|
|
"{source_refs,approval_ids}",
|
|
"{source_refs,alert_ids}",
|
|
"{source_refs,sentry_issue_ids}",
|
|
"{source_refs,signoz_alerts}",
|
|
)
|
|
|
|
async def fetch_branch(where_sql: str, params: dict[str, Any]) -> None:
|
|
if len(rows_by_id) >= limit:
|
|
return
|
|
rows = await _fetch_all(
|
|
db,
|
|
f"""
|
|
SELECT
|
|
{columns}
|
|
FROM awooop_conversation_event
|
|
WHERE project_id = :project_id
|
|
AND ({where_sql})
|
|
ORDER BY received_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{
|
|
"project_id": project_id,
|
|
"source_id": source_id,
|
|
"limit": min(branch_limit, limit - len(rows_by_id)),
|
|
**params,
|
|
},
|
|
)
|
|
for row in rows:
|
|
event_id = str(row.get("event_id") or "")
|
|
if event_id:
|
|
rows_by_id.setdefault(event_id, row)
|
|
|
|
await fetch_branch("run_id::text = :source_id", {})
|
|
await fetch_branch("provider_event_id = :source_id", {})
|
|
for source_ref_path in source_ref_paths:
|
|
await fetch_branch(
|
|
f"coalesce(source_envelope #> '{source_ref_path}', '[]'::jsonb) ? :source_id",
|
|
{},
|
|
)
|
|
|
|
if fingerprint_value:
|
|
await fetch_branch(
|
|
"coalesce(source_envelope #> '{source_refs,fingerprints}', '[]'::jsonb) ? :fingerprint_value",
|
|
{"fingerprint_value": fingerprint_value},
|
|
)
|
|
|
|
if len(rows_by_id) < branch_limit:
|
|
await fetch_branch(
|
|
"content_preview ILIKE :source_needle",
|
|
{"source_needle": f"%{source_id}%"},
|
|
)
|
|
if fingerprint_needle and len(rows_by_id) < branch_limit:
|
|
await fetch_branch(
|
|
"provider_event_id ILIKE :fingerprint_needle OR content_preview ILIKE :fingerprint_needle",
|
|
{"fingerprint_needle": fingerprint_needle},
|
|
)
|
|
|
|
return sorted(
|
|
rows_by_id.values(),
|
|
key=lambda row: str(row.get("received_at") or ""),
|
|
reverse=True,
|
|
)[:limit]
|
|
|
|
|
|
def _source_type(source_id: str, incident: dict[str, Any] | None, drift: dict[str, Any] | None) -> str:
|
|
if incident is not None:
|
|
return "incident"
|
|
if drift is not None:
|
|
return "drift_report"
|
|
try:
|
|
UUID(source_id)
|
|
except ValueError:
|
|
return "unknown"
|
|
return "run"
|
|
|
|
|
|
def _failed_mcp_tools(evidence_rows: list[dict[str, Any]]) -> list[str]:
|
|
failed: set[str] = set()
|
|
for evidence in evidence_rows:
|
|
mcp_health = evidence.get("mcp_health")
|
|
if isinstance(mcp_health, dict):
|
|
failed.update(str(tool) for tool, ok in mcp_health.items() if ok is False)
|
|
return sorted(failed)
|
|
|
|
|
|
def _approval_ids(approvals: list[dict[str, Any]]) -> list[str]:
|
|
return [str(row["id"]) for row in approvals if row.get("id")]
|
|
|
|
|
|
def _operation_ids(automation_ops: list[dict[str, Any]]) -> list[str]:
|
|
return [str(row["op_id"]) for row in automation_ops if row.get("op_id")]
|
|
|
|
|
|
def _auto_repair_ids(auto_repair_executions: list[dict[str, Any]]) -> list[str]:
|
|
return [str(row["id"]) for row in auto_repair_executions if row.get("id")]
|
|
|
|
|
|
def _incident_fingerprints(incident: dict[str, Any] | None) -> list[str]:
|
|
"""Extract durable alert fingerprints from incident signals, when present."""
|
|
if not incident:
|
|
return []
|
|
signals = incident.get("signals")
|
|
if not isinstance(signals, list):
|
|
return []
|
|
|
|
fingerprints: set[str] = set()
|
|
for signal in signals:
|
|
if not isinstance(signal, dict):
|
|
continue
|
|
direct = signal.get("fingerprint")
|
|
if direct:
|
|
fingerprints.add(str(direct))
|
|
labels = signal.get("labels")
|
|
if isinstance(labels, dict):
|
|
value = labels.get("fingerprint")
|
|
if value:
|
|
fingerprints.add(str(value))
|
|
return sorted(fingerprints)
|
|
|
|
|
|
def _looks_like_no_action(value: Any) -> bool:
|
|
text = str(value or "").upper()
|
|
return (
|
|
"NO_ACTION" in text
|
|
or "NO-ACTION" in text
|
|
or "NOACTION" in text
|
|
or text.startswith("OBSERVE")
|
|
or text.startswith("INVESTIGATE")
|
|
)
|
|
|
|
|
|
def _approval_has_no_action(approvals: list[dict[str, Any]]) -> bool:
|
|
return any(_looks_like_no_action(row.get("action")) for row in approvals)
|
|
|
|
|
|
def _approval_suppresses_repair_execution(approvals: list[dict[str, Any]]) -> bool:
|
|
for row in approvals:
|
|
metadata = row.get("extra_metadata")
|
|
if not isinstance(metadata, dict):
|
|
continue
|
|
execution_kind = str(metadata.get("execution_kind") or "").lower()
|
|
if execution_kind in {"no_action", "diagnostic", "parse_failed", "unsupported_action"}:
|
|
return True
|
|
if metadata.get("repair_executed") is False and not execution_kind:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _is_no_action_operation(row: dict[str, Any]) -> bool:
|
|
"""Return true for durable audit rows that represent observation, not repair."""
|
|
if str(row.get("operation_type") or "") != "playbook_executed":
|
|
return False
|
|
return any(
|
|
_looks_like_no_action(row.get(key))
|
|
for key in (
|
|
"input_action",
|
|
"output_action",
|
|
"output_reason",
|
|
"output_not_used_reason",
|
|
)
|
|
)
|
|
|
|
|
|
def _is_audit_only_operation(row: dict[str, Any]) -> bool:
|
|
operation_type = str(row.get("operation_type") or "")
|
|
status = str(row.get("status") or "").lower()
|
|
if status == "dry_run":
|
|
return True
|
|
return operation_type in {
|
|
"ansible_candidate_matched",
|
|
"ansible_execution_skipped",
|
|
}
|
|
|
|
|
|
def _effective_execution_ops(automation_ops: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
return [
|
|
row
|
|
for row in automation_ops
|
|
if not _is_no_action_operation(row) and not _is_audit_only_operation(row)
|
|
]
|
|
|
|
|
|
def build_incident_reconciliation(
|
|
*,
|
|
incident: dict[str, Any] | None,
|
|
approvals: list[dict[str, Any]],
|
|
evidence_rows: list[dict[str, Any]],
|
|
automation_ops: list[dict[str, Any]],
|
|
auto_repair_executions: list[dict[str, Any]] | None = None,
|
|
timeline_events: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
"""Build a read-only consistency report across incident lifecycle tables."""
|
|
if incident is None:
|
|
return {
|
|
"schema_version": "incident_reconciliation_v1",
|
|
"applicable": False,
|
|
"consistency_status": "not_applicable",
|
|
"operator_next_state": "not_applicable",
|
|
"facts": {},
|
|
"mismatches": [],
|
|
}
|
|
|
|
incident_status = str(incident.get("status") or "unknown").upper()
|
|
incident_closed = incident_status in {"RESOLVED", "CLOSED"}
|
|
latest_approval = approvals[0] if approvals else None
|
|
approval_status = str((latest_approval or {}).get("status") or "none").upper()
|
|
approval_action = str((latest_approval or {}).get("action") or "")
|
|
approval_resolved = bool((latest_approval or {}).get("resolved_at"))
|
|
attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows)
|
|
succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows)
|
|
repair_rows = auto_repair_executions or []
|
|
approval_suppressed = _approval_suppresses_repair_execution(approvals)
|
|
effective_ops = [] if approval_suppressed else _effective_execution_ops(automation_ops)
|
|
executed_ops = [
|
|
row
|
|
for row in effective_ops
|
|
if str(row.get("status") or "").lower()
|
|
in {"success", "completed", "executed"}
|
|
]
|
|
successful_repairs = [row for row in repair_rows if row.get("success") is True]
|
|
effective_execution_count = len(executed_ops) + len(successful_repairs)
|
|
latest_verification = _latest_verification_result(incident, evidence_rows)
|
|
mismatches: list[dict[str, Any]] = []
|
|
|
|
def add(code: str, severity: str, message: str) -> None:
|
|
mismatches.append({
|
|
"code": code,
|
|
"severity": severity,
|
|
"message": message,
|
|
})
|
|
|
|
if (
|
|
latest_approval
|
|
and not incident_closed
|
|
and (approval_resolved or approval_status in {"APPROVED", "REJECTED"})
|
|
):
|
|
if effective_execution_count:
|
|
add(
|
|
"incident_open_after_successful_execution",
|
|
"high",
|
|
"Execution evidence exists while the incident is still open.",
|
|
)
|
|
else:
|
|
add(
|
|
"incident_open_after_approval_resolved",
|
|
"high",
|
|
"Approval reached a terminal state while the incident is still open.",
|
|
)
|
|
|
|
if approval_status == "APPROVED" and not effective_execution_count:
|
|
add(
|
|
"approval_approved_without_execution_record",
|
|
"high",
|
|
"Approval is approved but automation_operation_log has no linked execution record.",
|
|
)
|
|
|
|
if (
|
|
approval_status == "APPROVED"
|
|
and "NO_ACTION" in approval_action.upper()
|
|
and not effective_execution_count
|
|
):
|
|
add(
|
|
"approval_no_action_without_execution",
|
|
"high",
|
|
"Approval resolved to NO_ACTION and no executor produced a successful operation.",
|
|
)
|
|
|
|
if successful_repairs and str(latest_verification or "").lower() == "degraded":
|
|
add(
|
|
"verification_degraded_after_auto_repair",
|
|
"medium",
|
|
"Auto-repair succeeded, but post-execution verification reported degraded evidence.",
|
|
)
|
|
|
|
if attempted > 0 and succeeded == 0:
|
|
add(
|
|
"evidence_all_sensors_failed",
|
|
"medium",
|
|
"Evidence collection attempted sensors but none succeeded.",
|
|
)
|
|
|
|
if latest_approval and not timeline_events:
|
|
add(
|
|
"timeline_missing_for_approval",
|
|
"medium",
|
|
"Approval exists but timeline_events has no linked lifecycle entries.",
|
|
)
|
|
|
|
high_count = sum(1 for row in mismatches if row["severity"] == "high")
|
|
medium_count = sum(1 for row in mismatches if row["severity"] == "medium")
|
|
if high_count:
|
|
consistency_status = "blocked"
|
|
operator_next_state = "manual_required"
|
|
elif medium_count:
|
|
consistency_status = "degraded"
|
|
operator_next_state = "investigate"
|
|
else:
|
|
consistency_status = "consistent"
|
|
operator_next_state = "continue"
|
|
|
|
return {
|
|
"schema_version": "incident_reconciliation_v1",
|
|
"applicable": True,
|
|
"consistency_status": consistency_status,
|
|
"operator_next_state": operator_next_state,
|
|
"facts": {
|
|
"incident_id": incident.get("incident_id"),
|
|
"incident_status": incident_status,
|
|
"incident_closed": incident_closed,
|
|
"latest_approval_id": (latest_approval or {}).get("id"),
|
|
"latest_approval_status": approval_status,
|
|
"latest_approval_action": approval_action,
|
|
"approval_resolved": approval_resolved,
|
|
"evidence_records": len(evidence_rows),
|
|
"sensors_attempted": attempted,
|
|
"sensors_succeeded": succeeded,
|
|
"automation_operation_records": len(automation_ops),
|
|
"executed_operation_records": len(executed_ops),
|
|
"auto_repair_execution_records": len(repair_rows),
|
|
"successful_auto_repair_records": len(successful_repairs),
|
|
"effective_execution_records": effective_execution_count,
|
|
"latest_verification_result": latest_verification,
|
|
"timeline_events": len(timeline_events),
|
|
},
|
|
"mismatches": mismatches,
|
|
}
|
|
|
|
|
|
def _truth_status(
|
|
*,
|
|
incident: dict[str, Any] | None,
|
|
approvals: list[dict[str, Any]],
|
|
evidence_rows: list[dict[str, Any]],
|
|
automation_ops: list[dict[str, Any]],
|
|
drift: dict[str, Any] | None,
|
|
drift_repeat_count: int,
|
|
gateway_mcp_total: int,
|
|
legacy_mcp_total: int,
|
|
outbound_visible_total: int,
|
|
inbound_visible_total: int = 0,
|
|
auto_repair_executions: list[dict[str, Any]] | None = None,
|
|
) -> dict[str, Any]:
|
|
"""Derive the current operator-visible truth-chain stage."""
|
|
blockers: list[str] = []
|
|
needs_human = False
|
|
stage = "not_found"
|
|
stage_status = "missing"
|
|
|
|
if drift is not None:
|
|
stage = "dedup_or_repeat_updated" if drift_repeat_count > 1 else "received"
|
|
stage_status = str(drift.get("status") or "unknown")
|
|
interpretation = drift.get("interpretation") or {}
|
|
confidence = interpretation.get("confidence") if isinstance(interpretation, dict) else None
|
|
if stage_status == "pending":
|
|
needs_human = True
|
|
blockers.append("drift_report_pending_without_resolution")
|
|
if confidence in (0, 0.0):
|
|
blockers.append("drift_ai_confidence_zero")
|
|
|
|
if incident is None and drift is None and inbound_visible_total > 0:
|
|
stage = "inbound_received"
|
|
stage_status = "observed"
|
|
|
|
if incident is not None:
|
|
incident_status = str(incident.get("status") or "unknown")
|
|
repair_rows = auto_repair_executions or []
|
|
approval_suppressed = _approval_suppresses_repair_execution(approvals)
|
|
effective_ops = [] if approval_suppressed else _effective_execution_ops(automation_ops)
|
|
has_execution_records = bool(effective_ops or repair_rows)
|
|
latest_verification = str(
|
|
_latest_verification_result(incident, evidence_rows) or ""
|
|
).lower()
|
|
stage = "received"
|
|
stage_status = incident_status.lower()
|
|
if incident_status in {"RESOLVED", "CLOSED"}:
|
|
stage = "resolved"
|
|
stage_status = "success"
|
|
elif evidence_rows:
|
|
attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows)
|
|
succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows)
|
|
if attempted > 0 and succeeded == 0:
|
|
stage = "evidence_degraded"
|
|
stage_status = "warning"
|
|
needs_human = True
|
|
blockers.append("all_evidence_sensors_failed")
|
|
else:
|
|
stage = "evidence_collected"
|
|
stage_status = "completed"
|
|
|
|
approval_statuses = {str(row.get("status") or "").upper() for row in approvals}
|
|
approval_actions = " ".join(str(row.get("action") or "") for row in approvals).upper()
|
|
approval_no_action = _approval_has_no_action(approvals)
|
|
if any(status in {"PENDING", "WAITING_APPROVAL"} for status in approval_statuses):
|
|
stage = "approval_required"
|
|
stage_status = "waiting"
|
|
needs_human = True
|
|
blockers.append("pending_human_approval")
|
|
elif "EXPIRED" in approval_statuses and not has_execution_records:
|
|
stage = "approval_expired"
|
|
stage_status = "expired"
|
|
needs_human = True
|
|
blockers.append("approval_expired_without_operator_decision")
|
|
elif "REJECTED" in approval_statuses and not has_execution_records:
|
|
stage = "approval_rejected"
|
|
stage_status = "closed"
|
|
elif "EXECUTION_FAILED" in approval_statuses and not has_execution_records:
|
|
stage = "execution_failed"
|
|
stage_status = "error"
|
|
needs_human = True
|
|
blockers.append("approval_execution_failed_without_execution_record")
|
|
elif not has_execution_records and (approval_no_action or "NO_ACTION" in approval_actions):
|
|
if approval_statuses:
|
|
stage = "manual_required"
|
|
stage_status = "blocked"
|
|
needs_human = True
|
|
blockers.append("approval_resolved_no_action_without_execution")
|
|
elif "APPROVED" in approval_statuses and not has_execution_records:
|
|
stage = "execution_missing"
|
|
stage_status = "blocked"
|
|
needs_human = True
|
|
blockers.append("approved_without_execution_record")
|
|
|
|
op_statuses = {str(row.get("status") or "").lower() for row in effective_ops}
|
|
repair_successes = {row.get("success") for row in repair_rows}
|
|
execution_succeeded = (op_statuses & {"success", "completed"}) or True in repair_successes
|
|
execution_failed = (op_statuses & {"failed", "error"}) or False in repair_successes
|
|
if op_statuses or repair_successes:
|
|
if execution_succeeded:
|
|
stage = "execution_succeeded"
|
|
stage_status = "success"
|
|
elif execution_failed:
|
|
stage = "execution_failed"
|
|
stage_status = "error"
|
|
needs_human = True
|
|
else:
|
|
stage = "execution_started"
|
|
stage_status = "running"
|
|
|
|
if incident_status == "INVESTIGATING" and approvals:
|
|
if execution_succeeded:
|
|
blockers.append("incident_open_after_successful_execution")
|
|
if latest_verification != "success":
|
|
needs_human = True
|
|
elif not has_execution_records:
|
|
blockers.append("incident_still_investigating_after_approval")
|
|
|
|
if gateway_mcp_total == 0:
|
|
blockers.append("awooop_mcp_gateway_audit_empty")
|
|
if legacy_mcp_total == 0 and incident is not None:
|
|
blockers.append("legacy_mcp_audit_missing")
|
|
if outbound_visible_total == 0:
|
|
blockers.append("outbound_mirror_not_visible_for_source")
|
|
|
|
return {
|
|
"current_stage": stage,
|
|
"stage_status": stage_status,
|
|
"needs_human": needs_human,
|
|
"blockers": blockers,
|
|
}
|
|
|
|
|
|
def _latest_verification_result(
|
|
incident: dict[str, Any] | None,
|
|
evidence_rows: list[dict[str, Any]],
|
|
) -> str | None:
|
|
if incident and incident.get("verification_result"):
|
|
return str(incident["verification_result"])
|
|
for row in evidence_rows:
|
|
if row.get("verification_result"):
|
|
return str(row["verification_result"])
|
|
return None
|
|
|
|
|
|
def build_automation_quality(
|
|
*,
|
|
incident: dict[str, Any] | None,
|
|
approvals: list[dict[str, Any]],
|
|
evidence_rows: list[dict[str, Any]],
|
|
automation_ops: list[dict[str, Any]],
|
|
auto_repair_executions: list[dict[str, Any]],
|
|
gateway_mcp_summary: dict[str, Any],
|
|
legacy_mcp_summary: dict[str, Any],
|
|
outbound_rows: list[dict[str, Any]],
|
|
km_entries: list[dict[str, Any]],
|
|
timeline_events: list[dict[str, Any]],
|
|
) -> dict[str, Any]:
|
|
"""Summarize whether a card reached the real automation flywheel."""
|
|
if incident is None:
|
|
return {
|
|
"schema_version": "automation_quality_v1",
|
|
"applicable": False,
|
|
"verdict": "not_applicable",
|
|
"score": 0,
|
|
"gates": [],
|
|
"facts": {},
|
|
"blockers": [],
|
|
}
|
|
|
|
blockers: list[str] = []
|
|
gates: list[dict[str, Any]] = []
|
|
|
|
def gate(name: str, status: str, detail: str | None = None) -> None:
|
|
gates.append({"name": name, "status": status, "detail": detail})
|
|
if status in {"failed", "missing"}:
|
|
blockers.append(name)
|
|
|
|
evidence_attempted = sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows)
|
|
evidence_succeeded = sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows)
|
|
gateway_total = int(gateway_mcp_summary.get("total") or 0)
|
|
legacy_total = int(legacy_mcp_summary.get("total") or 0)
|
|
approval_statuses = {str(row.get("status") or "").upper() for row in approvals}
|
|
approval_actions = " ".join(str(row.get("action") or "") for row in approvals).upper()
|
|
approval_no_action = _approval_has_no_action(approvals)
|
|
approval_suppressed = _approval_suppresses_repair_execution(approvals)
|
|
effective_ops = (
|
|
[]
|
|
if approval_suppressed
|
|
else _effective_execution_ops(automation_ops)
|
|
)
|
|
noop_ops = [row for row in automation_ops if _is_no_action_operation(row)]
|
|
audit_only_ops = [row for row in automation_ops if _is_audit_only_operation(row)]
|
|
automation_statuses = {str(row.get("status") or "").lower() for row in effective_ops}
|
|
auto_repair_successes = {row.get("success") for row in auto_repair_executions}
|
|
has_execution = bool(effective_ops or auto_repair_executions)
|
|
verification_result = _latest_verification_result(incident, evidence_rows)
|
|
|
|
gate("source_persisted", "passed", str(incident.get("incident_id")))
|
|
gate("outbound_recorded", "passed" if outbound_rows else "missing", str(len(outbound_rows)))
|
|
if not evidence_rows:
|
|
gate("evidence_collected", "missing", "no incident_evidence rows")
|
|
elif evidence_attempted > 0 and evidence_succeeded == 0:
|
|
gate("evidence_collected", "failed", f"{evidence_succeeded}/{evidence_attempted}")
|
|
elif evidence_succeeded > 0:
|
|
gate("evidence_collected", "passed", f"{evidence_succeeded}/{evidence_attempted}")
|
|
else:
|
|
gate("evidence_collected", "warning", f"{evidence_succeeded}/{evidence_attempted}")
|
|
|
|
if gateway_total > 0:
|
|
gate("mcp_gateway_observed", "passed", str(gateway_total))
|
|
elif legacy_total > 0:
|
|
gate("mcp_gateway_observed", "warning", f"legacy_only={legacy_total}")
|
|
else:
|
|
gate("mcp_gateway_observed", "missing", "no mcp audit")
|
|
|
|
if any(status in {"PENDING", "WAITING_APPROVAL"} for status in approval_statuses):
|
|
gate("approval_state", "warning", "waiting_approval")
|
|
elif "EXPIRED" in approval_statuses and not has_execution:
|
|
gate("approval_state", "warning", "expired_without_execution")
|
|
elif "REJECTED" in approval_statuses and not has_execution:
|
|
gate("approval_state", "passed", "rejected_no_execution")
|
|
elif approval_statuses and approval_suppressed and not has_execution:
|
|
gate("approval_state", "warning", "approved_diagnostic_or_observe_only")
|
|
elif approval_statuses and (approval_no_action or "NO_ACTION" in approval_actions) and not has_execution:
|
|
gate("approval_state", "failed", "approved_no_action_without_execution")
|
|
elif approvals:
|
|
gate("approval_state", "passed", ",".join(sorted(approval_statuses)))
|
|
else:
|
|
gate("approval_state", "not_applicable", "no approval")
|
|
|
|
gate("execution_recorded", "passed" if has_execution else "missing", str(len(effective_ops) + len(auto_repair_executions)))
|
|
gate("auto_repair_recorded", "passed" if auto_repair_executions else "missing", str(len(auto_repair_executions)))
|
|
|
|
verification_status = str(verification_result or "").lower()
|
|
|
|
if not has_execution:
|
|
gate("verification_recorded", "not_applicable", "no execution")
|
|
elif verification_status == "success":
|
|
gate("verification_recorded", "passed", verification_result)
|
|
elif verification_result:
|
|
gate("verification_recorded", "warning", verification_result)
|
|
else:
|
|
gate("verification_recorded", "missing", "execution without verification_result")
|
|
|
|
if not has_execution:
|
|
gate("learning_recorded", "not_applicable", "no execution")
|
|
elif km_entries:
|
|
gate("learning_recorded", "passed", str(len(km_entries)))
|
|
else:
|
|
gate("learning_recorded", "missing", "execution without KM entry")
|
|
|
|
gate("timeline_recorded", "passed" if timeline_events else "missing", str(len(timeline_events)))
|
|
|
|
if has_execution and (
|
|
False in auto_repair_successes or automation_statuses & {"failed", "error"}
|
|
):
|
|
verdict = "execution_failed"
|
|
elif has_execution and verification_status == "success":
|
|
verdict = "auto_repaired_verified"
|
|
elif has_execution and verification_result:
|
|
verdict = "auto_repaired_verification_degraded"
|
|
elif has_execution:
|
|
verdict = "execution_unverified"
|
|
elif "EXPIRED" in approval_statuses:
|
|
verdict = "approval_expired_ai_retry"
|
|
elif "REJECTED" in approval_statuses:
|
|
verdict = "approval_rejected_no_execution"
|
|
elif approval_suppressed:
|
|
verdict = "manual_required_diagnostic_only"
|
|
elif approval_statuses and (approval_no_action or "NO_ACTION" in approval_actions):
|
|
verdict = "manual_required_no_action"
|
|
elif any(status in {"PENDING", "WAITING_APPROVAL"} for status in approval_statuses):
|
|
verdict = "approval_required"
|
|
elif evidence_rows or gateway_total or legacy_total:
|
|
verdict = "observed_not_executed"
|
|
else:
|
|
verdict = "received_only"
|
|
|
|
score_weights = {
|
|
"source_persisted": 10,
|
|
"outbound_recorded": 10,
|
|
"evidence_collected": 15,
|
|
"mcp_gateway_observed": 15,
|
|
"approval_state": 10,
|
|
"execution_recorded": 15,
|
|
"auto_repair_recorded": 10,
|
|
"verification_recorded": 10,
|
|
"learning_recorded": 3,
|
|
"timeline_recorded": 2,
|
|
}
|
|
score = 0
|
|
for row in gates:
|
|
weight = score_weights.get(str(row["name"]), 0)
|
|
if row["status"] == "passed":
|
|
score += weight
|
|
elif row["status"] == "not_applicable" and row["name"] == "approval_state":
|
|
score += weight
|
|
elif row["status"] == "warning":
|
|
score += weight // 2
|
|
|
|
return {
|
|
"schema_version": "automation_quality_v1",
|
|
"applicable": True,
|
|
"verdict": verdict,
|
|
"score": score,
|
|
"gates": gates,
|
|
"facts": {
|
|
"incident_id": incident.get("incident_id"),
|
|
"evidence_records": len(evidence_rows),
|
|
"sensors_attempted": evidence_attempted,
|
|
"sensors_succeeded": evidence_succeeded,
|
|
"mcp_gateway_total": gateway_total,
|
|
"legacy_mcp_total": legacy_total,
|
|
"approvals": len(approvals),
|
|
"automation_operation_records": len(automation_ops),
|
|
"effective_execution_records": len(effective_ops),
|
|
"noop_operation_records": len(noop_ops),
|
|
"audit_only_operation_records": len(audit_only_ops),
|
|
"approval_repair_suppressed": approval_suppressed,
|
|
"auto_repair_execution_records": len(auto_repair_executions),
|
|
"verification_result": verification_result,
|
|
"knowledge_entries": len(km_entries),
|
|
"timeline_events": len(timeline_events),
|
|
"outbound_messages": len(outbound_rows),
|
|
},
|
|
"blockers": blockers,
|
|
}
|
|
|
|
|
|
def _automation_quality_score_bucket(score: int) -> str:
|
|
if score >= 85:
|
|
return "green"
|
|
if score >= 60:
|
|
return "yellow"
|
|
return "red"
|
|
|
|
|
|
_AUTOMATION_FLOW_GATE_DEFINITIONS: tuple[dict[str, Any], ...] = (
|
|
{
|
|
"gate": "alert_intake",
|
|
"quality_gates": ("source_persisted", "outbound_recorded"),
|
|
"allow_not_applicable": False,
|
|
"next_action": "repair_alert_intake_or_outbound_mirror",
|
|
},
|
|
{
|
|
"gate": "mcp_investigation",
|
|
"quality_gates": ("evidence_collected", "mcp_gateway_observed"),
|
|
"allow_not_applicable": False,
|
|
"next_action": "route_incident_to_mcp_gateway_and_evidence_collectors",
|
|
},
|
|
{
|
|
"gate": "approval_policy",
|
|
"quality_gates": ("approval_state",),
|
|
"allow_not_applicable": True,
|
|
"next_action": "resolve_pending_or_expired_human_gate",
|
|
},
|
|
{
|
|
"gate": "execution_recorded",
|
|
"quality_gates": ("execution_recorded",),
|
|
"allow_not_applicable": False,
|
|
"next_action": "record_effective_execution_or_mark_manual_no_action",
|
|
},
|
|
{
|
|
"gate": "repair_recorded",
|
|
"quality_gates": ("auto_repair_recorded",),
|
|
"allow_not_applicable": False,
|
|
"next_action": "write_auto_repair_execution_or_blocker_reason",
|
|
},
|
|
{
|
|
"gate": "verification_recorded",
|
|
"quality_gates": ("verification_recorded",),
|
|
"allow_not_applicable": False,
|
|
"next_action": "run_post_execution_verification",
|
|
},
|
|
{
|
|
"gate": "knowledge_recorded",
|
|
"quality_gates": ("learning_recorded",),
|
|
"allow_not_applicable": False,
|
|
"next_action": "write_km_or_learning_evidence",
|
|
},
|
|
{
|
|
"gate": "operator_visible",
|
|
"quality_gates": ("outbound_recorded", "timeline_recorded"),
|
|
"allow_not_applicable": False,
|
|
"next_action": "repair_timeline_or_operator_notification_visibility",
|
|
},
|
|
)
|
|
|
|
|
|
def _quality_gate_status_map(quality: dict[str, Any]) -> dict[str, str]:
|
|
statuses: dict[str, str] = {}
|
|
for row in quality.get("gates") or []:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
name = str(row.get("name") or "")
|
|
if not name:
|
|
continue
|
|
statuses[name] = str(row.get("status") or "missing")
|
|
return statuses
|
|
|
|
|
|
def _automation_flow_gate_record_status(
|
|
definition: dict[str, Any],
|
|
gate_statuses: dict[str, str],
|
|
) -> tuple[str, dict[str, str]]:
|
|
aliases = definition.get("source_gate_aliases") or {}
|
|
source_statuses: dict[str, str] = {}
|
|
normalized: list[str] = []
|
|
for source_gate in definition["quality_gates"]:
|
|
mapped_gate = str(aliases.get(source_gate, source_gate))
|
|
raw_status = gate_statuses.get(mapped_gate, "missing")
|
|
status = str(raw_status or "missing")
|
|
source_statuses[mapped_gate] = status
|
|
if status == "not_applicable" and not definition.get("allow_not_applicable"):
|
|
status = "missing"
|
|
normalized.append(status)
|
|
|
|
if any(status == "failed" for status in normalized):
|
|
return "failed", source_statuses
|
|
if any(status == "missing" for status in normalized):
|
|
return "missing", source_statuses
|
|
if any(status == "warning" for status in normalized):
|
|
return "warning", source_statuses
|
|
return "passed", source_statuses
|
|
|
|
|
|
def _automation_flow_gate_summary(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
applicable_records = [
|
|
record
|
|
for record in records
|
|
if isinstance(record.get("automation_quality"), dict)
|
|
and record["automation_quality"].get("applicable") is True
|
|
]
|
|
evaluated_total = len(applicable_records)
|
|
gate_rows: list[dict[str, Any]] = []
|
|
|
|
for definition in _AUTOMATION_FLOW_GATE_DEFINITIONS:
|
|
counts = {"passed": 0, "warning": 0, "missing": 0, "failed": 0}
|
|
examples: list[dict[str, Any]] = []
|
|
for record in applicable_records:
|
|
quality = record["automation_quality"]
|
|
incident = record.get("incident") if isinstance(record.get("incident"), dict) else {}
|
|
truth_status = (
|
|
record.get("truth_status")
|
|
if isinstance(record.get("truth_status"), dict)
|
|
else {}
|
|
)
|
|
status, source_statuses = _automation_flow_gate_record_status(
|
|
definition,
|
|
_quality_gate_status_map(quality),
|
|
)
|
|
counts[status] += 1
|
|
if status != "passed" and len(examples) < 5:
|
|
examples.append({
|
|
"incident_id": incident.get("incident_id"),
|
|
"alertname": incident.get("alertname"),
|
|
"verdict": quality.get("verdict"),
|
|
"truth_stage": truth_status.get("current_stage"),
|
|
"source_statuses": source_statuses,
|
|
"blockers": list(quality.get("blockers") or [])[:6],
|
|
})
|
|
|
|
blocked_total = counts["failed"] + counts["missing"]
|
|
if evaluated_total == 0:
|
|
status = "no_data"
|
|
elif blocked_total > 0:
|
|
status = "blocked"
|
|
elif counts["warning"] > 0:
|
|
status = "warning"
|
|
else:
|
|
status = "passed"
|
|
passed_percent = (
|
|
round((counts["passed"] / evaluated_total) * 100, 1)
|
|
if evaluated_total
|
|
else 0.0
|
|
)
|
|
gate_rows.append({
|
|
"gate": definition["gate"],
|
|
"status": status,
|
|
"passed_total": counts["passed"],
|
|
"warning_total": counts["warning"],
|
|
"missing_total": counts["missing"],
|
|
"failed_total": counts["failed"],
|
|
"evaluated_total": evaluated_total,
|
|
"passed_percent": passed_percent,
|
|
"quality_gates": list(definition["quality_gates"]),
|
|
"next_action": definition["next_action"],
|
|
"examples": examples,
|
|
})
|
|
|
|
blocked_gates = [
|
|
row["gate"]
|
|
for row in gate_rows
|
|
if row["status"] in {"blocked", "no_data"}
|
|
]
|
|
warning_gates = [row["gate"] for row in gate_rows if row["status"] == "warning"]
|
|
if evaluated_total == 0:
|
|
overall_status = "no_data"
|
|
elif blocked_gates:
|
|
overall_status = "blocked"
|
|
elif warning_gates:
|
|
overall_status = "warning"
|
|
else:
|
|
overall_status = "passed"
|
|
|
|
return {
|
|
"schema_version": "automation_flow_gate_summary_v1",
|
|
"evaluated_total": evaluated_total,
|
|
"overall_status": overall_status,
|
|
"blocked_gates": blocked_gates,
|
|
"warning_gates": warning_gates,
|
|
"gates": gate_rows,
|
|
}
|
|
|
|
|
|
def _int_value(value: Any, fallback: int = 0) -> int:
|
|
try:
|
|
return int(value)
|
|
except (TypeError, ValueError):
|
|
return fallback
|
|
|
|
|
|
def _execution_backend_summary(records: list[dict[str, Any]]) -> dict[str, Any]:
|
|
summary = {
|
|
"operation_records_total": 0,
|
|
"effective_execution_records_total": 0,
|
|
"noop_operation_records_total": 0,
|
|
"audit_only_operation_records_total": 0,
|
|
"auto_repair_execution_records_total": 0,
|
|
"ansible_considered_total": 0,
|
|
"ansible_audit_record_total": 0,
|
|
"ansible_candidate_total": 0,
|
|
"ansible_check_mode_total": 0,
|
|
"ansible_apply_total": 0,
|
|
"ansible_rollback_total": 0,
|
|
"ansible_pending_check_mode_total": 0,
|
|
}
|
|
|
|
for record in records:
|
|
quality = record.get("automation_quality") if isinstance(record.get("automation_quality"), dict) else {}
|
|
facts = quality.get("facts") if isinstance(quality.get("facts"), dict) else {}
|
|
execution = record.get("execution") if isinstance(record.get("execution"), dict) else {}
|
|
ops = (
|
|
execution.get("automation_operation_log")
|
|
if isinstance(execution.get("automation_operation_log"), list)
|
|
else []
|
|
)
|
|
auto_repair_executions = (
|
|
execution.get("auto_repair_executions")
|
|
if isinstance(execution.get("auto_repair_executions"), list)
|
|
else []
|
|
)
|
|
ansible = execution.get("ansible") if isinstance(execution.get("ansible"), dict) else {}
|
|
ansible_records = ansible.get("records") if isinstance(ansible.get("records"), list) else []
|
|
catalog = (
|
|
ansible.get("candidate_catalog")
|
|
if isinstance(ansible.get("candidate_catalog"), dict)
|
|
else {}
|
|
)
|
|
candidates = catalog.get("candidates") if isinstance(catalog.get("candidates"), list) else []
|
|
|
|
summary["operation_records_total"] += _int_value(facts.get("automation_operation_records"), len(ops))
|
|
summary["effective_execution_records_total"] += _int_value(
|
|
facts.get("effective_execution_records"),
|
|
len(_effective_execution_ops([row for row in ops if isinstance(row, dict)])),
|
|
)
|
|
summary["noop_operation_records_total"] += _int_value(
|
|
facts.get("noop_operation_records"),
|
|
sum(1 for row in ops if isinstance(row, dict) and _is_no_action_operation(row)),
|
|
)
|
|
summary["audit_only_operation_records_total"] += _int_value(
|
|
facts.get("audit_only_operation_records"),
|
|
sum(1 for row in ops if isinstance(row, dict) and _is_audit_only_operation(row)),
|
|
)
|
|
summary["auto_repair_execution_records_total"] += _int_value(
|
|
facts.get("auto_repair_execution_records"),
|
|
len(auto_repair_executions),
|
|
)
|
|
|
|
if ansible.get("considered") is True:
|
|
summary["ansible_considered_total"] += 1
|
|
summary["ansible_audit_record_total"] += len(ansible_records)
|
|
summary["ansible_candidate_total"] += len(candidates)
|
|
terminal_check_mode_parent_ids = {
|
|
str(row.get("parent_op_id"))
|
|
for row in ansible_records
|
|
if isinstance(row, dict)
|
|
and str(row.get("operation_type") or "") in {
|
|
"ansible_check_mode_executed",
|
|
"ansible_execution_skipped",
|
|
}
|
|
and row.get("parent_op_id")
|
|
}
|
|
|
|
for row in ansible_records:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
operation_type = str(row.get("operation_type") or "")
|
|
status = str(row.get("status") or "").lower()
|
|
if operation_type == "ansible_check_mode_executed" and status != "pending":
|
|
summary["ansible_check_mode_total"] += 1
|
|
elif operation_type == "ansible_apply_executed":
|
|
summary["ansible_apply_total"] += 1
|
|
elif operation_type == "ansible_rollback_executed":
|
|
summary["ansible_rollback_total"] += 1
|
|
elif (
|
|
operation_type == "ansible_candidate_matched"
|
|
and str(row.get("op_id")) not in terminal_check_mode_parent_ids
|
|
):
|
|
summary["ansible_pending_check_mode_total"] += 1
|
|
|
|
return summary
|
|
|
|
|
|
def _ansible_observed_runtime_blockers(records: list[dict[str, Any]]) -> list[str]:
|
|
blockers: set[str] = set()
|
|
for record in records:
|
|
execution = record.get("execution") if isinstance(record.get("execution"), dict) else {}
|
|
ansible = execution.get("ansible") if isinstance(execution.get("ansible"), dict) else {}
|
|
ansible_records = ansible.get("records") if isinstance(ansible.get("records"), list) else []
|
|
for row in ansible_records:
|
|
if not isinstance(row, dict):
|
|
continue
|
|
if str(row.get("operation_type") or "") != "ansible_check_mode_executed":
|
|
continue
|
|
dry_run_result = row.get("dry_run_result") if isinstance(row.get("dry_run_result"), dict) else {}
|
|
blockers.update(
|
|
detect_ansible_transport_blockers(
|
|
row.get("error"),
|
|
dry_run_result.get("stdout_tail"),
|
|
dry_run_result.get("stderr_tail"),
|
|
)
|
|
)
|
|
return sorted(blockers)
|
|
|
|
|
|
def _ansible_playbook_roots(module_path: Path | None = None) -> list[Path]:
|
|
resolved_module_path = (module_path or Path(__file__)).resolve()
|
|
return [
|
|
Path("/app/infra/ansible"),
|
|
Path.cwd() / "infra" / "ansible",
|
|
*(parent / "infra" / "ansible" for parent in resolved_module_path.parents),
|
|
]
|
|
|
|
|
|
def _path_readable(path: Path) -> bool:
|
|
return path.exists() and path.is_file() and os.access(path, os.R_OK)
|
|
|
|
|
|
def _check_mode_uses_repair_forced_command_transport() -> bool:
|
|
return Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH) == Path("/etc/repair-ssh/id_ed25519")
|
|
|
|
|
|
def _ansible_runtime_readiness(
|
|
*,
|
|
check_mode_ssh_key_path: Path | None = None,
|
|
check_mode_known_hosts_path: Path | None = None,
|
|
repair_ssh_key_path: Path = Path("/etc/repair-ssh/id_ed25519"),
|
|
repair_known_hosts_path: Path = Path("/etc/repair-known-hosts/known_hosts"),
|
|
) -> dict[str, Any]:
|
|
playbook_roots = _ansible_playbook_roots()
|
|
playbook_root = next((path for path in playbook_roots if path.exists()), None)
|
|
playbook_paths = (
|
|
sorted((playbook_root / "playbooks").glob("*.yml"))
|
|
if playbook_root is not None and (playbook_root / "playbooks").exists()
|
|
else []
|
|
)
|
|
inventory_path = playbook_root / "inventory" / "hosts.yml" if playbook_root is not None else None
|
|
binary_path = shutil.which("ansible-playbook")
|
|
selected_ssh_key_path = check_mode_ssh_key_path or Path(settings.AWOOOP_ANSIBLE_CHECK_MODE_SSH_KEY_PATH)
|
|
selected_known_hosts_path = check_mode_known_hosts_path or Path(
|
|
settings.AWOOOP_ANSIBLE_CHECK_MODE_KNOWN_HOSTS_PATH
|
|
)
|
|
check_mode_ssh_key_readable = _path_readable(selected_ssh_key_path)
|
|
check_mode_known_hosts_readable = _path_readable(selected_known_hosts_path)
|
|
repair_ssh_key_readable = _path_readable(repair_ssh_key_path)
|
|
repair_known_hosts_readable = _path_readable(repair_known_hosts_path)
|
|
blockers: list[str] = []
|
|
if not binary_path:
|
|
blockers.append("ansible_playbook_binary_missing")
|
|
if playbook_root is None:
|
|
blockers.append("ansible_playbook_catalog_missing")
|
|
if inventory_path is None or not inventory_path.exists():
|
|
blockers.append("ansible_inventory_missing")
|
|
if not playbook_paths:
|
|
blockers.append("ansible_playbooks_missing")
|
|
if not check_mode_ssh_key_readable:
|
|
blockers.append("ansible_check_mode_ssh_key_missing")
|
|
if not check_mode_known_hosts_readable:
|
|
blockers.append("ansible_check_mode_known_hosts_missing")
|
|
|
|
return {
|
|
"ansible_playbook_binary_present": bool(binary_path),
|
|
"ansible_playbook_binary_path": binary_path,
|
|
"playbook_root_present": playbook_root is not None,
|
|
"playbook_root": str(playbook_root) if playbook_root is not None else None,
|
|
"inventory_present": bool(inventory_path and inventory_path.exists()),
|
|
"playbook_count": len(playbook_paths),
|
|
"check_mode_transport_profile": settings.AWOOOP_ANSIBLE_CHECK_MODE_TRANSPORT_PROFILE,
|
|
"check_mode_ssh_key_present": selected_ssh_key_path.exists(),
|
|
"check_mode_ssh_key_readable": check_mode_ssh_key_readable,
|
|
"check_mode_ssh_key_path": str(selected_ssh_key_path),
|
|
"check_mode_known_hosts_present": selected_known_hosts_path.exists(),
|
|
"check_mode_known_hosts_readable": check_mode_known_hosts_readable,
|
|
"check_mode_known_hosts_path": str(selected_known_hosts_path),
|
|
"check_mode_candidate_max_age_hours": settings.AWOOOP_ANSIBLE_CHECK_MODE_CANDIDATE_MAX_AGE_HOURS,
|
|
"repair_ssh_key_present": repair_ssh_key_path.exists(),
|
|
"repair_ssh_key_readable": repair_ssh_key_readable,
|
|
"repair_ssh_key_path": str(repair_ssh_key_path),
|
|
"repair_known_hosts_present": repair_known_hosts_path.exists(),
|
|
"repair_known_hosts_readable": repair_known_hosts_readable,
|
|
"repair_known_hosts_path": str(repair_known_hosts_path),
|
|
"can_run_check_mode": not blockers,
|
|
"blockers": blockers,
|
|
}
|
|
|
|
|
|
def summarize_automation_quality_records(
|
|
*,
|
|
project_id: str,
|
|
window_hours: int,
|
|
records: list[dict[str, Any]],
|
|
limit: int,
|
|
) -> dict[str, Any]:
|
|
"""Aggregate per-incident automation quality into an operator summary."""
|
|
verdicts: dict[str, dict[str, Any]] = {}
|
|
gate_failures: dict[str, dict[str, Any]] = {}
|
|
score_buckets: dict[str, int] = {"green": 0, "yellow": 0, "red": 0}
|
|
examples: list[dict[str, Any]] = []
|
|
total_score = 0
|
|
evaluated_total = 0
|
|
verified_total = 0
|
|
|
|
for record in records:
|
|
incident = record.get("incident") if isinstance(record.get("incident"), dict) else {}
|
|
truth_status = record.get("truth_status") if isinstance(record.get("truth_status"), dict) else {}
|
|
quality = record.get("automation_quality") if isinstance(record.get("automation_quality"), dict) else {}
|
|
if quality.get("applicable") is not True:
|
|
continue
|
|
|
|
evaluated_total += 1
|
|
score = int(quality.get("score") or 0)
|
|
total_score += score
|
|
bucket = _automation_quality_score_bucket(score)
|
|
score_buckets[bucket] += 1
|
|
|
|
verdict = str(quality.get("verdict") or "unknown")
|
|
if verdict == "auto_repaired_verified":
|
|
verified_total += 1
|
|
verdict_row = verdicts.setdefault(
|
|
verdict,
|
|
{
|
|
"verdict": verdict,
|
|
"total": 0,
|
|
"score_sum": 0,
|
|
"min_score": score,
|
|
"max_score": score,
|
|
"needs_human": False,
|
|
},
|
|
)
|
|
verdict_row["total"] += 1
|
|
verdict_row["score_sum"] += score
|
|
verdict_row["min_score"] = min(int(verdict_row["min_score"]), score)
|
|
verdict_row["max_score"] = max(int(verdict_row["max_score"]), score)
|
|
verdict_row["needs_human"] = bool(
|
|
verdict_row["needs_human"] or truth_status.get("needs_human")
|
|
)
|
|
|
|
for gate in quality.get("gates") or []:
|
|
if not isinstance(gate, dict):
|
|
continue
|
|
gate_status = str(gate.get("status") or "")
|
|
if gate_status not in {"failed", "missing"}:
|
|
continue
|
|
gate_name = str(gate.get("name") or "unknown")
|
|
gate_row = gate_failures.setdefault(
|
|
gate_name,
|
|
{"gate": gate_name, "total": 0, "statuses": {}},
|
|
)
|
|
gate_row["total"] += 1
|
|
gate_row["statuses"][gate_status] = int(gate_row["statuses"].get(gate_status, 0)) + 1
|
|
|
|
examples.append({
|
|
"incident_id": incident.get("incident_id"),
|
|
"alertname": incident.get("alertname"),
|
|
"severity": incident.get("severity"),
|
|
"status": incident.get("status"),
|
|
"created_at": incident.get("created_at"),
|
|
"truth_stage": truth_status.get("current_stage"),
|
|
"truth_stage_status": truth_status.get("stage_status"),
|
|
"needs_human": bool(truth_status.get("needs_human")),
|
|
"verdict": verdict,
|
|
"score": score,
|
|
"score_bucket": bucket,
|
|
"blockers": list(quality.get("blockers") or [])[:8],
|
|
})
|
|
|
|
by_verdict = []
|
|
for row in verdicts.values():
|
|
total = int(row["total"])
|
|
row["avg_score"] = round(float(row.pop("score_sum")) / total, 1) if total else 0.0
|
|
by_verdict.append(row)
|
|
by_verdict.sort(key=lambda row: (-int(row["total"]), str(row["verdict"])))
|
|
|
|
failing_gates = sorted(
|
|
gate_failures.values(),
|
|
key=lambda row: (-int(row["total"]), str(row["gate"])),
|
|
)
|
|
|
|
ansible_runtime = _ansible_runtime_readiness()
|
|
observed_ansible_blockers = _ansible_observed_runtime_blockers(records)
|
|
if observed_ansible_blockers:
|
|
ansible_runtime["observed_transport_blockers"] = observed_ansible_blockers
|
|
if _check_mode_uses_repair_forced_command_transport():
|
|
ansible_runtime["blockers"] = sorted(
|
|
set(ansible_runtime.get("blockers") or []) | set(observed_ansible_blockers)
|
|
)
|
|
ansible_runtime["can_run_check_mode"] = False
|
|
else:
|
|
ansible_runtime["historical_transport_blockers"] = observed_ansible_blockers
|
|
|
|
return {
|
|
"schema_version": "automation_quality_summary_v1",
|
|
"project_id": project_id,
|
|
"window_hours": window_hours,
|
|
"limit": limit,
|
|
"incident_total": len(records),
|
|
"evaluated_total": evaluated_total,
|
|
"verified_auto_repair_total": verified_total,
|
|
"average_score": round(total_score / evaluated_total, 1) if evaluated_total else 0.0,
|
|
"score_buckets": score_buckets,
|
|
"by_verdict": by_verdict,
|
|
"gate_failures": failing_gates,
|
|
"automation_flow_gates": _automation_flow_gate_summary(records),
|
|
"execution_backend_summary": _execution_backend_summary(records),
|
|
"ansible_runtime": ansible_runtime,
|
|
"examples": examples[:25],
|
|
"production_claim": {
|
|
"can_claim_full_auto_repair": evaluated_total > 0 and verified_total == evaluated_total,
|
|
"reason": (
|
|
"all_evaluated_incidents_auto_repaired_verified"
|
|
if evaluated_total > 0 and verified_total == evaluated_total
|
|
else "some_incidents_are_not_auto_repaired_verified"
|
|
),
|
|
},
|
|
}
|
|
|
|
|
|
def _summarize_mcp(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
by_tool: dict[str, dict[str, Any]] = {}
|
|
success_count = 0
|
|
failure_count = 0
|
|
for row in rows:
|
|
key = f"{row.get('mcp_server') or 'unknown'}:{row.get('tool_name') or 'unknown'}"
|
|
item = by_tool.setdefault(
|
|
key,
|
|
{
|
|
"mcp_server": row.get("mcp_server"),
|
|
"tool_name": row.get("tool_name"),
|
|
"success": 0,
|
|
"failed": 0,
|
|
"last_error": None,
|
|
},
|
|
)
|
|
if row.get("success") is True:
|
|
item["success"] += 1
|
|
success_count += 1
|
|
else:
|
|
item["failed"] += 1
|
|
failure_count += 1
|
|
item["last_error"] = row.get("error_message")
|
|
return {
|
|
"total": len(rows),
|
|
"success": success_count,
|
|
"failed": failure_count,
|
|
"by_tool": list(by_tool.values()),
|
|
}
|
|
|
|
|
|
def _as_bool(value: Any) -> bool:
|
|
if isinstance(value, bool):
|
|
return value
|
|
return str(value).lower() == "true"
|
|
|
|
|
|
def _counter_bucket(
|
|
buckets: dict[str, dict[str, Any]],
|
|
key: str,
|
|
*,
|
|
label: str,
|
|
) -> dict[str, Any]:
|
|
return buckets.setdefault(
|
|
key,
|
|
{
|
|
label: key,
|
|
"total": 0,
|
|
"success": 0,
|
|
"failed": 0,
|
|
"blocked": 0,
|
|
},
|
|
)
|
|
|
|
|
|
def _summarize_gateway_mcp(rows: list[dict[str, Any]]) -> dict[str, Any]:
|
|
by_agent: dict[str, dict[str, Any]] = {}
|
|
by_tool: dict[str, dict[str, Any]] = {}
|
|
by_scope: dict[str, dict[str, Any]] = {}
|
|
success_count = 0
|
|
failed_count = 0
|
|
blocked_count = 0
|
|
first_class_count = 0
|
|
bridge_count = 0
|
|
policy_enforced_count = 0
|
|
approval_executor_count = 0
|
|
|
|
for row in rows:
|
|
gate_result = row.get("gate_result") if isinstance(row.get("gate_result"), dict) else {}
|
|
gateway_path = str(gate_result.get("gateway_path") or "")
|
|
schema_version = str(gate_result.get("schema_version") or "")
|
|
policy_enforced = _as_bool(gate_result.get("policy_enforced"))
|
|
required_scope = str(gate_result.get("required_scope") or "unknown")
|
|
status = str(row.get("result_status") or "unknown").lower()
|
|
is_blocked = status == "blocked" or row.get("block_gate") is not None
|
|
is_success = status == "success"
|
|
is_failed = status == "failed"
|
|
agent_id = str(row.get("agent_id") or "unknown")
|
|
tool_name = str(row.get("tool_name") or "unknown")
|
|
|
|
if gateway_path == "awooop_mcp_gateway" and policy_enforced:
|
|
first_class_count += 1
|
|
if schema_version == "legacy_mcp_bridge_v1" or policy_enforced is False:
|
|
bridge_count += 1
|
|
if policy_enforced:
|
|
policy_enforced_count += 1
|
|
if agent_id == "approval_executor":
|
|
approval_executor_count += 1
|
|
if is_success:
|
|
success_count += 1
|
|
if is_failed:
|
|
failed_count += 1
|
|
if is_blocked:
|
|
blocked_count += 1
|
|
|
|
for bucket in (
|
|
_counter_bucket(by_agent, agent_id, label="agent_id"),
|
|
_counter_bucket(by_tool, tool_name, label="tool_name"),
|
|
_counter_bucket(by_scope, required_scope, label="required_scope"),
|
|
):
|
|
bucket["total"] += 1
|
|
if is_success:
|
|
bucket["success"] += 1
|
|
if is_failed:
|
|
bucket["failed"] += 1
|
|
if is_blocked:
|
|
bucket["blocked"] += 1
|
|
|
|
blockers: list[str] = []
|
|
if blocked_count:
|
|
stage = "gateway_blocked"
|
|
stage_status = "blocked"
|
|
blockers.append("mcp_gateway_blocked")
|
|
elif first_class_count and failed_count:
|
|
stage = "provider_failed_after_gateway"
|
|
stage_status = "failed"
|
|
blockers.append("provider_failed_after_gateway")
|
|
elif success_count:
|
|
stage = "gateway_execution_succeeded"
|
|
stage_status = "success"
|
|
elif bridge_count and not first_class_count:
|
|
stage = "legacy_bridge_only"
|
|
stage_status = "observed"
|
|
blockers.append("legacy_bridge_not_policy_enforced")
|
|
else:
|
|
stage = "no_gateway_records"
|
|
stage_status = "missing"
|
|
|
|
return {
|
|
"total": len(rows),
|
|
"success": success_count,
|
|
"failed": failed_count,
|
|
"blocked": blocked_count,
|
|
"first_class_total": first_class_count,
|
|
"legacy_bridge_total": bridge_count,
|
|
"policy_enforced_total": policy_enforced_count,
|
|
"approval_executor_total": approval_executor_count,
|
|
"stage": stage,
|
|
"stage_status": stage_status,
|
|
"needs_human": bool(blocked_count or (first_class_count and failed_count)),
|
|
"blockers": blockers,
|
|
"by_agent": list(by_agent.values()),
|
|
"by_tool": list(by_tool.values()),
|
|
"by_scope": list(by_scope.values()),
|
|
}
|
|
|
|
|
|
async def fetch_truth_chain(source_id: str, project_id: str = "awoooi") -> dict[str, Any]:
|
|
"""Return a read-only truth chain for an incident, drift report, or run id."""
|
|
async with get_db_context(project_id) as db:
|
|
incident = await _fetch_one(
|
|
db,
|
|
"""
|
|
SELECT
|
|
incident_id,
|
|
project_id,
|
|
status::text AS status,
|
|
severity::text AS severity,
|
|
alertname,
|
|
alert_category,
|
|
notification_type,
|
|
created_at,
|
|
updated_at,
|
|
resolved_at,
|
|
verification_result,
|
|
frequency_snapshot,
|
|
signals,
|
|
decision_chain
|
|
FROM incidents
|
|
WHERE incident_id = :source_id
|
|
AND (project_id = :project_id OR project_id IS NULL)
|
|
""",
|
|
{"source_id": source_id, "project_id": project_id},
|
|
)
|
|
|
|
drift = await _fetch_one(
|
|
db,
|
|
"""
|
|
SELECT
|
|
report_id,
|
|
namespace,
|
|
status,
|
|
triggered_by,
|
|
high_count,
|
|
medium_count,
|
|
info_count,
|
|
scanned_at,
|
|
created_at,
|
|
resolved_at,
|
|
interpretation,
|
|
items,
|
|
narrative_text
|
|
FROM drift_reports
|
|
WHERE report_id = :source_id
|
|
""",
|
|
{"source_id": source_id},
|
|
)
|
|
|
|
runs = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
run_id,
|
|
project_id,
|
|
agent_id,
|
|
state,
|
|
trigger_type,
|
|
trigger_ref,
|
|
is_shadow,
|
|
step_count,
|
|
created_at,
|
|
started_at,
|
|
completed_at,
|
|
error_code,
|
|
error_detail
|
|
FROM awooop_run_state
|
|
WHERE project_id = :project_id
|
|
AND (run_id::text = :source_id OR trigger_ref = :source_id)
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"source_id": source_id, "project_id": project_id, "limit": _MAX_ROWS},
|
|
)
|
|
|
|
approvals: list[dict[str, Any]] = []
|
|
evidence_rows: list[dict[str, Any]] = []
|
|
timeline_events: list[dict[str, Any]] = []
|
|
legacy_mcp_rows: list[dict[str, Any]] = []
|
|
automation_ops: list[dict[str, Any]] = []
|
|
auto_repair_executions: list[dict[str, Any]] = []
|
|
km_entries: list[dict[str, Any]] = []
|
|
inbound_rows: list[dict[str, Any]] = []
|
|
gateway_mcp_rows: list[dict[str, Any]] = []
|
|
outbound_rows: list[dict[str, Any]] = []
|
|
|
|
if incident is not None:
|
|
incident_id = str(incident["incident_id"])
|
|
approvals = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
status::text AS status,
|
|
risk_level::text AS risk_level,
|
|
action,
|
|
description,
|
|
hit_count,
|
|
created_at,
|
|
updated_at,
|
|
resolved_at,
|
|
matched_playbook_id,
|
|
extra_metadata,
|
|
decision_fusion_details
|
|
FROM approval_records
|
|
WHERE incident_id = :incident_id
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "limit": _MAX_ROWS},
|
|
)
|
|
approval_ids = _approval_ids(approvals)
|
|
evidence_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
matched_playbook_id,
|
|
collected_at,
|
|
collection_duration_ms,
|
|
sensors_attempted,
|
|
sensors_succeeded,
|
|
evidence_summary,
|
|
metrics_snapshot,
|
|
mcp_health,
|
|
verification_result,
|
|
pre_execution_state,
|
|
post_execution_state,
|
|
self_healing_score,
|
|
self_healing_detail
|
|
FROM incident_evidence
|
|
WHERE incident_id = :incident_id
|
|
ORDER BY collected_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "limit": _MAX_ROWS},
|
|
)
|
|
timeline_events = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
event_type,
|
|
status,
|
|
title,
|
|
description,
|
|
actor,
|
|
actor_role,
|
|
risk_level,
|
|
approval_id,
|
|
incident_id,
|
|
created_at
|
|
FROM timeline_events
|
|
WHERE incident_id = :incident_id
|
|
OR approval_id = ANY(:approval_ids)
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
{
|
|
"incident_id": incident_id,
|
|
"approval_ids": approval_ids or ["__none__"],
|
|
"limit": _MAX_ROWS,
|
|
},
|
|
)
|
|
legacy_mcp_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
session_id,
|
|
flywheel_node,
|
|
mcp_server,
|
|
tool_name,
|
|
duration_ms,
|
|
success,
|
|
error_message,
|
|
incident_id,
|
|
agent_role,
|
|
created_at
|
|
FROM mcp_audit_log
|
|
WHERE incident_id = :incident_id
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "limit": _MAX_ROWS},
|
|
)
|
|
automation_ops = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
op_id,
|
|
operation_type,
|
|
status,
|
|
incident_id,
|
|
run_id,
|
|
parent_op_id,
|
|
actor,
|
|
dry_run_result,
|
|
error,
|
|
duration_ms,
|
|
tags,
|
|
input ->> 'action' AS input_action,
|
|
input ->> 'executor' AS input_executor,
|
|
input ->> 'execution_backend' AS input_execution_backend,
|
|
input ->> 'catalog_id' AS input_catalog_id,
|
|
input ->> 'execution_mode' AS input_execution_mode,
|
|
input ->> 'approval_source' AS input_approval_source,
|
|
input ->> 'apply_enabled' AS input_apply_enabled,
|
|
input ->> 'apply_executed' AS input_apply_executed,
|
|
input ->> 'check_mode_executed' AS input_check_mode_executed,
|
|
input ->> 'returncode' AS input_returncode,
|
|
input ->> 'playbook_id' AS input_playbook_id,
|
|
input ->> 'playbook_path' AS input_playbook_path,
|
|
input ->> 'ansible_playbook_path' AS input_ansible_playbook_path,
|
|
input ->> 'check_mode' AS input_check_mode,
|
|
input ->> 'not_used_reason' AS input_not_used_reason,
|
|
output ->> 'action' AS output_action,
|
|
output ->> 'reason' AS output_reason,
|
|
output ->> 'executor' AS output_executor,
|
|
output ->> 'execution_backend' AS output_execution_backend,
|
|
output ->> 'catalog_id' AS output_catalog_id,
|
|
output ->> 'execution_mode' AS output_execution_mode,
|
|
output ->> 'approval_source' AS output_approval_source,
|
|
output ->> 'apply_enabled' AS output_apply_enabled,
|
|
output ->> 'apply_executed' AS output_apply_executed,
|
|
output ->> 'check_mode_executed' AS output_check_mode_executed,
|
|
output ->> 'returncode' AS output_returncode,
|
|
output ->> 'playbook_id' AS output_playbook_id,
|
|
output ->> 'playbook_path' AS output_playbook_path,
|
|
output ->> 'ansible_playbook_path' AS output_ansible_playbook_path,
|
|
output ->> 'check_mode' AS output_check_mode,
|
|
output ->> 'not_used_reason' AS output_not_used_reason,
|
|
dry_run_result ->> 'returncode' AS dry_run_returncode,
|
|
dry_run_result ->> 'apply_executed' AS dry_run_apply_executed,
|
|
dry_run_result ->> 'check_mode_executed' AS dry_run_check_mode_executed,
|
|
created_at
|
|
FROM automation_operation_log
|
|
WHERE incident_id::text = :incident_id
|
|
OR coalesce(input::text, '') LIKE :needle
|
|
OR coalesce(output::text, '') LIKE :needle
|
|
OR coalesce(array_to_string(tags, ','), '') LIKE :needle
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "needle": f"%{incident_id}%", "limit": _MAX_ROWS},
|
|
)
|
|
auto_repair_executions = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
playbook_id,
|
|
playbook_name,
|
|
success,
|
|
executed_steps,
|
|
error_message,
|
|
triggered_by,
|
|
similarity_score,
|
|
risk_level,
|
|
execution_time_ms,
|
|
created_at
|
|
FROM auto_repair_executions
|
|
WHERE incident_id = :incident_id
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "limit": _MAX_ROWS},
|
|
)
|
|
km_entries = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
title,
|
|
entry_type::text AS entry_type,
|
|
status::text AS status,
|
|
related_incident_id,
|
|
related_approval_id,
|
|
created_at
|
|
FROM knowledge_entries
|
|
WHERE related_incident_id = :incident_id
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{"incident_id": incident_id, "limit": _MAX_ROWS},
|
|
)
|
|
|
|
drift_repeats: dict[str, Any] = {
|
|
"occurrences_12h": 0,
|
|
"first_scanned_at": None,
|
|
"last_scanned_at": None,
|
|
"reports": [],
|
|
}
|
|
if drift is not None:
|
|
recent_drift_reports = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
report_id,
|
|
namespace,
|
|
status,
|
|
scanned_at,
|
|
created_at,
|
|
items,
|
|
interpretation,
|
|
narrative_text
|
|
FROM drift_reports
|
|
WHERE created_at > now() - interval '24 hours'
|
|
AND namespace = :namespace
|
|
ORDER BY scanned_at DESC
|
|
LIMIT 200
|
|
""",
|
|
{"namespace": drift["namespace"]},
|
|
)
|
|
drift_repeats = build_drift_repeat_state(drift, recent_drift_reports)
|
|
|
|
gateway_mcp_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
call_id,
|
|
project_id,
|
|
run_id,
|
|
trace_id,
|
|
agent_id,
|
|
tool_name,
|
|
gate_result,
|
|
result_status,
|
|
block_gate,
|
|
block_reason,
|
|
latency_ms,
|
|
created_at
|
|
FROM awooop_mcp_gateway_audit
|
|
WHERE project_id = :project_id
|
|
AND (
|
|
trace_id = :source_id
|
|
OR run_id::text = :source_id
|
|
)
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
{"source_id": source_id, "project_id": project_id, "limit": _MAX_ROWS},
|
|
)
|
|
incident_fingerprints = _incident_fingerprints(incident)
|
|
fingerprint_needle = (
|
|
f"%{incident_fingerprints[0]}%"
|
|
if incident_fingerprints
|
|
else ""
|
|
)
|
|
fingerprint_value = incident_fingerprints[0] if incident_fingerprints else ""
|
|
inbound_rows = await _fetch_inbound_conversation_event_rows(
|
|
db,
|
|
project_id=project_id,
|
|
source_id=source_id,
|
|
fingerprint_needle=fingerprint_needle,
|
|
fingerprint_value=fingerprint_value,
|
|
limit=_MAX_ROWS,
|
|
)
|
|
outbound_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
message_id,
|
|
project_id,
|
|
run_id,
|
|
channel_type,
|
|
message_type,
|
|
content_hash,
|
|
content_preview,
|
|
content_redacted,
|
|
redaction_version,
|
|
source_envelope,
|
|
provider_message_id,
|
|
send_status,
|
|
queued_at,
|
|
sent_at,
|
|
triggered_by_state
|
|
FROM awooop_outbound_message
|
|
WHERE project_id = :project_id
|
|
AND (
|
|
run_id::text = :source_id
|
|
OR content_preview ILIKE :needle
|
|
OR coalesce(source_envelope #> '{source_refs,incident_ids}', '[]'::jsonb) ? :source_id
|
|
OR coalesce(source_envelope #> '{source_refs,code_refs}', '[]'::jsonb) ? :source_id
|
|
)
|
|
ORDER BY queued_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{
|
|
"source_id": source_id,
|
|
"project_id": project_id,
|
|
"needle": f"%{source_id}%",
|
|
"limit": _MAX_ROWS,
|
|
},
|
|
)
|
|
|
|
source_type = _source_type(source_id, incident, drift)
|
|
legacy_mcp_summary = _summarize_mcp(legacy_mcp_rows)
|
|
gateway_mcp_summary = _summarize_gateway_mcp(gateway_mcp_rows)
|
|
truth_status = _truth_status(
|
|
incident=incident,
|
|
approvals=approvals,
|
|
evidence_rows=evidence_rows,
|
|
automation_ops=automation_ops,
|
|
drift=drift,
|
|
drift_repeat_count=int(drift_repeats.get("occurrences_12h") or 0),
|
|
gateway_mcp_total=len(gateway_mcp_rows),
|
|
legacy_mcp_total=legacy_mcp_summary["total"],
|
|
outbound_visible_total=len(outbound_rows),
|
|
inbound_visible_total=len(inbound_rows),
|
|
auto_repair_executions=auto_repair_executions,
|
|
)
|
|
if incident is None and drift is None and not runs and gateway_mcp_rows:
|
|
truth_status = {
|
|
"current_stage": gateway_mcp_summary["stage"],
|
|
"stage_status": gateway_mcp_summary["stage_status"],
|
|
"needs_human": gateway_mcp_summary["needs_human"],
|
|
"blockers": gateway_mcp_summary["blockers"],
|
|
}
|
|
reconciliation = build_incident_reconciliation(
|
|
incident=incident,
|
|
approvals=approvals,
|
|
evidence_rows=evidence_rows,
|
|
automation_ops=automation_ops,
|
|
auto_repair_executions=auto_repair_executions,
|
|
timeline_events=timeline_events,
|
|
)
|
|
|
|
evidence_totals = {
|
|
"records": len(evidence_rows),
|
|
"sensors_attempted": sum(int(row.get("sensors_attempted") or 0) for row in evidence_rows),
|
|
"sensors_succeeded": sum(int(row.get("sensors_succeeded") or 0) for row in evidence_rows),
|
|
"failed_tools": _failed_mcp_tools(evidence_rows),
|
|
}
|
|
automation_quality = build_automation_quality(
|
|
incident=incident,
|
|
approvals=approvals,
|
|
evidence_rows=evidence_rows,
|
|
automation_ops=automation_ops,
|
|
auto_repair_executions=auto_repair_executions,
|
|
gateway_mcp_summary=gateway_mcp_summary,
|
|
legacy_mcp_summary=legacy_mcp_summary,
|
|
outbound_rows=outbound_rows,
|
|
km_entries=km_entries,
|
|
timeline_events=timeline_events,
|
|
)
|
|
automation_quality["operator_outcome"] = build_operator_outcome(
|
|
truth_status=truth_status,
|
|
automation_quality=automation_quality,
|
|
source_id=source_id,
|
|
)
|
|
|
|
result = {
|
|
"project_id": project_id,
|
|
"source_id": source_id,
|
|
"source_type": source_type,
|
|
"found": (
|
|
incident is not None
|
|
or drift is not None
|
|
or bool(runs)
|
|
or bool(gateway_mcp_rows)
|
|
or bool(inbound_rows)
|
|
or bool(outbound_rows)
|
|
),
|
|
"truth_status": truth_status,
|
|
"linked_ids": {
|
|
"incident_id": incident.get("incident_id") if incident else None,
|
|
"approval_ids": _approval_ids(approvals),
|
|
"run_ids": [row["run_id"] for row in runs],
|
|
"drift_report_id": drift.get("report_id") if drift else None,
|
|
"operation_ids": _operation_ids(automation_ops),
|
|
"auto_repair_execution_ids": _auto_repair_ids(auto_repair_executions),
|
|
"conversation_event_ids": [row["event_id"] for row in inbound_rows],
|
|
},
|
|
"incident": incident,
|
|
"drift": {
|
|
"report": drift,
|
|
"repeat_state": drift_repeats,
|
|
},
|
|
"approvals": approvals,
|
|
"evidence": {
|
|
"summary": evidence_totals,
|
|
"records": evidence_rows,
|
|
},
|
|
"mcp": {
|
|
"awooop_gateway": {
|
|
**gateway_mcp_summary,
|
|
"records": gateway_mcp_rows,
|
|
},
|
|
"legacy": {
|
|
**legacy_mcp_summary,
|
|
"records": legacy_mcp_rows,
|
|
},
|
|
},
|
|
"execution": {
|
|
"automation_operation_log": automation_ops,
|
|
"auto_repair_executions": auto_repair_executions,
|
|
"ansible": build_ansible_truth(automation_ops, incident=incident, drift=drift),
|
|
},
|
|
"automation_quality": automation_quality,
|
|
"operator_outcome": automation_quality["operator_outcome"],
|
|
"reconciliation": reconciliation,
|
|
"learning": {
|
|
"knowledge_entries": km_entries,
|
|
},
|
|
"timeline_events": timeline_events,
|
|
"channel": {
|
|
"inbound_events_visible": len(inbound_rows),
|
|
"inbound_events": inbound_rows,
|
|
"outbound_messages_visible": len(outbound_rows),
|
|
"outbound_messages": outbound_rows,
|
|
"visibility_note": (
|
|
"If inbound is zero while Alertmanager fired, or outbound is zero while "
|
|
"Telegram delivered a card, channel mirrors or RLS project context are "
|
|
"not reliable sources of truth yet."
|
|
),
|
|
},
|
|
}
|
|
logger.info(
|
|
"awooop_truth_chain_fetched",
|
|
project_id=project_id,
|
|
source_id=source_id,
|
|
source_type=source_type,
|
|
current_stage=truth_status["current_stage"],
|
|
)
|
|
return result
|
|
|
|
|
|
def _empty_incident_groups(incident_ids: list[str]) -> dict[str, list[dict[str, Any]]]:
|
|
return {incident_id: [] for incident_id in incident_ids}
|
|
|
|
|
|
def _append_unique_row(group: list[dict[str, Any]], row: dict[str, Any]) -> None:
|
|
if row not in group:
|
|
group.append(row)
|
|
|
|
|
|
def _row_source_refs(row: dict[str, Any]) -> dict[str, Any]:
|
|
envelope = row.get("source_envelope")
|
|
if not isinstance(envelope, dict):
|
|
return {}
|
|
refs = envelope.get("source_refs")
|
|
return refs if isinstance(refs, dict) else {}
|
|
|
|
|
|
def _source_ref_contains(row: dict[str, Any], keys: tuple[str, ...], incident_id: str) -> bool:
|
|
refs = _row_source_refs(row)
|
|
for key in keys:
|
|
value = refs.get(key)
|
|
if isinstance(value, list) and incident_id in {str(item) for item in value}:
|
|
return True
|
|
if str(value or "") == incident_id:
|
|
return True
|
|
return False
|
|
|
|
|
|
def _group_rows_by_incident_reference(
|
|
*,
|
|
rows: list[dict[str, Any]],
|
|
incident_ids: list[str],
|
|
direct_fields: tuple[str, ...],
|
|
text_fields: tuple[str, ...] = (),
|
|
source_ref_keys: tuple[str, ...] = (),
|
|
) -> dict[str, list[dict[str, Any]]]:
|
|
"""Group batch-fetched rows back to incidents using durable direct/source refs."""
|
|
incident_set = set(incident_ids)
|
|
groups = _empty_incident_groups(incident_ids)
|
|
for row in rows:
|
|
direct_matches = {
|
|
str(row.get(field) or "")
|
|
for field in direct_fields
|
|
if str(row.get(field) or "") in incident_set
|
|
}
|
|
for incident_id in direct_matches:
|
|
groups[incident_id].append(row)
|
|
if direct_matches:
|
|
continue
|
|
|
|
text_blob = "\n".join(str(row.get(field) or "") for field in text_fields)
|
|
for incident_id in incident_ids:
|
|
if (
|
|
(text_blob and incident_id in text_blob)
|
|
or (
|
|
source_ref_keys
|
|
and _source_ref_contains(row, source_ref_keys, incident_id)
|
|
)
|
|
):
|
|
_append_unique_row(groups[incident_id], row)
|
|
return groups
|
|
|
|
|
|
def _group_timeline_events_by_incident(
|
|
*,
|
|
rows: list[dict[str, Any]],
|
|
incident_ids: list[str],
|
|
approval_to_incident: dict[str, str],
|
|
) -> dict[str, list[dict[str, Any]]]:
|
|
groups = _empty_incident_groups(incident_ids)
|
|
incident_set = set(incident_ids)
|
|
for row in rows:
|
|
incident_id = str(row.get("incident_id") or "")
|
|
if incident_id in incident_set:
|
|
groups[incident_id].append(row)
|
|
continue
|
|
approval_incident_id = approval_to_incident.get(str(row.get("approval_id") or ""))
|
|
if approval_incident_id:
|
|
groups[approval_incident_id].append(row)
|
|
return groups
|
|
|
|
|
|
def _build_summary_quality_records(
|
|
*,
|
|
incidents: list[dict[str, Any]],
|
|
approvals_by_incident: dict[str, list[dict[str, Any]]],
|
|
evidence_by_incident: dict[str, list[dict[str, Any]]],
|
|
timeline_by_incident: dict[str, list[dict[str, Any]]],
|
|
legacy_mcp_by_incident: dict[str, list[dict[str, Any]]],
|
|
automation_ops_by_incident: dict[str, list[dict[str, Any]]],
|
|
auto_repair_by_incident: dict[str, list[dict[str, Any]]],
|
|
km_by_incident: dict[str, list[dict[str, Any]]],
|
|
gateway_mcp_by_incident: dict[str, list[dict[str, Any]]],
|
|
outbound_by_incident: dict[str, list[dict[str, Any]]],
|
|
) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
for incident in incidents:
|
|
incident_id = str(incident.get("incident_id") or "")
|
|
if not incident_id:
|
|
continue
|
|
approvals = approvals_by_incident.get(incident_id, [])
|
|
evidence_rows = evidence_by_incident.get(incident_id, [])
|
|
timeline_events = timeline_by_incident.get(incident_id, [])
|
|
legacy_mcp_rows = legacy_mcp_by_incident.get(incident_id, [])
|
|
automation_ops = automation_ops_by_incident.get(incident_id, [])
|
|
auto_repair_executions = auto_repair_by_incident.get(incident_id, [])
|
|
km_entries = km_by_incident.get(incident_id, [])
|
|
gateway_mcp_rows = gateway_mcp_by_incident.get(incident_id, [])
|
|
outbound_rows = outbound_by_incident.get(incident_id, [])
|
|
legacy_mcp_summary = _summarize_mcp(legacy_mcp_rows)
|
|
gateway_mcp_summary = _summarize_gateway_mcp(gateway_mcp_rows)
|
|
truth_status = _truth_status(
|
|
incident=incident,
|
|
approvals=approvals,
|
|
evidence_rows=evidence_rows,
|
|
automation_ops=automation_ops,
|
|
drift=None,
|
|
drift_repeat_count=0,
|
|
gateway_mcp_total=len(gateway_mcp_rows),
|
|
legacy_mcp_total=legacy_mcp_summary["total"],
|
|
outbound_visible_total=len(outbound_rows),
|
|
inbound_visible_total=0,
|
|
auto_repair_executions=auto_repair_executions,
|
|
)
|
|
automation_quality = build_automation_quality(
|
|
incident=incident,
|
|
approvals=approvals,
|
|
evidence_rows=evidence_rows,
|
|
automation_ops=automation_ops,
|
|
auto_repair_executions=auto_repair_executions,
|
|
gateway_mcp_summary=gateway_mcp_summary,
|
|
legacy_mcp_summary=legacy_mcp_summary,
|
|
outbound_rows=outbound_rows,
|
|
km_entries=km_entries,
|
|
timeline_events=timeline_events,
|
|
)
|
|
automation_quality["operator_outcome"] = build_operator_outcome(
|
|
truth_status=truth_status,
|
|
automation_quality=automation_quality,
|
|
source_id=incident_id,
|
|
)
|
|
records.append({
|
|
"incident": incident,
|
|
"truth_status": truth_status,
|
|
"automation_quality": automation_quality,
|
|
"execution": {
|
|
"automation_operation_log": automation_ops,
|
|
"auto_repair_executions": auto_repair_executions,
|
|
"ansible": build_ansible_truth(automation_ops, incident=incident, drift=None),
|
|
},
|
|
})
|
|
return records
|
|
|
|
|
|
async def fetch_automation_quality_summary(
|
|
*,
|
|
project_id: str = "awoooi",
|
|
hours: int = 24,
|
|
limit: int = 200,
|
|
refresh: bool = False,
|
|
) -> dict[str, Any]:
|
|
"""Return a recent incident-level quality summary for the automation flywheel."""
|
|
started_at = perf_counter()
|
|
bounded_hours = max(1, min(int(hours), 168))
|
|
bounded_limit = max(1, min(int(limit), 500))
|
|
normalized_project_id = project_id or "awoooi"
|
|
cache_key = {
|
|
"project_id": normalized_project_id,
|
|
"hours": bounded_hours,
|
|
"limit": bounded_limit,
|
|
}
|
|
if not refresh:
|
|
cached_summary = await get_cached_operator_summary_async(
|
|
"truth_chain_quality_summary",
|
|
cache_key,
|
|
ttl_seconds=_QUALITY_SUMMARY_CACHE_TTL_SECONDS,
|
|
)
|
|
if cached_summary is not None:
|
|
duration_seconds = perf_counter() - started_at
|
|
record_quality_summary_observation(
|
|
project_id=normalized_project_id,
|
|
hours=bounded_hours,
|
|
limit=bounded_limit,
|
|
cache_status="hit",
|
|
success=True,
|
|
duration_seconds=duration_seconds,
|
|
)
|
|
logger.info(
|
|
"awooop_automation_quality_summary_cache_hit",
|
|
project_id=normalized_project_id,
|
|
window_hours=bounded_hours,
|
|
limit=bounded_limit,
|
|
ttl_seconds=_QUALITY_SUMMARY_CACHE_TTL_SECONDS,
|
|
duration_seconds=round(duration_seconds, 3),
|
|
)
|
|
cached_summary = dict(cached_summary)
|
|
cached_summary["cache_status"] = "hit"
|
|
cached_summary["aggregation_duration_seconds"] = round(duration_seconds, 3)
|
|
return cached_summary
|
|
|
|
cutoff = datetime.now(UTC) - timedelta(hours=bounded_hours)
|
|
|
|
async with get_db_context(normalized_project_id) as db:
|
|
incidents = await _fetch_all(
|
|
db,
|
|
"""
|
|
WITH source_candidates AS (
|
|
SELECT
|
|
incident_id::text AS incident_id,
|
|
created_at AS seen_at
|
|
FROM incidents
|
|
WHERE (project_id = :project_id OR project_id IS NULL)
|
|
AND created_at >= :cutoff
|
|
|
|
UNION ALL
|
|
|
|
SELECT
|
|
coalesce(incident_id::text, input ->> 'incident_id') AS incident_id,
|
|
created_at AS seen_at
|
|
FROM automation_operation_log
|
|
WHERE created_at >= :cutoff
|
|
AND operation_type IN (
|
|
'ansible_candidate_matched',
|
|
'ansible_check_mode_executed',
|
|
'ansible_execution_skipped',
|
|
'ansible_apply_executed',
|
|
'ansible_rollback_executed'
|
|
)
|
|
AND coalesce(incident_id::text, input ->> 'incident_id', '') <> ''
|
|
),
|
|
source_ids AS (
|
|
SELECT
|
|
incident_id,
|
|
max(seen_at) AS recent_evidence_at
|
|
FROM source_candidates
|
|
WHERE incident_id IS NOT NULL AND incident_id <> ''
|
|
GROUP BY incident_id
|
|
)
|
|
SELECT
|
|
incidents.incident_id,
|
|
incidents.project_id,
|
|
incidents.status::text AS status,
|
|
incidents.severity::text AS severity,
|
|
incidents.alertname,
|
|
incidents.alert_category,
|
|
incidents.notification_type,
|
|
incidents.created_at,
|
|
incidents.updated_at,
|
|
incidents.resolved_at,
|
|
incidents.verification_result,
|
|
source_ids.recent_evidence_at
|
|
FROM source_ids
|
|
JOIN incidents ON incidents.incident_id = source_ids.incident_id
|
|
WHERE (incidents.project_id = :project_id OR incidents.project_id IS NULL)
|
|
ORDER BY source_ids.recent_evidence_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{
|
|
"project_id": normalized_project_id,
|
|
"cutoff": cutoff,
|
|
"limit": bounded_limit,
|
|
},
|
|
)
|
|
|
|
incident_ids = [
|
|
str(incident.get("incident_id") or "")
|
|
for incident in incidents
|
|
if incident.get("incident_id")
|
|
]
|
|
if not incident_ids:
|
|
records = []
|
|
else:
|
|
batch_params = {"incident_ids": incident_ids, "limit": _MAX_ROWS * len(incident_ids)}
|
|
approvals = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
status::text AS status,
|
|
risk_level::text AS risk_level,
|
|
action,
|
|
description,
|
|
hit_count,
|
|
created_at,
|
|
updated_at,
|
|
resolved_at,
|
|
matched_playbook_id,
|
|
extra_metadata,
|
|
decision_fusion_details
|
|
FROM approval_records
|
|
WHERE incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
approval_to_incident = {
|
|
str(row.get("id")): str(row.get("incident_id"))
|
|
for row in approvals
|
|
if row.get("id") and row.get("incident_id")
|
|
}
|
|
approval_ids = list(approval_to_incident.keys()) or ["__none__"]
|
|
evidence_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
matched_playbook_id,
|
|
collected_at,
|
|
collection_duration_ms,
|
|
sensors_attempted,
|
|
sensors_succeeded,
|
|
evidence_summary,
|
|
metrics_snapshot,
|
|
mcp_health,
|
|
verification_result,
|
|
pre_execution_state,
|
|
post_execution_state,
|
|
self_healing_score,
|
|
self_healing_detail
|
|
FROM incident_evidence
|
|
WHERE incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
ORDER BY collected_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
timeline_events = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
event_type,
|
|
status,
|
|
title,
|
|
description,
|
|
actor,
|
|
actor_role,
|
|
risk_level,
|
|
approval_id,
|
|
incident_id,
|
|
created_at
|
|
FROM timeline_events
|
|
WHERE incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
OR approval_id::text = ANY(CAST(:approval_ids AS text[]))
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
{**batch_params, "approval_ids": approval_ids},
|
|
)
|
|
legacy_mcp_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
session_id,
|
|
flywheel_node,
|
|
mcp_server,
|
|
tool_name,
|
|
duration_ms,
|
|
success,
|
|
error_message,
|
|
incident_id,
|
|
agent_role,
|
|
created_at
|
|
FROM mcp_audit_log
|
|
WHERE incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
automation_ops = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
op_id,
|
|
operation_type,
|
|
status,
|
|
incident_id,
|
|
run_id,
|
|
parent_op_id,
|
|
actor,
|
|
dry_run_result,
|
|
error,
|
|
duration_ms,
|
|
tags,
|
|
input ->> 'action' AS input_action,
|
|
input ->> 'executor' AS input_executor,
|
|
input ->> 'execution_backend' AS input_execution_backend,
|
|
input ->> 'catalog_id' AS input_catalog_id,
|
|
input ->> 'execution_mode' AS input_execution_mode,
|
|
input ->> 'approval_source' AS input_approval_source,
|
|
input ->> 'apply_enabled' AS input_apply_enabled,
|
|
input ->> 'apply_executed' AS input_apply_executed,
|
|
input ->> 'check_mode_executed' AS input_check_mode_executed,
|
|
input ->> 'returncode' AS input_returncode,
|
|
input ->> 'playbook_id' AS input_playbook_id,
|
|
input ->> 'playbook_path' AS input_playbook_path,
|
|
input ->> 'ansible_playbook_path' AS input_ansible_playbook_path,
|
|
input ->> 'check_mode' AS input_check_mode,
|
|
input ->> 'not_used_reason' AS input_not_used_reason,
|
|
output ->> 'action' AS output_action,
|
|
output ->> 'reason' AS output_reason,
|
|
output ->> 'executor' AS output_executor,
|
|
output ->> 'execution_backend' AS output_execution_backend,
|
|
output ->> 'catalog_id' AS output_catalog_id,
|
|
output ->> 'execution_mode' AS output_execution_mode,
|
|
output ->> 'approval_source' AS output_approval_source,
|
|
output ->> 'apply_enabled' AS output_apply_enabled,
|
|
output ->> 'apply_executed' AS output_apply_executed,
|
|
output ->> 'check_mode_executed' AS output_check_mode_executed,
|
|
output ->> 'returncode' AS output_returncode,
|
|
output ->> 'playbook_id' AS output_playbook_id,
|
|
output ->> 'playbook_path' AS output_playbook_path,
|
|
output ->> 'ansible_playbook_path' AS output_ansible_playbook_path,
|
|
output ->> 'check_mode' AS output_check_mode,
|
|
output ->> 'not_used_reason' AS output_not_used_reason,
|
|
dry_run_result ->> 'returncode' AS dry_run_returncode,
|
|
dry_run_result ->> 'apply_executed' AS dry_run_apply_executed,
|
|
dry_run_result ->> 'check_mode_executed' AS dry_run_check_mode_executed,
|
|
coalesce(input::text, '') AS input_text,
|
|
coalesce(output::text, '') AS output_text,
|
|
coalesce(array_to_string(tags, ','), '') AS tags_text,
|
|
created_at
|
|
FROM automation_operation_log
|
|
WHERE incident_id::text = ANY(CAST(:incident_ids AS text[]))
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM unnest(CAST(:incident_ids AS text[])) AS source_ids(incident_id)
|
|
WHERE coalesce(input::text, '') LIKE '%' || source_ids.incident_id || '%'
|
|
OR coalesce(output::text, '') LIKE '%' || source_ids.incident_id || '%'
|
|
OR coalesce(array_to_string(tags, ','), '') LIKE '%' || source_ids.incident_id || '%'
|
|
)
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
auto_repair_executions = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
incident_id,
|
|
playbook_id,
|
|
playbook_name,
|
|
success,
|
|
executed_steps,
|
|
error_message,
|
|
triggered_by,
|
|
similarity_score,
|
|
risk_level,
|
|
execution_time_ms,
|
|
created_at
|
|
FROM auto_repair_executions
|
|
WHERE incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
km_entries = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
id,
|
|
title,
|
|
entry_type::text AS entry_type,
|
|
status::text AS status,
|
|
related_incident_id,
|
|
related_approval_id,
|
|
created_at
|
|
FROM knowledge_entries
|
|
WHERE related_incident_id = ANY(CAST(:incident_ids AS text[]))
|
|
ORDER BY created_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
batch_params,
|
|
)
|
|
gateway_mcp_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
call_id,
|
|
project_id,
|
|
run_id,
|
|
trace_id,
|
|
agent_id,
|
|
tool_name,
|
|
gate_result,
|
|
result_status,
|
|
block_gate,
|
|
block_reason,
|
|
latency_ms,
|
|
created_at
|
|
FROM awooop_mcp_gateway_audit
|
|
WHERE project_id = :project_id
|
|
AND (
|
|
trace_id = ANY(CAST(:incident_ids AS text[]))
|
|
OR run_id::text = ANY(CAST(:incident_ids AS text[]))
|
|
)
|
|
ORDER BY created_at ASC
|
|
LIMIT :limit
|
|
""",
|
|
{**batch_params, "project_id": normalized_project_id},
|
|
)
|
|
outbound_rows = await _fetch_all(
|
|
db,
|
|
"""
|
|
SELECT
|
|
message_id,
|
|
project_id,
|
|
run_id,
|
|
channel_type,
|
|
message_type,
|
|
content_hash,
|
|
content_preview,
|
|
content_redacted,
|
|
redaction_version,
|
|
source_envelope,
|
|
provider_message_id,
|
|
send_status,
|
|
queued_at,
|
|
sent_at,
|
|
triggered_by_state
|
|
FROM awooop_outbound_message
|
|
WHERE project_id = :project_id
|
|
AND (
|
|
run_id::text = ANY(CAST(:incident_ids AS text[]))
|
|
OR EXISTS (
|
|
SELECT 1
|
|
FROM unnest(CAST(:incident_ids AS text[])) AS source_ids(incident_id)
|
|
WHERE coalesce(content_preview, '') ILIKE '%' || source_ids.incident_id || '%'
|
|
OR coalesce(source_envelope::text, '') LIKE '%' || source_ids.incident_id || '%'
|
|
)
|
|
)
|
|
ORDER BY queued_at DESC
|
|
LIMIT :limit
|
|
""",
|
|
{**batch_params, "project_id": normalized_project_id},
|
|
)
|
|
|
|
approvals_by_incident = _group_rows_by_incident_reference(
|
|
rows=approvals,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("incident_id",),
|
|
)
|
|
evidence_by_incident = _group_rows_by_incident_reference(
|
|
rows=evidence_rows,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("incident_id",),
|
|
)
|
|
timeline_by_incident = _group_timeline_events_by_incident(
|
|
rows=timeline_events,
|
|
incident_ids=incident_ids,
|
|
approval_to_incident=approval_to_incident,
|
|
)
|
|
legacy_mcp_by_incident = _group_rows_by_incident_reference(
|
|
rows=legacy_mcp_rows,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("incident_id",),
|
|
)
|
|
automation_ops_by_incident = _group_rows_by_incident_reference(
|
|
rows=automation_ops,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("incident_id",),
|
|
text_fields=("input_text", "output_text", "tags_text"),
|
|
)
|
|
auto_repair_by_incident = _group_rows_by_incident_reference(
|
|
rows=auto_repair_executions,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("incident_id",),
|
|
)
|
|
km_by_incident = _group_rows_by_incident_reference(
|
|
rows=km_entries,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("related_incident_id",),
|
|
)
|
|
gateway_mcp_by_incident = _group_rows_by_incident_reference(
|
|
rows=gateway_mcp_rows,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("trace_id", "run_id"),
|
|
)
|
|
outbound_by_incident = _group_rows_by_incident_reference(
|
|
rows=outbound_rows,
|
|
incident_ids=incident_ids,
|
|
direct_fields=("run_id",),
|
|
text_fields=("content_preview",),
|
|
source_ref_keys=("incident_ids", "code_refs"),
|
|
)
|
|
records = _build_summary_quality_records(
|
|
incidents=incidents,
|
|
approvals_by_incident=approvals_by_incident,
|
|
evidence_by_incident=evidence_by_incident,
|
|
timeline_by_incident=timeline_by_incident,
|
|
legacy_mcp_by_incident=legacy_mcp_by_incident,
|
|
automation_ops_by_incident=automation_ops_by_incident,
|
|
auto_repair_by_incident=auto_repair_by_incident,
|
|
km_by_incident=km_by_incident,
|
|
gateway_mcp_by_incident=gateway_mcp_by_incident,
|
|
outbound_by_incident=outbound_by_incident,
|
|
)
|
|
|
|
summary = summarize_automation_quality_records(
|
|
project_id=normalized_project_id,
|
|
window_hours=bounded_hours,
|
|
records=records,
|
|
limit=bounded_limit,
|
|
)
|
|
logger.info(
|
|
"awooop_automation_quality_summary_fetched",
|
|
project_id=normalized_project_id,
|
|
window_hours=bounded_hours,
|
|
incident_total=summary["incident_total"],
|
|
evaluated_total=summary["evaluated_total"],
|
|
can_claim_full_auto_repair=summary["production_claim"]["can_claim_full_auto_repair"],
|
|
cache_status="miss",
|
|
cache_ttl_seconds=_QUALITY_SUMMARY_CACHE_TTL_SECONDS,
|
|
)
|
|
stored_summary = await store_operator_summary_async(
|
|
"truth_chain_quality_summary",
|
|
cache_key,
|
|
summary,
|
|
ttl_seconds=_QUALITY_SUMMARY_CACHE_TTL_SECONDS,
|
|
)
|
|
duration_seconds = perf_counter() - started_at
|
|
record_quality_summary_observation(
|
|
project_id=normalized_project_id,
|
|
hours=bounded_hours,
|
|
limit=bounded_limit,
|
|
cache_status="miss",
|
|
success=True,
|
|
duration_seconds=duration_seconds,
|
|
)
|
|
stored_summary = dict(stored_summary)
|
|
stored_summary["cache_status"] = "miss"
|
|
stored_summary["aggregation_duration_seconds"] = round(duration_seconds, 3)
|
|
return stored_summary
|