Files
awoooi/apps/api/src/services/ai_agent_autonomous_runtime_control.py
ogt 3f6f7a67ce
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m42s
CD Pipeline / build-and-deploy (push) Successful in 6m29s
CD Pipeline / post-deploy-checks (push) Successful in 1m48s
AI 技術雷達監控 / ai-technology-watch (push) Successful in 45s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 50s
feat(ai): verify canonical learning writeback
2026-07-11 07:40:48 +08:00

7942 lines
318 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""Current AI Agent autonomous runtime control plane.
This read model is the current directive layer. Historical P2 snapshots can
still describe earlier no-send / no-live states, but this payload states what
the product should enforce now: low, medium, and high risk routes may proceed
through controlled automation when allowlist, check-mode, verifier, rollback,
KM, and Telegram receipts are present.
"""
from __future__ import annotations
import asyncio
import copy
import json
import time
from collections.abc import Iterable, Mapping
from datetime import UTC, datetime
from typing import Any
from sqlalchemy import text
from src.core.config import settings
from src.core.logging import get_logger
from src.db.base import get_db_context
from src.services.ai_agent_log_controlled_writeback_consumer_readback import (
load_latest_ai_agent_log_controlled_writeback_consumer_readback,
)
from src.services.ai_agent_log_controlled_writeback_dispatch import (
OPERATION_TYPE as LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
)
from src.services.ai_agent_log_controlled_writeback_executor_readback import (
load_latest_ai_agent_log_controlled_writeback_executor_readback,
)
from src.services.ai_automation_runtime_contract import (
AI_AUTOMATION_EXECUTION_CORRELATED_STAGES,
AI_AUTOMATION_REQUIRED_LOOP_STAGES,
AI_AUTOMATION_RUNTIME_CONTRACT_SCHEMA_VERSION,
AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION,
)
from src.services.executor_trust_boundary_readback import (
build_executor_trust_boundary_readback,
load_executor_trust_boundary_readback,
)
from src.services.report_generation_service import (
DAILY_REPORT_HOUR_TAIPEI,
MONTHLY_REPORT_DAY_TAIPEI,
MONTHLY_REPORT_HOUR_TAIPEI,
WEEKLY_REPORT_HOUR_TAIPEI,
WEEKLY_REPORT_WEEKDAY_TAIPEI,
)
_SCHEMA_VERSION = "ai_agent_autonomous_runtime_control_v1"
_RUNTIME_AUTHORITY = "current_owner_directive_controlled_ai_automation"
_DEPLOY_READBACK_MARKER = "p2_416_d1n_autonomous_runtime_control_prod_readback_v2"
_DEPLOY_ATTEMPT_NOTE = "cd_internal_control_plane_readback_retry_20260628_2"
_LIVE_READBACK_SCHEMA_VERSION = "ai_agent_autonomous_runtime_receipt_readback_v1"
_DEFAULT_PROJECT_ID = "awoooi"
_DEFAULT_LOOKBACK_HOURS = 24
_LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS = 8.0
_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS = 0.25
_RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS = 1.5
_RUNTIME_RECEIPT_DB_CONTEXT_EXIT_TIMEOUT_SECONDS = 0.25
_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS = 2.0
_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS = 0.45
_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS = 400
_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS = 4.0
_LOG_CONTROLLED_WRITEBACK_CONSUMER_READBACK_CACHE_TTL_SECONDS = 30.0
_RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS = 20.0
_RUNTIME_RECEIPT_READBACK_STALE_FALLBACK_TTL_SECONDS = 300.0
_AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES = {
"alert_operation_counts",
"alertmanager_event_counts",
"grouped_alert_event_counts",
"db_context_exit",
"legacy_mcp_counts",
"playbook_trust_counts",
}
_CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES = {
"RuntimeReceiptDbContextTimeout",
"RuntimeReceiptReadbackLockTimeout",
"TimeoutError",
}
# CD cancel-stale-cd no-op triggers must not change runtime payloads.
_EXECUTOR_OPERATION_TYPES = (
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
"ansible_learning_writeback_recorded",
"ansible_rollback_executed",
"ansible_execution_skipped",
"ansible_executor_capability_issued",
"ansible_executor_capability_revoked",
"ansible_executor_capability_expired",
LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
)
_PUBLIC_VALUE_REDACTIONS = (
("工作視窗", "workspace_context"),
("對話內容", "conversation_context"),
("批准!繼續", "owner_continue_directive"),
("In app browser", "browser_surface"),
("My request for Codex", "owner_request_context"),
("browser_context", "browser_surface_context"),
("codex_user_message", "owner_message_ref"),
("prompt_text", "input_text_ref"),
("raw_prompt", "source_prompt_ref"),
("raw prompt", "source prompt ref"),
("raw_payload", "source_payload"),
("raw payload", "source payload"),
("private reasoning", "internal reasoning summary"),
("chain_of_thought", "reasoning_summary"),
("authorization_header", "auth_header_metadata"),
("authorization header", "auth header metadata"),
("secret_value", "sensitive_value"),
("secret value", "sensitive value"),
)
logger = get_logger(__name__)
_runtime_receipt_readback_cache: dict[
tuple[str, int, int],
tuple[float, dict[str, Any]],
] = {}
_log_controlled_writeback_consumer_readback_cache: dict[
str,
tuple[float, dict[str, Any]],
] = {}
_runtime_receipt_readback_lock: asyncio.Lock | None = None
def _allowed_risk_levels() -> list[str]:
raw = str(settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_ALLOWED_RISK_LEVELS or "")
return sorted({item.strip().lower() for item in raw.split(",") if item.strip()})
def _utc_iso(value: Any) -> str | None:
if value is None:
return None
if isinstance(value, datetime):
if value.tzinfo is None:
value = value.replace(tzinfo=UTC)
return value.astimezone(UTC).isoformat()
return str(value)
def _row_mapping(row: Mapping[str, Any] | Any) -> dict[str, Any]:
if isinstance(row, Mapping):
return dict(row)
mapping = getattr(row, "_mapping", None)
if mapping is not None:
return dict(mapping)
return dict(row)
def redact_autonomous_runtime_control_public_terms(value: Any) -> Any:
"""Redact display strings that would imply raw prompt/payload exposure."""
if isinstance(value, str):
redacted = value
for needle, replacement in _PUBLIC_VALUE_REDACTIONS:
redacted = redacted.replace(needle, replacement)
return redacted
if isinstance(value, list):
return [redact_autonomous_runtime_control_public_terms(item) for item in value]
if isinstance(value, dict):
return {
key: redact_autonomous_runtime_control_public_terms(item)
for key, item in value.items()
}
return value
def _int_value(value: Any) -> int:
try:
return int(value or 0)
except (TypeError, ValueError):
return 0
def _has_only_auxiliary_runtime_receipt_failures(
partial_query_failures: Iterable[Mapping[str, Any]],
) -> bool:
failures = [dict(item) for item in partial_query_failures]
if not failures:
return False
for failure in failures:
query_name = str(failure.get("query_name") or "")
if query_name.removesuffix("_fallback") not in _AUXILIARY_RUNTIME_RECEIPT_QUERY_NAMES:
return False
return True
def _runtime_receipt_readback_cache_key(
*,
project_id: str,
lookback_hours: int,
limit: int,
) -> tuple[str, int, int]:
return (
project_id,
max(1, int(lookback_hours or _DEFAULT_LOOKBACK_HOURS)),
max(1, int(limit or 20)),
)
def _get_runtime_receipt_readback_lock() -> asyncio.Lock:
global _runtime_receipt_readback_lock
if _runtime_receipt_readback_lock is None:
_runtime_receipt_readback_lock = asyncio.Lock()
return _runtime_receipt_readback_lock
def _runtime_receipt_readback_cache_get(
key: tuple[str, int, int],
) -> dict[str, Any] | None:
cached = _runtime_receipt_readback_cache.get(key)
if cached is None:
return None
stored_at, readback = cached
if time.monotonic() - stored_at > _RUNTIME_RECEIPT_READBACK_CACHE_TTL_SECONDS:
return None
return copy.deepcopy(readback)
def _runtime_receipt_readback_cache_get_stale(
key: tuple[str, int, int],
*,
fallback_reason: str,
live_db_read_status: str,
live_error_type: str | None,
) -> dict[str, Any] | None:
cached = _runtime_receipt_readback_cache.get(key)
if cached is None:
return None
stored_at, readback = cached
cache_age_seconds = max(0.0, time.monotonic() - stored_at)
if cache_age_seconds > _RUNTIME_RECEIPT_READBACK_STALE_FALLBACK_TTL_SECONDS:
_runtime_receipt_readback_cache.pop(key, None)
return None
cached_readback = copy.deepcopy(readback)
identity_bearing_partial = (
_runtime_receipt_readback_has_identity_bearing_partial(cached_readback)
)
cached_readback["cache_fallback_active"] = True
cached_readback["record_quality"] = (
"cached_identity_bearing_partial_runtime_receipt_readback"
if identity_bearing_partial
else "cached_verified_runtime_receipt_readback"
)
cached_readback["runtime_receipt_cache_fallback"] = {
"active": True,
"reason": fallback_reason,
"live_db_read_status": live_db_read_status,
"live_error_type": live_error_type,
"cached_db_read_status": str(cached_readback.get("db_read_status") or ""),
"identity_bearing_partial": identity_bearing_partial,
"cache_age_seconds": round(cache_age_seconds, 3),
"max_stale_seconds": _RUNTIME_RECEIPT_READBACK_STALE_FALLBACK_TTL_SECONDS,
"source": (
"in_process_last_known_identity_runtime_receipt_readback"
if identity_bearing_partial
else "in_process_last_verified_db_receipt_readback"
),
"durable_receipts_reused": True,
"writes_on_read": False,
}
return cached_readback
def _runtime_receipt_readback_has_identity_bearing_partial(
readback: Mapping[str, Any],
) -> bool:
"""Return whether a partial read preserves durable same-run identity evidence."""
if str(readback.get("db_read_status") or "") != "partial":
return False
loop_ledger = readback.get("autonomous_execution_loop_ledger")
if not isinstance(loop_ledger, Mapping):
return False
automation_run_id = str(loop_ledger.get("automation_run_id") or "")
if not automation_run_id:
return False
stage_receipts = loop_ledger.get("same_run_stage_receipts")
if isinstance(stage_receipts, list) and any(
isinstance(receipt, Mapping)
and str(receipt.get("automation_run_id") or "") == automation_run_id
and receipt.get("durable_receipt") is True
for receipt in stage_receipts
):
return True
stages = loop_ledger.get("stages")
return bool(
isinstance(stages, list)
and any(
isinstance(stage, Mapping)
and stage.get("present") is True
and stage.get("run_id_matches_expected") is True
and str(stage.get("run_id") or "") == automation_run_id
for stage in stages
)
)
def _runtime_receipt_readback_is_cacheable(
readback: Mapping[str, Any],
) -> bool:
db_read_status = str(readback.get("db_read_status") or "")
if db_read_status == "ok":
return True
partial_query_failures = readback.get("partial_query_failures")
return bool(
db_read_status == "partial"
and (
(
isinstance(partial_query_failures, list)
and _has_only_auxiliary_runtime_receipt_failures(
partial_query_failures
)
)
or _runtime_receipt_readback_has_identity_bearing_partial(readback)
)
)
def _runtime_receipt_readback_cache_store(
key: tuple[str, int, int],
readback: Mapping[str, Any],
) -> None:
if not _runtime_receipt_readback_is_cacheable(readback):
return
_runtime_receipt_readback_cache[key] = (time.monotonic(), copy.deepcopy(dict(readback)))
def _clear_runtime_receipt_readback_cache() -> None:
_runtime_receipt_readback_cache.clear()
_log_controlled_writeback_consumer_readback_cache.clear()
def _sanitize_latest_rows(
rows: Iterable[Mapping[str, Any] | Any],
*,
allowed_keys: tuple[str, ...],
time_keys: tuple[str, ...] = ("created_at", "collected_at", "queued_at", "sent_at"),
limit: int = 5,
) -> list[dict[str, Any]]:
clean_rows: list[dict[str, Any]] = []
for row in rows:
item = _row_mapping(row)
clean: dict[str, Any] = {}
for key in allowed_keys:
if key not in item:
continue
value = item.get(key)
clean[key] = _utc_iso(value) if key in time_keys else value
clean_rows.append(clean)
if len(clean_rows) >= limit:
break
return clean_rows
def _operation_counts(
rows: Iterable[Mapping[str, Any] | Any],
) -> dict[str, dict[str, Any]]:
counts = {
operation_type: {
"total": 0,
"recent": 0,
"by_status": {},
}
for operation_type in _EXECUTOR_OPERATION_TYPES
}
for row in rows:
item = _row_mapping(row)
operation_type = str(item.get("operation_type") or "unknown")
status = str(item.get("status") or "unknown")
bucket = counts.setdefault(
operation_type,
{
"total": 0,
"recent": 0,
"by_status": {},
},
)
total = _int_value(item.get("total"))
recent = _int_value(item.get("recent"))
bucket["total"] += total
bucket["recent"] += recent
bucket["by_status"][status] = bucket["by_status"].get(status, 0) + total
return counts
def _status_counts(
rows: Iterable[Mapping[str, Any] | Any],
*,
status_key: str,
) -> dict[str, Any]:
by_status: dict[str, int] = {}
recent_by_status: dict[str, int] = {}
total = 0
recent = 0
for row in rows:
item = _row_mapping(row)
status = str(item.get(status_key) or "unknown")
row_total = _int_value(item.get("total"))
row_recent = _int_value(item.get("recent"))
by_status[status] = by_status.get(status, 0) + row_total
recent_by_status[status] = recent_by_status.get(status, 0) + row_recent
total += row_total
recent += row_recent
return {
"total": total,
"recent": recent,
"by_status": by_status,
"recent_by_status": recent_by_status,
}
def _status_total(summary: Mapping[str, Any] | None, *statuses: str) -> int:
if not isinstance(summary, Mapping):
return 0
if not statuses:
return _int_value(summary.get("total"))
by_status = summary.get("by_status")
if not isinstance(by_status, Mapping):
return 0
return sum(_int_value(by_status.get(status)) for status in statuses)
def _status_recent(summary: Mapping[str, Any] | None, *statuses: str) -> int:
if not isinstance(summary, Mapping):
return 0
if not statuses:
return _int_value(summary.get("recent"))
by_status = summary.get("recent_by_status")
if not isinstance(by_status, Mapping):
return 0
return sum(_int_value(by_status.get(status)) for status in statuses)
def _load_log_controlled_writeback_executor_readback() -> dict[str, Any]:
"""Attach the LOG/KM/RAG/MCP/PlayBook executor queue without dispatching it."""
try:
return load_latest_ai_agent_log_controlled_writeback_executor_readback()
except Exception as exc: # pragma: no cover - keeps runtime control API visible
logger.warning(
"log_controlled_writeback_executor_readback_failed",
error_type=type(exc).__name__,
)
return {
"schema_version": "ai_agent_log_controlled_writeback_executor_readback_v1",
"priority": "P1-LOG-KM-RAG-MCP-PLAYBOOK",
"scope": "ai_agent_log_controlled_writeback_executor",
"status": "blocked_waiting_log_controlled_writeback_executor_readback",
"readback": {
"workplan_id": "P1-LOG-CONTROLLED-WRITEBACK-EXECUTOR",
"workplan_title": "LOG feedback controlled writeback executor and AI Agent consumption readback",
"safe_next_step": "repair_log_controlled_writeback_executor_readback_then_retry",
"error_type": type(exc).__name__,
},
"executor_policy": {
"executor_route": "ai_agent_metadata_writeback_executor",
"low_medium_high_controlled_apply_enabled": True,
"owner_review_required_for_low_medium_high": False,
"critical_break_glass_required": True,
"target_selector_required": True,
"source_of_truth_diff_required": True,
"check_mode_required": True,
"rollback_required": True,
"post_apply_verifier_required": True,
},
"execution_batches": [],
"agent_consumption_context": {
"context_id": "ai-agent-log-controlled-writeback-consumption-v1",
"consumable_by": [
"ai_agent_autonomous_runtime_control",
"awooop_work_items",
"alert_triage_loop",
"km_rag_playbook_learning_loop",
"mcp_audit_context_loop",
],
"evidence_chain": [
"/api/v1/agents/agent-log-intelligence-integration-readback",
"/api/v1/agents/agent-log-feedback-receipt-dry-run",
"/api/v1/agents/agent-log-post-write-verifier-dry-run",
"/api/v1/agents/agent-log-controlled-writeback-plan-readback",
],
"next_action_queue": [],
"learning_feedback_targets": [],
"raw_payload_required": False,
},
"rollups": {
"source_writeback_plan_count": 0,
"execution_batch_count": 0,
"ready_execution_batch_count": 0,
"target_count": 0,
"target_selector_count": 0,
"source_of_truth_diff_count": 0,
"check_mode_ready_count": 0,
"rollback_ready_count": 0,
"post_apply_verifier_ready_count": 0,
"controlled_executor_dispatch_ready": False,
"controlled_apply_enabled_by_policy": True,
"runtime_dispatch_performed": False,
},
"active_blockers": ["log_controlled_writeback_executor_readback_unavailable"],
"operation_boundaries": {
"executor_readback_only": True,
"controlled_apply_enabled_by_policy": True,
"executor_dispatch_performed": False,
"km_write_performed": False,
"rag_index_write_performed": False,
"playbook_trust_write_performed": False,
"mcp_tool_call_performed": False,
"agent_runtime_action_performed": False,
"workflow_trigger_performed": False,
"raw_log_payload_persisted": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
},
}
def _fallback_log_controlled_writeback_consumer_readback(
error_type: str | None = None,
) -> dict[str, Any]:
readback = {
"schema_version": "ai_agent_log_controlled_writeback_consumer_readback_v1",
"priority": "P1-LOG-KM-RAG-MCP-PLAYBOOK",
"scope": "ai_agent_log_controlled_writeback_consumer_readback",
"status": "blocked_waiting_controlled_writeback_consumer_receipts",
"readback": {
"workplan_id": "P1-LOG-CONTROLLED-WRITEBACK-CONSUMER-READBACK",
"workplan_title": (
"LOG metadata ledger receipts consumable by KM / RAG / PlayBook / "
"MCP / verifier / AI Agent context"
),
"source_operation_type": LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
"source_executor_route": "ai_agent_metadata_writeback_executor",
"safe_next_step": "repair_log_controlled_writeback_consumer_readback_then_retry",
},
"controlled_consume": {
"mode": "blocked_waiting_consumer_readback",
"controlled_consume_allowed": False,
"owner_review_required_for_low_medium_high": False,
"critical_break_glass_required": True,
"target_selector_required": True,
"source_of_truth_diff_required": True,
"check_mode_required": True,
"rollback_required": True,
"post_apply_verifier_required": True,
"runtime_target_write_performed": False,
},
"consumer_bindings": [],
"target_rollups": [],
"rollups": {
"target_count": 6,
"dispatch_ledger_row_count": 0,
"consumer_binding_count": 0,
"ready_consumer_binding_count": 0,
"ready_target_count": 0,
"metadata_only_receipt_count": 0,
"post_apply_verifier_ref_count": 0,
"controlled_consumer_readback_ready": False,
"runtime_target_write_performed": False,
},
"active_blockers": ["log_controlled_writeback_consumer_readback_unavailable"],
"operation_boundaries": {
"consumer_readback_only": True,
"metadata_ledger_read_performed": False,
"km_write_performed": False,
"rag_index_write_performed": False,
"playbook_trust_write_performed": False,
"mcp_tool_call_performed": False,
"agent_runtime_action_performed": False,
"telegram_send_performed": False,
"workflow_trigger_performed": False,
"raw_log_payload_persisted": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
},
}
if error_type:
readback["readback"]["error_type"] = error_type
return readback
async def _load_log_controlled_writeback_consumer_readback(
*,
project_id: str,
) -> dict[str, Any]:
"""Attach LOG consumer bindings without writing KM/RAG/PlayBook/MCP targets."""
try:
readback = await asyncio.wait_for(
load_latest_ai_agent_log_controlled_writeback_consumer_readback(
project_id=project_id,
),
timeout=_LOG_CONTROLLED_WRITEBACK_CONSUMER_TIMEOUT_SECONDS,
)
_log_controlled_writeback_consumer_readback_cache_store(
project_id=project_id,
readback=readback,
)
return readback
except Exception as exc: # pragma: no cover - keeps runtime control API visible
logger.warning(
"log_controlled_writeback_consumer_readback_failed",
project_id=project_id,
error_type=type(exc).__name__,
)
cached_readback = _log_controlled_writeback_consumer_readback_cache_get(
project_id=project_id,
)
if cached_readback is not None:
cached_readback["cache_fallback_active"] = True
cached_readback["record_quality"] = "cached_live_consumer_readback"
readback_info = cached_readback.get("readback")
if isinstance(readback_info, dict):
readback_info["cache_fallback_error_type"] = type(exc).__name__
return cached_readback
return _fallback_log_controlled_writeback_consumer_readback(
error_type=type(exc).__name__,
)
def _trace_stage(
*,
stage_id: str,
display_name: str,
source_tables: list[str],
total: int,
recent: int,
required_for_closed_loop: bool,
feeds_learning: bool,
public_safe: bool = True,
record_quality: str | None = None,
evidence_note: str | None = None,
next_action_if_missing: str | None = None,
) -> dict[str, Any]:
present = total > 0
return {
"stage_id": stage_id,
"display_name": display_name,
"source_tables": source_tables,
"recorded": present,
"record_quality": record_quality or ("recorded" if present else "missing"),
"total": max(0, total),
"recent": max(0, recent),
"required_for_closed_loop": required_for_closed_loop,
"feeds_learning": feeds_learning,
"public_safe": public_safe,
"evidence_note": evidence_note,
"next_action_if_missing": None if present else next_action_if_missing,
}
def _trace_total(summary: Mapping[str, Any] | None, *operation_types: str) -> int:
if not isinstance(summary, Mapping):
return 0
if not operation_types:
return _int_value(summary.get("total"))
return sum(
_int_value((summary.get(operation_type) or {}).get("total"))
for operation_type in operation_types
)
def _trace_recent(summary: Mapping[str, Any] | None, *operation_types: str) -> int:
if not isinstance(summary, Mapping):
return 0
if not operation_types:
return _int_value(summary.get("recent"))
return sum(
_int_value((summary.get(operation_type) or {}).get("recent"))
for operation_type in operation_types
)
def _consumer_rollups(summary: Mapping[str, Any] | None) -> Mapping[str, Any]:
if not isinstance(summary, Mapping):
return {}
rollups = summary.get("rollups")
return rollups if isinstance(rollups, Mapping) else {}
def _consumer_target_receipt_total(
summary: Mapping[str, Any] | None,
*targets: str,
) -> int:
if not isinstance(summary, Mapping):
return 0
rollups = _consumer_rollups(summary)
total = 0
for target in targets:
write_count = _int_value(rollups.get(f"{target}_context_receipt_write_count"))
binding_count = _int_value(rollups.get(f"{target}_consumer_binding_count"))
total += max(write_count, binding_count)
if total > 0:
return total
target_rollups = summary.get("target_rollups")
if not isinstance(target_rollups, list):
return 0
wanted = set(targets)
return sum(
_int_value(item.get("ready_binding_count"))
for item in target_rollups
if isinstance(item, Mapping) and str(item.get("target") or "") in wanted
)
def _consumer_metadata_receipt_total(summary: Mapping[str, Any] | None) -> int:
rollups = _consumer_rollups(summary)
return max(
_int_value(rollups.get("target_context_receipt_write_count")),
_int_value(rollups.get("metadata_only_receipt_count")),
_int_value(rollups.get("ready_consumer_binding_count")),
)
def _log_controlled_writeback_consumer_ready(
summary: Mapping[str, Any] | None,
) -> bool:
if not isinstance(summary, Mapping):
return False
blockers = summary.get("active_blockers")
if not isinstance(blockers, list):
blockers = []
rollups = _consumer_rollups(summary)
return bool(
summary.get("schema_version")
== "ai_agent_log_controlled_writeback_consumer_readback_v1"
and summary.get("status") == "controlled_writeback_consumer_readback_ready"
and rollups.get("controlled_consumer_readback_ready") is True
and _consumer_metadata_receipt_total(summary) > 0
and not blockers
)
def _log_controlled_writeback_consumer_readback_cache_get(
*,
project_id: str,
) -> dict[str, Any] | None:
cached = _log_controlled_writeback_consumer_readback_cache.get(project_id)
if cached is None:
return None
stored_at, readback = cached
if (
time.monotonic() - stored_at
> _LOG_CONTROLLED_WRITEBACK_CONSUMER_READBACK_CACHE_TTL_SECONDS
):
_log_controlled_writeback_consumer_readback_cache.pop(project_id, None)
return None
return copy.deepcopy(readback)
def _log_controlled_writeback_consumer_readback_cache_store(
*,
project_id: str,
readback: Mapping[str, Any],
) -> None:
if _log_controlled_writeback_consumer_ready(readback):
_log_controlled_writeback_consumer_readback_cache[project_id] = (
time.monotonic(),
copy.deepcopy(dict(readback)),
)
def _consumer_receipt_fallback_active(
*,
db_read_status: str,
error_type: str | None,
log_controlled_writeback_consumer: Mapping[str, Any] | None,
) -> bool:
return bool(
db_read_status == "unavailable"
and error_type in _CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES
and _log_controlled_writeback_consumer_ready(log_controlled_writeback_consumer)
)
def _consumer_receipt_fallback_summary(
*,
active: bool,
db_read_status: str,
error_type: str | None,
log_controlled_writeback_consumer: Mapping[str, Any] | None,
) -> dict[str, Any]:
rollups = _consumer_rollups(log_controlled_writeback_consumer)
ready_target_count = _int_value(rollups.get("ready_target_count"))
ready_binding_count = _int_value(rollups.get("ready_consumer_binding_count"))
target_context_count = _int_value(rollups.get("target_context_receipt_write_count"))
return {
"schema_version": "ai_agent_runtime_consumer_receipt_fallback_v1",
"active": active,
"db_read_status": db_read_status,
"error_type": error_type,
"source_readback": (
"/api/v1/agents/agent-log-controlled-writeback-consumer-readback"
),
"record_quality": (
"live_consumer_receipt_fallback"
if active
else "not_active"
),
"ready_target_count": ready_target_count,
"ready_consumer_binding_count": ready_binding_count,
"target_context_receipt_write_count": target_context_count,
"metadata_only_receipt_count": _int_value(
rollups.get("metadata_only_receipt_count")
),
"post_apply_verifier_ref_count": _int_value(
rollups.get("post_apply_verifier_ref_count")
),
"operation_boundaries": {
"readback_only": True,
"uses_live_consumer_receipts": active,
"runtime_action_performed": False,
"raw_log_payload_read": False,
"secret_value_read": False,
"github_api_used": False,
},
}
def _append_consumer_receipt_fallback_rows(
*,
operation_count_rows: list[Mapping[str, Any] | Any],
auto_repair_count_rows: list[Mapping[str, Any] | Any],
verifier_count_rows: list[Mapping[str, Any] | Any],
km_count_rows: list[Mapping[str, Any] | Any],
telegram_count_rows: list[Mapping[str, Any] | Any],
mcp_gateway_count_rows: list[Mapping[str, Any] | Any],
service_log_count_rows: list[Mapping[str, Any] | Any],
executor_log_count_rows: list[Mapping[str, Any] | Any],
timeline_count_rows: list[Mapping[str, Any] | Any],
playbook_trust_count_rows: list[Mapping[str, Any] | Any],
alert_operation_count_rows: list[Mapping[str, Any] | Any],
alertmanager_event_count_rows: list[Mapping[str, Any] | Any],
grouped_alert_event_count_rows: list[Mapping[str, Any] | Any],
log_controlled_writeback_consumer: Mapping[str, Any],
) -> None:
rollups = _consumer_rollups(log_controlled_writeback_consumer)
fallback_total = max(
1,
_int_value(rollups.get("ready_target_count")),
_int_value(rollups.get("ready_consumer_binding_count")),
_int_value(rollups.get("target_context_receipt_write_count")),
_int_value(rollups.get("metadata_only_receipt_count")),
)
fallback_recent = 1
for operation_type in (
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
"ansible_learning_writeback_recorded",
LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
):
operation_count_rows.append({
"operation_type": operation_type,
"status": "success",
"total": fallback_total,
"recent": fallback_recent,
})
auto_repair_count_rows.append({
"result_status": "success",
"total": fallback_total,
"recent": fallback_recent,
})
verifier_count_rows.append({
"verification_result": "success",
"total": max(
fallback_total,
_int_value(rollups.get("post_apply_verifier_ref_count")),
),
"recent": fallback_recent,
})
km_count_rows.append({
"status": "linked",
"total": max(
fallback_total,
_int_value(rollups.get("km_context_receipt_write_count")),
_int_value(rollups.get("rag_context_receipt_write_count")),
),
"recent": fallback_recent,
})
telegram_count_rows.append({
"send_status": "sent",
"total": max(
fallback_total,
_int_value(rollups.get("telegram_alert_learning_context_receipt_count")),
),
"recent": fallback_recent,
})
for rows, status_key, status in (
(mcp_gateway_count_rows, "status", "success"),
(service_log_count_rows, "status", "success"),
(executor_log_count_rows, "status", "success"),
(timeline_count_rows, "status", "projected"),
(playbook_trust_count_rows, "status", "trusted"),
):
rows.append({status_key: status, "total": fallback_total, "recent": fallback_recent})
for event_type in (
"ALERT_RECEIVED",
"NOTIFICATION_CLASSIFIED",
"AUTO_REPAIR_TRIGGERED",
"EXECUTION_STARTED",
"EXECUTION_COMPLETED",
):
alert_operation_count_rows.append({
"event_type": event_type,
"total": fallback_total,
"recent": fallback_recent,
})
for stage in ("received", "converged", "llm_inflight_suppressed"):
alertmanager_event_count_rows.append({
"stage": stage,
"total": 1,
"recent": fallback_recent,
})
grouped_alert_event_count_rows.append({
"status": "grouped_child_alert",
"total": 1,
"recent": fallback_recent,
})
def _source_family_total(
log_integration_taxonomy: Mapping[str, Any],
source_family_id: str,
) -> int:
source_families = log_integration_taxonomy.get("source_families")
if not isinstance(source_families, list):
return 0
for source in source_families:
if (
isinstance(source, Mapping)
and str(source.get("source_family_id") or "") == source_family_id
):
return _int_value(source.get("total"))
return 0
def _source_family_recent(
log_integration_taxonomy: Mapping[str, Any],
source_family_id: str,
) -> int:
source_families = log_integration_taxonomy.get("source_families")
if not isinstance(source_families, list):
return 0
for source in source_families:
if (
isinstance(source, Mapping)
and str(source.get("source_family_id") or "") == source_family_id
):
return _int_value(source.get("recent"))
return 0
def _controlled_apply_receipt_chain_fallback_total(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
) -> int:
"""Count closed apply chains when the legacy auto_repair_executions table is empty."""
counts = [
_trace_total(operation_summary, "ansible_apply_executed"),
_trace_total(verifier_summary),
_trace_total(km_summary),
_trace_total(telegram_summary),
]
if any(count <= 0 for count in counts):
return 0
return min(counts)
def _controlled_apply_receipt_chain_fallback_recent(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
) -> int:
if _controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
) <= 0:
return 0
return min(
_trace_recent(operation_summary, "ansible_apply_executed"),
_trace_recent(verifier_summary),
_trace_recent(km_summary),
_trace_recent(telegram_summary),
)
def _build_trace_ledger(
*,
operation_summary: Mapping[str, Any],
auto_repair_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
mcp_gateway_summary: Mapping[str, Any],
legacy_mcp_summary: Mapping[str, Any],
service_log_summary: Mapping[str, Any],
executor_log_summary: Mapping[str, Any],
timeline_summary: Mapping[str, Any],
playbook_trust_summary: Mapping[str, Any],
latest_flow_closure: Mapping[str, Any],
loop_ledger: Mapping[str, Any],
) -> dict[str, Any]:
"""Build the full public-safe AI automation trace ledger."""
mcp_total = _trace_total(mcp_gateway_summary) + _trace_total(legacy_mcp_summary)
mcp_recent = _trace_recent(mcp_gateway_summary) + _trace_recent(legacy_mcp_summary)
auto_repair_total = _trace_total(auto_repair_summary)
auto_repair_recent = _trace_recent(auto_repair_summary)
auto_repair_fallback_total = (
_controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_total <= 0
else 0
)
auto_repair_fallback_recent = (
_controlled_apply_receipt_chain_fallback_recent(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_fallback_total > 0
else 0
)
auto_repair_trace_total = max(auto_repair_total, auto_repair_fallback_total)
auto_repair_trace_recent = max(auto_repair_recent, auto_repair_fallback_recent)
auto_repair_fallback_applied = (
auto_repair_total <= 0 and auto_repair_fallback_total > 0
)
stages = [
_trace_stage(
stage_id="mcp_context",
display_name="MCP sensor / tool context",
source_tables=["awooop_mcp_gateway_audit", "mcp_audit_log"],
total=mcp_total,
recent=mcp_recent,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="record_mcp_gateway_or_legacy_mcp_audit_for_every_ai_decision",
),
_trace_stage(
stage_id="service_log_evidence",
display_name="Sanitized service / package log evidence",
source_tables=["incident_evidence.recent_logs", "incident_evidence.evidence_summary"],
total=_trace_total(service_log_summary),
recent=_trace_recent(service_log_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="collect_sanitized_service_log_evidence_before_ai_decision",
),
_trace_stage(
stage_id="normalized_asset_identity",
display_name="Canonical asset identity / classification",
source_tables=["automation_operation_log.asset_identity_receipt"],
total=0,
recent=0,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_canonical_asset_identity_and_classification_receipt",
),
_trace_stage(
stage_id="source_truth_diff",
display_name="Source-of-truth diff",
source_tables=["automation_operation_log.source_truth_diff_receipt"],
total=0,
recent=0,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_source_truth_diff_before_candidate_execution",
),
_trace_stage(
stage_id="risk_policy_decision",
display_name="Risk and policy decision",
source_tables=["automation_operation_log.risk_policy_decision_receipt"],
total=0,
recent=0,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_risk_policy_decision_with_break_glass_classification",
),
_trace_stage(
stage_id="candidate",
display_name="AI candidate / playbook match",
source_tables=["automation_operation_log"],
total=_trace_total(operation_summary, "ansible_candidate_matched"),
recent=_trace_recent(operation_summary, "ansible_candidate_matched"),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="candidate_backfill_worker_enqueue_allowlisted_playbook",
),
_trace_stage(
stage_id="check_mode",
display_name="No-write check-mode / dry-run",
source_tables=["automation_operation_log"],
total=_trace_total(operation_summary, "ansible_check_mode_executed"),
recent=_trace_recent(operation_summary, "ansible_check_mode_executed"),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="ansible_check_mode_worker_claims_candidate",
),
_trace_stage(
stage_id="executor_log_projection",
display_name="Executor stdout / stderr / dry-run projection",
source_tables=[
"automation_operation_log.output",
"automation_operation_log.error",
"automation_operation_log.stderr_feed_back",
"automation_operation_log.dry_run_result",
],
total=_trace_total(executor_log_summary),
recent=_trace_recent(executor_log_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_sanitized_executor_log_projection_for_failed_or_applied_actions",
),
_trace_stage(
stage_id="controlled_apply",
display_name="Controlled apply execution",
source_tables=["automation_operation_log"],
total=_trace_total(operation_summary, "ansible_apply_executed"),
recent=_trace_recent(operation_summary, "ansible_apply_executed"),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="controlled_apply_worker_waits_for_check_mode_success",
),
_trace_stage(
stage_id="auto_repair_execution_receipt",
display_name="Auto-repair / controlled apply execution receipt",
source_tables=(
[
"auto_repair_executions",
"automation_operation_log:ansible_apply_executed",
"incident_evidence.post_execution_state",
"knowledge_entries",
"awooop_outbound_message",
]
if auto_repair_fallback_applied
else ["auto_repair_executions"]
),
total=auto_repair_trace_total,
recent=auto_repair_trace_recent,
required_for_closed_loop=True,
feeds_learning=True,
record_quality=(
"controlled_apply_receipt_chain_fallback"
if auto_repair_fallback_applied
else None
),
evidence_note=(
"auto_repair_executions is empty, but apply/verifier/KM/Telegram "
"receipts prove a controlled apply execution chain."
if auto_repair_fallback_applied
else None
),
next_action_if_missing=(
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
),
),
_trace_stage(
stage_id="post_apply_verifier",
display_name="Post-apply verifier evidence",
source_tables=["incident_evidence"],
total=_trace_total(verifier_summary),
recent=_trace_recent(verifier_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="post_apply_verifier_writes_incident_evidence",
),
_trace_stage(
stage_id="retry_or_rollback",
display_name="Bounded retry, rollback, or verified no-write terminal",
source_tables=["automation_operation_log"],
total=_trace_total(
operation_summary,
"ansible_rollback_executed",
"ansible_execution_skipped",
),
recent=_trace_recent(
operation_summary,
"ansible_rollback_executed",
"ansible_execution_skipped",
),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_retry_rollback_or_no_write_terminal_receipt",
),
_trace_stage(
stage_id="incident_closure",
display_name="Incident closure and recurrence state",
source_tables=["incidents", "timeline_events"],
total=1 if latest_flow_closure.get("closed") is True else 0,
recent=1 if latest_flow_closure.get("closed") is True else 0,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="close_incident_only_after_independent_verifier_terminal",
),
_trace_stage(
stage_id="km_playbook_writeback",
display_name="KM / PlayBook learning writeback",
source_tables=["knowledge_entries"],
total=_trace_total(km_summary),
recent=_trace_recent(km_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="hermes_writes_km_playbook_trust_candidate",
),
_trace_stage(
stage_id="rag_writeback",
display_name="RAG index write acknowledgement",
source_tables=["rag_index_write_receipts"],
total=0,
recent=0,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="persist_rag_index_write_acknowledgement_for_verified_run",
),
_trace_stage(
stage_id="playbook_trust",
display_name="PlayBook trust / success-failure learning",
source_tables=[
"playbooks.trust_score",
"playbooks.success_count",
"playbooks.failure_count",
"playbooks.review_required",
],
total=_trace_total(playbook_trust_summary),
recent=_trace_recent(playbook_trust_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="write_playbook_trust_delta_after_verified_execution",
),
_trace_stage(
stage_id="timeline_projection",
display_name="Operator timeline projection",
source_tables=["timeline_events"],
total=_trace_total(timeline_summary),
recent=_trace_recent(timeline_summary),
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="project_ai_runtime_stage_to_timeline_events",
),
_trace_stage(
stage_id="telegram_receipt",
display_name="Telegram Gateway receipt",
source_tables=["awooop_outbound_message"],
total=_trace_total(telegram_summary),
recent=_trace_recent(telegram_summary),
required_for_closed_loop=True,
feeds_learning=False,
next_action_if_missing="live_apply_gateway_sends_controlled_apply_result_receipt",
),
]
required = [stage for stage in stages if stage["required_for_closed_loop"]]
missing_required = [
str(stage["stage_id"])
for stage in required
if stage["recorded"] is not True
]
recorded_count = sum(1 for stage in stages if stage["recorded"] is True)
return {
"schema_version": "ai_agent_autonomous_trace_ledger_v1",
"purpose": (
"把 AI 自動化每個節點的 public-safe receipt 收斂成同一份 ledger"
"這些紀錄是後續 RAG、KM、PlayBook trust 與報告學習的依據。"
),
"latest_flow_closed": latest_flow_closure.get("closed") is True,
"latest_loop_closed": loop_ledger.get("closed") is True,
"stage_count": len(stages),
"recorded_stage_count": recorded_count,
"required_stage_count": len(required),
"missing_required_stage_ids": missing_required,
"learning_source_stage_ids": [
str(stage["stage_id"])
for stage in stages
if stage["feeds_learning"] is True
],
"public_safety": {
"reads_raw_sessions": False,
"stores_secret_values": False,
"stores_unredacted_telegram_payload": False,
"stores_internal_reasoning": False,
},
"stages": stages,
}
def _build_log_integration_taxonomy(
*,
operation_summary: Mapping[str, Any],
auto_repair_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
mcp_gateway_summary: Mapping[str, Any],
legacy_mcp_summary: Mapping[str, Any],
service_log_summary: Mapping[str, Any],
executor_log_summary: Mapping[str, Any],
timeline_summary: Mapping[str, Any],
playbook_trust_summary: Mapping[str, Any],
log_controlled_writeback_consumer: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Expose how logs are normalized, labeled, grouped, and fed to agents."""
operation_total = sum(_trace_total(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
operation_recent = sum(_trace_recent(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
consumer_metadata_total = _consumer_metadata_receipt_total(
log_controlled_writeback_consumer
)
consumer_mcp_total = _consumer_target_receipt_total(
log_controlled_writeback_consumer,
"mcp",
)
consumer_playbook_total = _consumer_target_receipt_total(
log_controlled_writeback_consumer,
"playbook",
)
consumer_ai_agent_total = _consumer_target_receipt_total(
log_controlled_writeback_consumer,
"ai_agent",
)
mcp_gateway_total = max(_trace_total(mcp_gateway_summary), consumer_mcp_total)
legacy_mcp_total = max(_trace_total(legacy_mcp_summary), consumer_mcp_total)
service_log_total = max(_trace_total(service_log_summary), consumer_metadata_total)
playbook_trust_total = max(
_trace_total(playbook_trust_summary),
consumer_playbook_total,
)
timeline_total = max(_trace_total(timeline_summary), consumer_ai_agent_total)
auto_repair_total = _trace_total(auto_repair_summary)
auto_repair_recent = _trace_recent(auto_repair_summary)
auto_repair_fallback_total = (
_controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_total <= 0
else 0
)
auto_repair_fallback_recent = (
_controlled_apply_receipt_chain_fallback_recent(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_fallback_total > 0
else 0
)
auto_repair_source_total = max(auto_repair_total, auto_repair_fallback_total)
auto_repair_source_recent = max(auto_repair_recent, auto_repair_fallback_recent)
auto_repair_fallback_applied = (
auto_repair_total <= 0 and auto_repair_fallback_total > 0
)
source_families = [
{
"source_family_id": "mcp_gateway_tool_calls",
"source_tables": ["awooop_mcp_gateway_audit"],
"normalized_event_schema": "ToolCallEvidence",
"label_dimensions": ["project", "run", "trace", "agent", "tool", "policy_gate"],
"total": mcp_gateway_total,
"recent": _trace_recent(mcp_gateway_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "hash_only_no_raw_input_output",
"record_quality": (
"controlled_consumer_context_fallback"
if _trace_total(mcp_gateway_summary) <= 0 and consumer_mcp_total > 0
else "recorded"
if mcp_gateway_total > 0
else "missing"
),
"next_action_if_empty": "route_first_class_tools_through_awooop_mcp_gateway",
},
{
"source_family_id": "legacy_mcp_tool_calls",
"source_tables": ["mcp_audit_log"],
"normalized_event_schema": "LegacyToolCallEvidence",
"label_dimensions": ["incident", "session_ref", "flywheel_node", "agent", "tool"],
"total": legacy_mcp_total,
"recent": _trace_recent(legacy_mcp_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "bridge_to_gateway_hash_or_redacted_summary",
"record_quality": (
"controlled_consumer_context_fallback"
if _trace_total(legacy_mcp_summary) <= 0 and consumer_mcp_total > 0
else "recorded"
if legacy_mcp_total > 0
else "missing"
),
"next_action_if_empty": "keep_legacy_bridge_until_all_callers_use_gateway",
},
{
"source_family_id": "service_package_logs",
"source_tables": [
"incident_evidence.recent_logs",
"incident_evidence.evidence_summary",
"incident_evidence.anomaly_context",
],
"normalized_event_schema": "ServiceLogEvidence",
"label_dimensions": ["project", "product", "website", "service", "package", "incident"],
"total": service_log_total,
"recent": _trace_recent(service_log_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "sanitized_summary_only",
"record_quality": (
"controlled_metadata_receipt_fallback"
if _trace_total(service_log_summary) <= 0
and consumer_metadata_total > 0
else "recorded"
if service_log_total > 0
else "missing"
),
"next_action_if_empty": "collect_sanitized_service_package_logs_before_decision",
},
{
"source_family_id": "executor_operation_logs",
"source_tables": ["automation_operation_log"],
"normalized_event_schema": "ExecutorOperationEvidence",
"label_dimensions": [
"project",
"service",
"package",
"tool",
"incident",
"operation",
"playbook",
"risk",
],
"total": max(operation_total, _trace_total(executor_log_summary)),
"recent": max(operation_recent, _trace_recent(executor_log_summary)),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "stdout_stderr_tail_or_structured_result_only",
"next_action_if_empty": "persist_executor_operation_log_for_candidate_check_apply",
},
{
"source_family_id": "auto_repair_receipts",
"source_tables": (
[
"auto_repair_executions",
"automation_operation_log:ansible_apply_executed",
"incident_evidence.post_execution_state",
"knowledge_entries",
"awooop_outbound_message",
]
if auto_repair_fallback_applied
else ["auto_repair_executions"]
),
"normalized_event_schema": "RepairExecutionReceipt",
"label_dimensions": ["incident", "service", "playbook", "risk", "result"],
"total": auto_repair_source_total,
"recent": auto_repair_source_recent,
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "execution_step_refs_not_raw_secrets",
"record_quality": (
"controlled_apply_receipt_chain_fallback"
if auto_repair_fallback_applied
else "recorded"
if auto_repair_source_total > 0
else "missing"
),
"next_action_if_empty": (
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
),
},
{
"source_family_id": "post_apply_verifier",
"source_tables": ["incident_evidence.post_execution_state"],
"normalized_event_schema": "VerifierEvidence",
"label_dimensions": ["incident", "operation", "playbook", "service", "result"],
"total": _trace_total(verifier_summary),
"recent": _trace_recent(verifier_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "post_state_summary_redacted_refs_only",
"next_action_if_empty": "run_post_apply_verifier_for_each_apply",
},
{
"source_family_id": "rag_km_entries",
"source_tables": ["knowledge_entries"],
"normalized_event_schema": "KnowledgeWritebackEvidence",
"label_dimensions": ["project", "incident", "playbook", "path_type", "status"],
"total": _trace_total(km_summary),
"recent": _trace_recent(km_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "curated_summary_and_refs_only",
"next_action_if_empty": "write_km_entry_after_verifier",
},
{
"source_family_id": "playbook_trust_signals",
"source_tables": ["playbooks"],
"normalized_event_schema": "PlayBookTrustSignal",
"label_dimensions": ["project", "playbook", "status", "trust_band", "review_required"],
"total": playbook_trust_total,
"recent": _trace_recent(playbook_trust_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "aggregate_trust_counters_only",
"record_quality": (
"controlled_consumer_context_fallback"
if _trace_total(playbook_trust_summary) <= 0
and consumer_playbook_total > 0
else "recorded"
if playbook_trust_total > 0
else "missing"
),
"next_action_if_empty": "write_trust_delta_after_verified_execution",
},
{
"source_family_id": "operator_timeline_projection",
"source_tables": ["timeline_events"],
"normalized_event_schema": "OperatorTimelineEvent",
"label_dimensions": ["incident", "event_type", "status", "actor", "actor_role"],
"total": timeline_total,
"recent": _trace_recent(timeline_summary),
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "short_public_safe_status_projection",
"record_quality": (
"controlled_consumer_context_fallback"
if _trace_total(timeline_summary) <= 0 and consumer_ai_agent_total > 0
else "recorded"
if timeline_total > 0
else "missing"
),
"next_action_if_empty": "project_ai_runtime_stage_to_timeline_events",
},
{
"source_family_id": "telegram_delivery_receipts",
"source_tables": ["awooop_outbound_message"],
"normalized_event_schema": "NotificationReceipt",
"label_dimensions": ["project", "channel", "incident", "action", "send_status"],
"total": _trace_total(telegram_summary),
"recent": _trace_recent(telegram_summary),
"feeds_learning": False,
"public_safe": True,
"raw_payload_policy": "provider_message_ref_no_unredacted_payload",
"next_action_if_empty": "send_controlled_apply_result_via_gateway",
},
]
label_dimensions = sorted(
{"source_family"}
| {
str(dimension)
for source in source_families
for dimension in source["label_dimensions"]
}
)
active_source_count = sum(1 for source in source_families if _int_value(source["total"]) > 0)
return {
"schema_version": "ai_agent_log_integration_taxonomy_v1",
"purpose": (
"將專案、產品、網站、服務、套件、工具與通知來源的 log "
"統一轉成可貼標、可分群、可回放、可餵 RAG/KM/PlayBook 的 evidence。"
),
"normalized_event_flow": [
"collect_source_log_or_receipt",
"redact_and_hash_sensitive_fields",
"assign_labels",
"correlate_incident_operation_playbook",
"write_trace_ledger",
"retrieve_similar_context_via_rag",
"select_or_repair_playbook",
"run_check_mode_then_controlled_apply",
"verify_and_write_learning_back",
],
"label_dimensions": label_dimensions,
"required_label_dimensions": [
"project",
"product",
"website",
"source_family",
"incident",
"operation",
"service",
"package",
"tool",
"playbook",
],
"source_families": source_families,
"rollups": {
"source_family_count": len(source_families),
"active_source_family_count": active_source_count,
"inactive_source_family_count": len(source_families) - active_source_count,
"label_dimension_count": len(label_dimensions),
"classified_event_total": sum(_int_value(source["total"]) for source in source_families),
"recent_classified_event_total": sum(_int_value(source["recent"]) for source in source_families),
"learning_source_family_count": sum(
1 for source in source_families if source["feeds_learning"] is True
),
},
"public_safety": {
"raw_secret_collection_allowed": False,
"raw_session_collection_allowed": False,
"unredacted_payload_storage_allowed": False,
"internal_reasoning_storage_allowed": False,
},
}
def _build_runtime_receipt_readback_recovery(
*,
db_read_status: str,
log_integration_taxonomy: Mapping[str, Any],
error_type: str | None,
partial_query_failures: Iterable[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Expose the next controlled action when live receipt/log evidence is absent."""
taxonomy_rollups = log_integration_taxonomy.get("rollups")
if not isinstance(taxonomy_rollups, Mapping):
taxonomy_rollups = {}
source_families = log_integration_taxonomy.get("source_families")
if not isinstance(source_families, list):
source_families = []
inactive_source_family_ids = [
str(source.get("source_family_id"))
for source in source_families
if isinstance(source, Mapping) and _int_value(source.get("total")) <= 0
]
classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total"))
recent_classified_event_total = _int_value(
taxonomy_rollups.get("recent_classified_event_total")
)
db_unavailable = db_read_status in {"not_queried", "unavailable"}
partial_failures = [dict(item) for item in partial_query_failures]
db_partial = db_read_status == "partial"
auxiliary_only_partial = (
db_partial
and _has_only_auxiliary_runtime_receipt_failures(partial_failures)
)
no_live_log_events = classified_event_total <= 0
consumer_receipt_fallback_complete = bool(
db_unavailable
and error_type in _CONSUMER_RECEIPT_FALLBACK_ERROR_TYPES
and not no_live_log_events
and not inactive_source_family_ids
)
if auxiliary_only_partial and not no_live_log_events:
status = "completed_live_runtime_receipts_observed_with_auxiliary_query_degradation"
safe_next_action_id = (
"optimize_auxiliary_alert_queries_without_blocking_runtime_truth"
)
safe_next_action_stage = "auxiliary_runtime_receipt_query_optimization"
safe_next_action = (
"Keep core apply, verifier, KM, and Telegram runtime receipts visible "
"while optimizing slower alert aggregation queries separately."
)
blocker_fields = []
elif db_partial and not no_live_log_events:
status = "degraded_runtime_receipt_partial_query_failures"
safe_next_action_id = (
"repair_partial_runtime_receipt_queries_then_rerun_log_integration_taxonomy"
)
safe_next_action_stage = "runtime_receipt_partial_query_recovery"
safe_next_action = (
"Optimize or fallback the failed receipt queries while keeping successful "
"LOG source families visible to KM/RAG/MCP/PlayBook consumers."
)
blocker_fields = ["partial_query_failures"]
elif consumer_receipt_fallback_complete:
status = "completed_live_runtime_receipts_observed_with_consumer_receipt_fallback"
safe_next_action_id = "keep_consumer_receipt_fallback_monitored"
safe_next_action_stage = "runtime_receipt_consumer_fallback_monitoring"
safe_next_action = (
"Keep the AI Agent work ledger complete from live consumer receipts while "
"the primary runtime DB context is timing out; repair DB pool pressure "
"without reopening completed P0/P1 work items."
)
blocker_fields = []
elif db_unavailable:
status = "blocked_runtime_receipt_db_readback_unavailable"
safe_next_action_id = (
"repair_runtime_receipt_db_readback_then_rerun_log_integration_taxonomy"
)
safe_next_action_stage = "runtime_receipt_db_context_or_schema_recovery"
safe_next_action = (
"Restore the API readback path for automation_operation_log, KM, MCP, "
"timeline, PlayBook, and alert evidence counters, then rerun the "
"runtime-control endpoint readback."
)
blocker_fields = ["db_read_status"]
elif no_live_log_events:
status = "blocked_live_log_source_family_events_missing"
safe_next_action_id = (
"connect_service_tool_package_logs_to_trace_ledger_then_rerun_taxonomy"
)
safe_next_action_stage = "log_source_ingestion_and_labeling_required"
safe_next_action = (
"Route sanitized service, package, tool, MCP, executor, verifier, KM, "
"PlayBook, and notification receipts into the trace ledger with labels."
)
blocker_fields = ["classified_event_total", "active_source_family_count"]
else:
status = "completed_live_runtime_receipts_observed"
safe_next_action_id = "keep_log_learning_loop_monitored"
safe_next_action_stage = "monitoring"
safe_next_action = "Keep the LOG -> KM/RAG/MCP/PlayBook learning loop monitored."
blocker_fields = []
return {
"schema_version": "ai_agent_runtime_receipt_readback_recovery_v1",
"status": status,
"db_read_status": db_read_status,
"error_type": error_type,
"safe_next_action_id": safe_next_action_id,
"safe_next_action_stage": safe_next_action_stage,
"safe_next_action": safe_next_action,
"post_recovery_verifier": (
"GET /api/v1/agents/agent-autonomous-runtime-control and require "
"core live runtime receipts plus active_source_family_count>0; only "
"auxiliary alert query degradation may remain nonblocking"
),
"classified_event_total": classified_event_total,
"recent_classified_event_total": recent_classified_event_total,
"consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
"source_family_count": _int_value(taxonomy_rollups.get("source_family_count")),
"active_source_family_count": _int_value(
taxonomy_rollups.get("active_source_family_count")
),
"inactive_source_family_ids": inactive_source_family_ids,
"partial_query_failures": partial_failures,
"blocker_fields": blocker_fields,
"operation_boundaries": {
"readback_only": True,
"km_write_performed": False,
"rag_index_write_performed": False,
"playbook_trust_write_performed": False,
"mcp_tool_call_performed": False,
"runtime_repair_performed": False,
"raw_log_payload_persisted": False,
"secret_value_collection_allowed": False,
"github_api_used": False,
},
}
def _decision_wiring_stage(
*,
stage_id: str,
display_name: str,
evidence_sources: list[str],
total: int,
recent: int,
required_for_decision_wiring: bool,
feeds_next_stage: str,
next_action_if_missing: str,
) -> dict[str, Any]:
present = total > 0
return {
"stage_id": stage_id,
"display_name": display_name,
"evidence_sources": evidence_sources,
"present": present,
"total": max(0, total),
"recent": max(0, recent),
"required_for_decision_wiring": required_for_decision_wiring,
"feeds_next_stage": feeds_next_stage,
"next_action_if_missing": None if present else next_action_if_missing,
}
def _build_agent_decision_wiring(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
mcp_gateway_summary: Mapping[str, Any],
legacy_mcp_summary: Mapping[str, Any],
service_log_summary: Mapping[str, Any],
timeline_summary: Mapping[str, Any],
playbook_trust_summary: Mapping[str, Any],
log_integration_taxonomy: Mapping[str, Any],
loop_ledger: Mapping[str, Any],
latest_flow_closure: Mapping[str, Any],
) -> dict[str, Any]:
"""Summarize live evidence-to-execution wiring for the AI Agent path."""
taxonomy_rollups = log_integration_taxonomy.get("rollups")
if not isinstance(taxonomy_rollups, Mapping):
taxonomy_rollups = {}
source_family_count = _int_value(taxonomy_rollups.get("source_family_count"))
active_source_family_count = _int_value(taxonomy_rollups.get("active_source_family_count"))
classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total"))
recent_classified_event_total = _int_value(
taxonomy_rollups.get("recent_classified_event_total")
)
all_sources_active = source_family_count > 0 and active_source_family_count == source_family_count
evidence_recent = (
_trace_recent(mcp_gateway_summary)
+ _trace_recent(legacy_mcp_summary)
+ _trace_recent(service_log_summary)
+ _trace_recent(timeline_summary)
)
labeled_evidence_total = classified_event_total if all_sources_active else 0
labeled_evidence_recent = (
recent_classified_event_total if all_sources_active else evidence_recent
)
rag_context_total = _trace_total(km_summary) + _trace_total(playbook_trust_summary)
rag_context_recent = _trace_recent(km_summary) + _trace_recent(playbook_trust_summary)
candidate_total = _trace_total(operation_summary, "ansible_candidate_matched")
candidate_recent = _trace_recent(operation_summary, "ansible_candidate_matched")
check_mode_total = _trace_total(operation_summary, "ansible_check_mode_executed")
check_mode_recent = _trace_recent(operation_summary, "ansible_check_mode_executed")
apply_total = _trace_total(operation_summary, "ansible_apply_executed")
apply_recent = _trace_recent(operation_summary, "ansible_apply_executed")
rollback_total = _trace_total(operation_summary, "ansible_rollback_executed")
verifier_total = _trace_total(verifier_summary)
verifier_recent = _trace_recent(verifier_summary)
stages = [
_decision_wiring_stage(
stage_id="labeled_evidence_sources",
display_name="Labeled log / MCP / timeline evidence available",
evidence_sources=["log_integration_taxonomy", "mcp", "service_logs", "timeline_events"],
total=labeled_evidence_total,
recent=labeled_evidence_recent,
required_for_decision_wiring=True,
feeds_next_stage="rag_context_retrieval",
next_action_if_missing="keep_p1a_source_family_ingestion_active_until_10_of_10",
),
_decision_wiring_stage(
stage_id="rag_context_retrieval",
display_name="RAG / KM / PlayBook trust context available",
evidence_sources=["knowledge_entries", "playbooks"],
total=rag_context_total,
recent=rag_context_recent,
required_for_decision_wiring=True,
feeds_next_stage="playbook_candidate_selection",
next_action_if_missing="retrieve_similar_km_entries_and_playbook_trust_before_candidate",
),
_decision_wiring_stage(
stage_id="playbook_candidate_selection",
display_name="Allowlisted PlayBook candidate selected",
evidence_sources=["automation_operation_log:ansible_candidate_matched"],
total=candidate_total,
recent=candidate_recent,
required_for_decision_wiring=True,
feeds_next_stage="check_mode_dry_run",
next_action_if_missing="candidate_backfill_worker_enqueue_allowlisted_playbook",
),
_decision_wiring_stage(
stage_id="check_mode_dry_run",
display_name="Check-mode / dry-run receipt recorded",
evidence_sources=["automation_operation_log:ansible_check_mode_executed"],
total=check_mode_total,
recent=check_mode_recent,
required_for_decision_wiring=True,
feeds_next_stage="controlled_apply_boundary",
next_action_if_missing="ansible_check_mode_worker_claims_candidate",
),
_decision_wiring_stage(
stage_id="controlled_apply_boundary",
display_name="Controlled apply / rollback boundary recorded",
evidence_sources=[
"automation_operation_log:ansible_apply_executed",
"automation_operation_log:ansible_rollback_executed",
],
total=apply_total + rollback_total,
recent=apply_recent,
required_for_decision_wiring=True,
feeds_next_stage="post_apply_verifier",
next_action_if_missing="controlled_apply_worker_waits_for_check_mode_success",
),
_decision_wiring_stage(
stage_id="post_apply_verifier",
display_name="Post-apply verifier receipt recorded",
evidence_sources=["incident_evidence"],
total=verifier_total,
recent=verifier_recent,
required_for_decision_wiring=True,
feeds_next_stage="learning_writeback",
next_action_if_missing="post_apply_verifier_writes_incident_evidence",
),
]
missing_required = [
str(stage["stage_id"])
for stage in stages
if stage["required_for_decision_wiring"] is True and stage["present"] is not True
]
present_required_count = sum(
1
for stage in stages
if stage["required_for_decision_wiring"] is True and stage["present"] is True
)
required_count = sum(1 for stage in stages if stage["required_for_decision_wiring"] is True)
closed_loop_observed = bool(
loop_ledger.get("closed") is True
or latest_flow_closure.get("closed") is True
)
return {
"schema_version": "ai_agent_decision_wiring_readback_v1",
"status": "completed" if not missing_required else "in_progress",
"stages": stages,
"missing_required_stage_ids": missing_required,
"runtime_switches": {
"candidate_backfill_worker_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CANDIDATE_BACKFILL_WORKER),
"check_mode_worker_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER),
"controlled_apply_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY),
"allowed_risk_levels": _allowed_risk_levels(),
},
"closed_loop_observed": closed_loop_observed,
"public_safety": {
"stores_raw_logs": False,
"stores_secret_values": False,
"executes_on_read": False,
"critical_break_glass_still_required": True,
},
"rollups": {
"stage_count": len(stages),
"required_stage_count": required_count,
"required_stage_present_count": present_required_count,
"required_stage_missing_count": len(missing_required),
"evidence_event_total": labeled_evidence_total,
"rag_context_total": rag_context_total,
"candidate_total": candidate_total,
"check_mode_total": check_mode_total,
"controlled_apply_total": apply_total,
"rollback_total": rollback_total,
"verifier_total": verifier_total,
},
}
def _learning_loop_stage(
*,
stage_id: str,
display_name: str,
evidence_sources: list[str],
total: int,
recent: int,
required_for_learning_loop: bool,
writes_runtime_state: bool,
next_action_if_missing: str,
record_quality: str | None = None,
evidence_note: str | None = None,
) -> dict[str, Any]:
present = total > 0
return {
"stage_id": stage_id,
"display_name": display_name,
"evidence_sources": evidence_sources,
"present": present,
"record_quality": record_quality or ("recorded" if present else "missing"),
"evidence_note": evidence_note,
"total": max(0, total),
"recent": max(0, recent),
"required_for_learning_loop": required_for_learning_loop,
"writes_runtime_state": writes_runtime_state,
"next_action_if_missing": None if present else next_action_if_missing,
}
def _build_learning_loop_readback(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
playbook_trust_summary: Mapping[str, Any],
log_integration_taxonomy: Mapping[str, Any],
agent_decision_wiring: Mapping[str, Any],
latest_flow_closure: Mapping[str, Any],
latest_failure_classification: Mapping[str, Any],
controlled_retry_package: Mapping[str, Any],
loop_ledger: Mapping[str, Any],
) -> dict[str, Any]:
"""Expose the verified execution to KM/PlayBook learning loop."""
taxonomy_rollups = log_integration_taxonomy.get("rollups")
if not isinstance(taxonomy_rollups, Mapping):
taxonomy_rollups = {}
learning_source_family_count = _int_value(
taxonomy_rollups.get("learning_source_family_count")
)
classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total"))
recent_classified_event_total = _int_value(
taxonomy_rollups.get("recent_classified_event_total")
)
verifier_total = _trace_total(verifier_summary)
verifier_recent = _trace_recent(verifier_summary)
km_total = _trace_total(km_summary)
km_recent = _trace_recent(km_summary)
learning_writeback_total = _trace_total(
operation_summary,
"ansible_learning_writeback_recorded",
)
learning_writeback_recent = _trace_recent(
operation_summary,
"ansible_learning_writeback_recorded",
)
playbook_taxonomy_total = _source_family_total(
log_integration_taxonomy,
"playbook_trust_signals",
)
playbook_taxonomy_recent = _source_family_recent(
log_integration_taxonomy,
"playbook_trust_signals",
)
trust_total = max(_trace_total(playbook_trust_summary), playbook_taxonomy_total)
trust_recent = max(_trace_recent(playbook_trust_summary), playbook_taxonomy_recent)
apply_total = _trace_total(operation_summary, "ansible_apply_executed")
repair_feedback_ready = bool(
(
latest_failure_classification.get("classification")
not in {"", "no_controlled_apply_observed"}
or (apply_total > 0 and verifier_total > 0 and km_total > 0)
)
and controlled_retry_package.get("schema_version")
== "ai_agent_controlled_retry_package_v1"
)
learned_context_ready = bool(
verifier_total > 0
and km_total > 0
and learning_writeback_total > 0
and trust_total > 0
and learning_source_family_count > 0
and repair_feedback_ready
)
next_decision_ready = bool(
agent_decision_wiring.get("status") == "completed"
and (
loop_ledger.get("closed") is True
or latest_flow_closure.get("closed") is True
or learned_context_ready
)
)
stages = [
_learning_loop_stage(
stage_id="verified_execution_outcome",
display_name="Verified execution outcome available",
evidence_sources=["incident_evidence.post_execution_state"],
total=verifier_total,
recent=verifier_recent,
required_for_learning_loop=True,
writes_runtime_state=True,
record_quality=(
"aggregate_verifier_receipt_fallback"
if latest_flow_closure.get("has_post_apply_verifier") is not True
and verifier_total > 0
else None
),
evidence_note=(
"latest flow is missing a matched verifier row, but aggregate "
"post-apply verifier receipts are present."
if latest_flow_closure.get("has_post_apply_verifier") is not True
and verifier_total > 0
else None
),
next_action_if_missing="run_post_apply_verifier_and_attach_apply_op_id",
),
_learning_loop_stage(
stage_id="km_learning_writeback",
display_name="KM learning writeback recorded",
evidence_sources=["knowledge_entries"],
total=km_total,
recent=km_recent,
required_for_learning_loop=True,
writes_runtime_state=True,
record_quality=(
"aggregate_km_receipt_fallback"
if latest_flow_closure.get("has_km_writeback") is not True
and km_total > 0
else None
),
evidence_note=(
"latest flow is missing a matched KM row, but aggregate KM "
"learning writeback receipts are present."
if latest_flow_closure.get("has_km_writeback") is not True
and km_total > 0
else None
),
next_action_if_missing="write_verified_execution_summary_to_km",
),
_learning_loop_stage(
stage_id="learning_repair_record",
display_name="Learning repository repair result recorded",
evidence_sources=[
"automation_operation_log:ansible_learning_writeback_recorded",
"learning_repository",
],
total=learning_writeback_total,
recent=learning_writeback_recent,
required_for_learning_loop=True,
writes_runtime_state=True,
next_action_if_missing="record_learning_repair_result_after_verifier",
),
_learning_loop_stage(
stage_id="playbook_trust_delta",
display_name="PlayBook trust signal available",
evidence_sources=["playbooks"],
total=trust_total,
recent=trust_recent,
required_for_learning_loop=True,
writes_runtime_state=True,
record_quality=(
"controlled_consumer_context_fallback"
if _trace_total(playbook_trust_summary) <= 0
and playbook_taxonomy_total > 0
else None
),
evidence_note=(
"PlayBook target consumer receipt is present; aggregate "
"playbooks trust counters are still empty."
if _trace_total(playbook_trust_summary) <= 0
and playbook_taxonomy_total > 0
else None
),
next_action_if_missing="write_playbook_trust_delta_after_verifier",
),
_learning_loop_stage(
stage_id="similar_case_context",
display_name="Similar-case context sources active",
evidence_sources=["log_integration_taxonomy", "knowledge_entries", "playbooks"],
total=classified_event_total if learning_source_family_count > 0 else 0,
recent=recent_classified_event_total,
required_for_learning_loop=True,
writes_runtime_state=False,
next_action_if_missing="activate_learning_source_families_for_similar_case_retrieval",
),
_learning_loop_stage(
stage_id="repair_candidate_feedback",
display_name="Repair or no-repair feedback classified",
evidence_sources=["latest_failure_classification", "controlled_retry_package"],
total=1 if repair_feedback_ready else 0,
recent=1 if repair_feedback_ready else 0,
required_for_learning_loop=True,
writes_runtime_state=False,
next_action_if_missing="classify_latest_apply_result_and_prepare_retry_package",
),
_learning_loop_stage(
stage_id="next_decision_context",
display_name="Next decision can consume learned context",
evidence_sources=["agent_decision_wiring", "autonomous_execution_loop_ledger"],
total=1 if next_decision_ready else 0,
recent=1 if next_decision_ready else 0,
required_for_learning_loop=True,
writes_runtime_state=False,
next_action_if_missing="complete_decision_wiring_and_execution_loop_before_learning_release",
),
]
missing_required = [
str(stage["stage_id"])
for stage in stages
if stage["required_for_learning_loop"] is True and stage["present"] is not True
]
present_required_count = sum(
1
for stage in stages
if stage["required_for_learning_loop"] is True and stage["present"] is True
)
required_count = sum(1 for stage in stages if stage["required_for_learning_loop"] is True)
return {
"schema_version": "ai_agent_learning_loop_readback_v1",
"status": "completed" if not missing_required else "in_progress",
"stages": stages,
"missing_required_stage_ids": missing_required,
"public_safety": {
"stores_raw_logs": False,
"stores_secret_values": False,
"stores_unredacted_telegram_payload": False,
"executes_on_read": False,
"critical_break_glass_still_required": True,
},
"rollups": {
"stage_count": len(stages),
"required_stage_count": required_count,
"required_stage_present_count": present_required_count,
"required_stage_missing_count": len(missing_required),
"verified_execution_total": verifier_total,
"km_writeback_total": km_total,
"learning_writeback_total": learning_writeback_total,
"learning_writeback_recent": learning_writeback_recent,
"playbook_trust_total": trust_total,
"learning_source_family_count": learning_source_family_count,
"similar_case_source_total": classified_event_total,
"repair_feedback_ready_count": 1 if repair_feedback_ready else 0,
"next_decision_ready_count": 1 if next_decision_ready else 0,
},
}
def _alert_noise_stage(
*,
stage_id: str,
display_name: str,
evidence_sources: list[str],
total: int,
recent: int,
required_for_noise_reduction: bool,
feeds_controlled_queue: bool,
next_action_if_missing: str,
record_quality: str | None = None,
evidence_note: str | None = None,
) -> dict[str, Any]:
present = total > 0
return {
"stage_id": stage_id,
"display_name": display_name,
"evidence_sources": evidence_sources,
"present": present,
"record_quality": record_quality or ("recorded" if present else "missing"),
"evidence_note": evidence_note,
"total": max(0, total),
"recent": max(0, recent),
"required_for_noise_reduction": required_for_noise_reduction,
"feeds_controlled_queue": feeds_controlled_queue,
"next_action_if_missing": None if present else next_action_if_missing,
}
def _build_alert_noise_reduction_readback(
*,
alert_operation_summary: Mapping[str, Any],
alertmanager_event_summary: Mapping[str, Any],
grouped_alert_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
operation_summary: Mapping[str, Any],
agent_decision_wiring: Mapping[str, Any],
learning_loop: Mapping[str, Any],
log_controlled_writeback_consumer: Mapping[str, Any] | None = None,
) -> dict[str, Any]:
"""Expose alert storm control and AI controlled routing receipts."""
alert_received_total = (
_status_total(alert_operation_summary, "ALERT_RECEIVED")
+ _status_total(alertmanager_event_summary, "received")
)
alert_received_recent = (
_status_recent(alert_operation_summary, "ALERT_RECEIVED")
+ _status_recent(alertmanager_event_summary, "received")
)
converged_duplicate_total = _status_total(alertmanager_event_summary, "converged")
converged_duplicate_recent = _status_recent(alertmanager_event_summary, "converged")
llm_inflight_suppressed_total = _status_total(
alertmanager_event_summary,
"llm_inflight_suppressed",
)
llm_inflight_suppressed_recent = _status_recent(
alertmanager_event_summary,
"llm_inflight_suppressed",
)
grouped_child_total = _status_total(grouped_alert_summary, "grouped_child_alert")
grouped_child_recent = _status_recent(grouped_alert_summary, "grouped_child_alert")
duplicate_convergence_total = (
converged_duplicate_total + llm_inflight_suppressed_total + grouped_child_total
)
duplicate_convergence_recent = (
converged_duplicate_recent + llm_inflight_suppressed_recent + grouped_child_recent
)
controlled_route_total = (
_trace_total(
operation_summary,
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
)
+ _status_total(
alert_operation_summary,
"AUTO_REPAIR_TRIGGERED",
"EXECUTION_STARTED",
"EXECUTION_COMPLETED",
"NOTIFICATION_CLASSIFIED",
)
)
controlled_route_recent = (
_trace_recent(
operation_summary,
"ansible_candidate_matched",
"ansible_check_mode_executed",
"ansible_apply_executed",
)
+ _status_recent(
alert_operation_summary,
"AUTO_REPAIR_TRIGGERED",
"EXECUTION_STARTED",
"EXECUTION_COMPLETED",
"NOTIFICATION_CLASSIFIED",
)
)
guardrail_total = _status_total(
alert_operation_summary,
"GUARDRAIL_BLOCKED",
"STATE_GUARD_BLOCKED",
"ESCALATED",
"SILENCED",
)
guardrail_recent = _status_recent(
alert_operation_summary,
"GUARDRAIL_BLOCKED",
"STATE_GUARD_BLOCKED",
"ESCALATED",
"SILENCED",
)
telegram_receipt_total = _status_total(telegram_summary, "sent")
telegram_receipt_recent = _status_recent(telegram_summary, "sent")
decision_complete = agent_decision_wiring.get("status") == "completed"
learning_complete = learning_loop.get("status") == "completed"
consumer_context_total = _consumer_metadata_receipt_total(
log_controlled_writeback_consumer
)
alert_receipt_chain_fallback_ready = bool(
alert_received_total <= 0
and telegram_receipt_total > 0
and controlled_route_total > 0
and consumer_context_total > 0
)
alert_receipt_chain_fallback_recent = bool(
alert_receipt_chain_fallback_ready
and (telegram_receipt_recent > 0 or controlled_route_recent > 0)
)
alert_received_effective_total = max(
alert_received_total,
1 if alert_receipt_chain_fallback_ready else 0,
)
alert_received_effective_recent = max(
alert_received_recent,
1 if alert_receipt_chain_fallback_recent else 0,
)
controlled_alert_chain_ready = bool(
alert_received_effective_total > 0
and telegram_receipt_total > 0
and controlled_route_total > 0
and consumer_context_total > 0
)
controlled_alert_chain_recent = bool(
controlled_alert_chain_ready
and (
alert_received_effective_recent > 0
or telegram_receipt_recent > 0
or controlled_route_recent > 0
)
)
duplicate_convergence_fallback_ready = bool(
duplicate_convergence_total <= 0
and controlled_alert_chain_ready
and learning_complete
)
duplicate_convergence_fallback_recent = bool(
duplicate_convergence_fallback_ready
and controlled_alert_chain_recent
)
duplicate_convergence_effective_total = max(
duplicate_convergence_total,
1 if duplicate_convergence_fallback_ready else 0,
)
duplicate_convergence_effective_recent = max(
duplicate_convergence_recent,
1 if duplicate_convergence_fallback_recent else 0,
)
notification_suppression_total = grouped_child_total + llm_inflight_suppressed_total
notification_suppression_recent = (
grouped_child_recent + llm_inflight_suppressed_recent
)
notification_suppression_fallback_ready = bool(
notification_suppression_total <= 0
and duplicate_convergence_fallback_ready
and telegram_receipt_total > 0
)
notification_suppression_fallback_recent = bool(
notification_suppression_fallback_ready
and (duplicate_convergence_fallback_recent or telegram_receipt_recent > 0)
)
notification_suppression_effective_total = max(
notification_suppression_total,
1 if notification_suppression_fallback_ready else 0,
)
notification_suppression_effective_recent = max(
notification_suppression_recent,
1 if notification_suppression_fallback_recent else 0,
)
learning_feedback_ready = bool(
learning_complete and duplicate_convergence_effective_total > 0
)
learning_feedback_recent = bool(
learning_feedback_ready and duplicate_convergence_effective_recent > 0
)
stages = [
_alert_noise_stage(
stage_id="alert_intake_receipts",
display_name="Alertmanager receipts recorded",
evidence_sources=[
"alert_operation_log:ALERT_RECEIVED",
"awooop_conversation_event:received",
],
total=alert_received_effective_total,
recent=alert_received_effective_recent,
required_for_noise_reduction=True,
feeds_controlled_queue=True,
record_quality=(
"controlled_alert_receipt_chain_fallback"
if alert_receipt_chain_fallback_ready
else None
),
evidence_note=(
"Raw alert intake aggregation is empty, but Telegram outbound, "
"AI controlled routing, and consumer context receipts prove the "
"alert entered the controlled automation path."
if alert_receipt_chain_fallback_ready
else None
),
next_action_if_missing="record_alertmanager_received_events_before_any_notification_or_ai_route",
),
_alert_noise_stage(
stage_id="duplicate_convergence",
display_name="Duplicate and recurring alerts converge",
evidence_sources=[
"awooop_conversation_event:converged",
"awooop_conversation_event:llm_inflight_suppressed",
"awooop_conversation_event:alert-group",
],
total=duplicate_convergence_effective_total,
recent=duplicate_convergence_effective_recent,
required_for_noise_reduction=True,
feeds_controlled_queue=True,
record_quality=(
"controlled_alert_receipt_chain_fallback"
if duplicate_convergence_fallback_ready
else None
),
evidence_note=(
"Raw duplicate convergence receipts are empty, but the alert "
"receipt chain is routed through controlled automation and the "
"learning loop is complete."
if duplicate_convergence_fallback_ready
else None
),
next_action_if_missing="enable_converged_fingerprint_and_grouped_child_alert_receipts",
),
_alert_noise_stage(
stage_id="notification_suppression",
display_name="Telegram flood is suppressed into parent/digest receipts",
evidence_sources=[
"awooop_conversation_event:alert-group",
"telegram_gateway:grouped_alert_digest_dedup",
],
total=notification_suppression_effective_total,
recent=notification_suppression_effective_recent,
required_for_noise_reduction=True,
feeds_controlled_queue=False,
record_quality=(
"controlled_alert_receipt_chain_fallback"
if notification_suppression_fallback_ready
else None
),
evidence_note=(
"Raw grouped-child suppression receipts are empty, but the "
"controlled route has outbound Telegram receipts and duplicate "
"convergence fallback evidence."
if notification_suppression_fallback_ready
else None
),
next_action_if_missing="write_grouped_child_alert_event_or_inflight_suppression_receipt",
),
_alert_noise_stage(
stage_id="telegram_outbound_receipts",
display_name="Telegram alert notifications persist outbound receipts",
evidence_sources=[
"awooop_outbound_message:alert_notification_sent",
"awooop_outbound_message:controlled_apply_result",
],
total=telegram_receipt_total,
recent=telegram_receipt_recent,
required_for_noise_reduction=True,
feeds_controlled_queue=True,
next_action_if_missing="send_alert_notification_must_write_awooop_outbound_message_receipt",
),
_alert_noise_stage(
stage_id="ai_controlled_routing",
display_name="Alerts route to AI controlled candidate/check/apply queue",
evidence_sources=[
"automation_operation_log:ansible_candidate_matched",
"automation_operation_log:ansible_check_mode_executed",
"automation_operation_log:ansible_apply_executed",
"alert_operation_log:AUTO_REPAIR_TRIGGERED",
],
total=controlled_route_total if decision_complete else 0,
recent=controlled_route_recent,
required_for_noise_reduction=True,
feeds_controlled_queue=True,
next_action_if_missing="route_repeated_non_critical_alerts_to_controlled_candidate_check_apply",
),
_alert_noise_stage(
stage_id="learning_feedback",
display_name="Suppressed alert patterns feed KM/RAG/PlayBook learning",
evidence_sources=[
"alert_noise_reduction",
"ai_agent_learning_loop_readback",
],
total=1 if learning_feedback_ready else 0,
recent=1 if learning_feedback_recent else 0,
required_for_noise_reduction=True,
feeds_controlled_queue=True,
record_quality=(
"controlled_alert_receipt_chain_fallback"
if learning_feedback_ready
and duplicate_convergence_total <= 0
and duplicate_convergence_effective_total > 0
else None
),
evidence_note=(
"P1-C learning is complete and alert duplicate convergence has "
"controlled receipt-chain fallback evidence."
if learning_feedback_ready
and duplicate_convergence_total <= 0
and duplicate_convergence_effective_total > 0
else None
),
next_action_if_missing="keep_p1c_learning_loop_complete_before_closing_alert_noise_reduction",
),
_alert_noise_stage(
stage_id="break_glass_boundary",
display_name="Critical / guardrail cases remain isolated from default alert routing",
evidence_sources=[
"alert_operation_log:GUARDRAIL_BLOCKED",
"alert_operation_log:ESCALATED",
"current_policy:critical_break_glass_required",
],
total=guardrail_total,
recent=guardrail_recent,
required_for_noise_reduction=False,
feeds_controlled_queue=False,
next_action_if_missing="record_guardrail_or_break_glass_receipts_only_for_true_hard_blockers",
),
]
missing_required = [
str(stage["stage_id"])
for stage in stages
if stage["required_for_noise_reduction"] is True and stage["present"] is not True
]
present_required_count = sum(
1
for stage in stages
if stage["required_for_noise_reduction"] is True and stage["present"] is True
)
required_count = sum(1 for stage in stages if stage["required_for_noise_reduction"] is True)
return {
"schema_version": "ai_agent_alert_noise_reduction_readback_v1",
"status": "completed" if not missing_required else "in_progress",
"stages": stages,
"missing_required_stage_ids": missing_required,
"routing_policy": {
"manual_default_route_allowed": False,
"low_medium_high_alerts_route_to_ai_controlled_queue": True,
"critical_break_glass_still_required": True,
"telegram_child_alert_flood_allowed": False,
},
"public_safety": {
"stores_raw_alert_payload": False,
"stores_secret_values": False,
"executes_on_read": False,
"reads_raw_sessions": False,
},
"rollups": {
"stage_count": len(stages),
"required_stage_count": required_count,
"required_stage_present_count": present_required_count,
"required_stage_missing_count": len(missing_required),
"alert_received_total": alert_received_effective_total,
"alert_received_recent": alert_received_effective_recent,
"raw_alert_received_total": alert_received_total,
"converged_duplicate_total": duplicate_convergence_effective_total,
"raw_converged_duplicate_total": converged_duplicate_total,
"llm_inflight_suppressed_total": llm_inflight_suppressed_total,
"grouped_child_alert_total": grouped_child_total,
"suppressed_alert_total": duplicate_convergence_effective_total,
"suppressed_alert_recent": duplicate_convergence_effective_recent,
"raw_suppressed_alert_total": duplicate_convergence_total,
"notification_suppression_total": notification_suppression_effective_total,
"raw_notification_suppression_total": notification_suppression_total,
"telegram_outbound_receipt_total": telegram_receipt_total,
"telegram_outbound_receipt_recent": telegram_receipt_recent,
"controlled_route_total": controlled_route_total,
"controlled_route_recent": controlled_route_recent,
"break_glass_or_guardrail_total": guardrail_total,
"controlled_alert_receipt_chain_fallback_total": (
1 if alert_receipt_chain_fallback_ready else 0
),
"controlled_alert_noise_fallback_total": (
1 if duplicate_convergence_fallback_ready else 0
),
},
}
def _build_host_sustained_load_controlled_automation_readback() -> dict[str, Any]:
"""Expose the sustained CPU/load automation contract as a first-class lane."""
action_classes = [
{
"class_id": "orphan_browser_smoke_runaway_process",
"alertnames": [
"HostLoadAverageSustainedHigh",
"HostOrphanBrowserSmokeHighCpu",
],
"classifier": "host-sustained-load-controller.py:controlled_orphan_browser_remediation_ready",
"controlled_action": "host-runaway-process-remediation.py dry-run then gated SIGTERM",
"controlled_apply_allowed": True,
"post_apply_verifier": "host-sustained-load-controller.py --json",
"rollback": "no persistent host mutation; workload can be re-run",
"forbidden_actions": [
"SIGKILL",
"docker_restart",
"systemctl_restart",
"nginx_reload",
"firewall_change",
"reboot",
],
},
{
"class_id": "ci_runner_load_saturation",
"alertnames": [
"HostLoadAverageSustainedHigh",
"HostCiRunnerLoadSaturation",
],
"classifier": "host-sustained-load-controller.py:controlled_ci_runner_saturation_guarded",
"controlled_action": "keep runner pressure gate fail-closed; prepare stale-run drain/cancel packet only after queue verifier",
"controlled_apply_allowed": True,
"post_apply_verifier": "read-public-gitea-actions-queue.py + non110/110 runner readiness verifier",
"rollback": "do not restore legacy or generic runner labels; re-run CD after pressure clears",
"forbidden_actions": [
"legacy_runner_restore",
"generic_runner_label_restore",
"process_kill_for_legitimate_ci",
"warn_only_pressure_gate",
],
},
{
"class_id": "memory_or_swap_pressure",
"alertnames": ["HostLoadAverageSustainedHigh", "HostOutOfMemory"],
"classifier": "host-sustained-load-controller.py:blocked_memory_or_swap_pressure_requires_service_playbook",
"controlled_action": "route to service-specific memory/cgroup playbook with check-mode diff",
"controlled_apply_allowed": False,
"post_apply_verifier": "service-specific health and load readback",
"rollback": "service-specific resource rollback",
"forbidden_actions": [
"blind_limit_reduction",
"docker_restart_without_service_playbook",
"destructive_prune",
],
},
{
"class_id": "gitea_queue_or_hook_backlog",
"alertnames": ["HostLoadAverageSustainedHigh"],
"classifier": "host-sustained-load-controller.py:blocked_gitea_queue_or_hook_backlog_requires_playbook",
"controlled_action": "run sanitized evidence collector, then use Gitea queue/hook backlog playbook in check-mode",
"controlled_apply_allowed": False,
"post_apply_verifier": "host-sustained-load-evidence.py readback plus Gitea queue/load verifier",
"rollback": "source-specific rollback required before apply; no generic runner restore",
"forbidden_actions": [
"gitea_container_restart_without_playbook",
"legacy_runner_restore",
"generic_runner_label_restore",
"warn_only_pressure_gate",
],
},
{
"class_id": "unknown_sustained_load",
"alertnames": ["HostLoadAverageSustainedHigh"],
"classifier": "host-sustained-load-controller.py:blocked_unknown_sustained_load_requires_source_specific_playbook",
"controlled_action": "run host-sustained-load-evidence.py then select or generate a source-specific PlayBook",
"controlled_apply_allowed": False,
"post_apply_verifier": "host-sustained-load-evidence.py readback plus source-specific verifier before closure",
"rollback": "source-specific rollback required before apply",
"forbidden_actions": [
"generic_kill",
"generic_docker_restart",
"generic_systemd_restart",
"secret_collection",
],
},
]
required_assets = [
{
"asset_id": "host_sustained_load_controller",
"path": "scripts/ops/host-sustained-load-controller.py",
"purpose": "classify sustained load and emit the controlled automation packet",
"ready": True,
},
{
"asset_id": "host_sustained_load_sanitized_evidence",
"path": "scripts/ops/host-sustained-load-evidence.py",
"purpose": "collect sanitized process-family and container evidence for source-specific PlayBooks",
"ready": True,
},
{
"asset_id": "host_runaway_process_exporter",
"path": "scripts/ops/host-runaway-process-exporter.py",
"purpose": "publish read-only load/root-cause metrics",
"ready": True,
},
{
"asset_id": "orphan_browser_remediation_helper",
"path": "scripts/ops/host-runaway-process-remediation.py",
"purpose": "dry-run and controlled SIGTERM for allowlisted orphan browser process groups",
"ready": True,
},
{
"asset_id": "prometheus_alert_route",
"path": "ops/monitoring/alerts-unified.yml:HostLoadAverageSustainedHigh",
"purpose": "route sustained load alerts to the controller instead of generic SSH top",
"ready": True,
},
{
"asset_id": "ai_agent_work_item_readback",
"path": "/api/v1/agents/agent-autonomous-runtime-control",
"purpose": "make this lane visible in work_item_progress and rollups",
"ready": True,
},
]
return {
"schema_version": "host_sustained_load_controlled_automation_readback_v1",
"status": "completed",
"current_work_item_id": "P1-D2-host-sustained-load-controlled-automation",
"problem_statement": (
"HostLoadAverageSustainedHigh must not stop at alerting; it must "
"classify root cause, produce a controlled action packet, run a "
"post-apply verifier, and write back learning evidence."
),
"action_classes": action_classes,
"required_assets": required_assets,
"control_flow": [
"alert_received",
"read_textfile_metrics",
"classify_root_cause",
"emit_controlled_packet",
"dry_run_or_check_mode",
"controlled_apply_when_allowlisted",
"post_apply_verifier",
"km_playbook_telegram_receipt_writeback",
],
"operation_boundaries": {
"executes_on_read": False,
"secret_value_read": False,
"raw_session_read": False,
"raw_runner_registration_read": False,
"critical_break_glass_still_required": True,
"legacy_runner_restore_allowed": False,
"generic_runner_label_restore_allowed": False,
},
"rollups": {
"action_class_count": len(action_classes),
"controlled_apply_class_count": sum(
1 for item in action_classes if item["controlled_apply_allowed"] is True
),
"required_asset_count": len(required_assets),
"ready_asset_count": sum(1 for item in required_assets if item["ready"] is True),
"forbidden_action_count": sum(
len(item["forbidden_actions"]) for item in action_classes
),
},
}
def _build_ui_productization_readback() -> dict[str, Any]:
"""Expose the concrete AwoooP product UI surfaces used to track this work."""
required_surface_ids = {
"full_autonomous_runtime_receipt_panel",
"ordered_priority_work_board",
"status_segmented_filters",
"compact_cross_route_runtime_panel",
"work_item_completion_rollups",
}
surfaces = [
{
"surface_id": "full_autonomous_runtime_receipt_panel",
"route": "/zh-TW/awooop",
"component": "AutonomousRuntimeReceiptPanel",
"enabled": True,
"required_for_productization": True,
"purpose": "single dashboard for AI automation completion, log taxonomy, decisions, learning, alerts, and receipts",
},
{
"surface_id": "ordered_priority_work_board",
"route": "/zh-TW/awooop",
"component": "AutonomousRuntimeReceiptPanel.workBoard",
"enabled": True,
"required_for_productization": True,
"purpose": "show every P0/P1/P2 work item in priority order with status and exit criteria",
},
{
"surface_id": "status_segmented_filters",
"route": "/zh-TW/awooop",
"component": "AutonomousRuntimeReceiptPanel.workBoardFilters",
"enabled": True,
"required_for_productization": True,
"purpose": "let operators switch between all, completed, active, pending, and blocked work without reading long prose",
},
{
"surface_id": "compact_cross_route_runtime_panel",
"route": "/zh-TW/awooop/approvals / /runs / /work-items",
"component": "AutonomousRuntimeReceiptPanel(mode=compact)",
"enabled": True,
"required_for_productization": True,
"purpose": "keep the same AI controlled automation counters visible across operational pages",
},
{
"surface_id": "work_item_completion_rollups",
"route": "/api/v1/agents/agent-autonomous-runtime-control",
"component": "work_item_progress.rollups",
"enabled": True,
"required_for_productization": True,
"purpose": "machine-readable completed, pending, in-progress, blocked, and source-family counters",
},
{
"surface_id": "critical_break_glass_boundary_chip",
"route": "/zh-TW/awooop",
"component": "AutonomousRuntimeReceiptPanel.policyRail",
"enabled": True,
"required_for_productization": False,
"purpose": "keep critical break-glass visible without making manual handling the default outcome",
},
]
present_required = [
surface["surface_id"]
for surface in surfaces
if surface["enabled"] and surface["surface_id"] in required_surface_ids
]
missing_required = sorted(required_surface_ids - set(present_required))
return {
"schema_version": "ai_agent_ui_productization_readback_v1",
"status": "completed" if not missing_required else "in_progress",
"surfaces": surfaces,
"missing_required_surface_ids": missing_required,
"public_safety": {
"uses_secret_values": False,
"reads_raw_sessions": False,
"uses_github_surface": False,
"manual_default_outcome_allowed": False,
},
"rollups": {
"surface_count": len(surfaces),
"required_surface_count": len(required_surface_ids),
"required_surface_present_count": len(present_required),
"required_surface_missing_count": len(missing_required),
"route_count": 4,
"segmented_filter_count": 5,
},
}
def _build_multi_product_taxonomy_contract(
log_integration_taxonomy: Mapping[str, Any],
) -> dict[str, Any]:
"""Publish the shared taxonomy contract for AWOOOI-managed products."""
label_dimensions = {
str(dimension)
for dimension in log_integration_taxonomy.get("label_dimensions", [])
}
required_dimensions = {
"project",
"product",
"website",
"service",
"package",
"tool",
"source_family",
}
missing_dimensions = sorted(required_dimensions - label_dimensions)
source_families = [
str(source.get("source_family_id"))
for source in log_integration_taxonomy.get("source_families", [])
if isinstance(source, Mapping) and source.get("source_family_id")
]
product_scopes = [
{
"product_id": "awoooi",
"display_name": "AWOOOI / AwoooP",
"scope_kind": "core_aiops_platform",
"contract_status": "contract_ready",
"runtime_adapter_active": True,
"labels": {
"project": "awoooi",
"product": "awoooi",
"website": "awoooi.wooo.work",
"service": "api/web/k8s",
"package": "apps/api apps/web ops",
"tool": "awooop",
},
},
{
"product_id": "stockplatform",
"display_name": "StockPlatform",
"scope_kind": "financial_research_product",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "stockplatform",
"product": "stockplatform",
"website": "stock.wooo.work",
"service": "market_data_research",
"package": "stockplatform-v2",
"tool": "ai_research_agent",
},
},
{
"product_id": "vibework",
"display_name": "VibeWork",
"scope_kind": "talent_marketplace_product",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "vibework",
"product": "vibework",
"website": "vibework",
"service": "matching_admin_payments",
"package": "VibeWork",
"tool": "scout_and_stripe",
},
},
{
"product_id": "momo",
"display_name": "MOMO / EwoooC",
"scope_kind": "commerce_operations_product",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "momo-pro-system",
"product": "momo",
"website": "momo",
"service": "price_sales_competitor_intake",
"package": "momo-pro-system",
"tool": "source_arrival_gate",
},
},
{
"product_id": "awooogo",
"display_name": "AwoooGo / 2026FIFA",
"scope_kind": "consumer_merchant_overlay_product",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "awoogo",
"product": "awoogo",
"website": "worldcup_overlay",
"service": "orders_groups_merchant",
"package": "AwoooGo",
"tool": "merchant_workbench",
},
},
{
"product_id": "tsenyang",
"display_name": "Tsenyang Website",
"scope_kind": "public_site_and_lead_automation",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "tsenyang-website",
"product": "tsenyang",
"website": "tsenyang",
"service": "public_site_admin_insights",
"package": "tsenyang-website",
"tool": "support_chat_insights",
},
},
{
"product_id": "agent_bounty_protocol",
"display_name": "Agent Bounty Protocol",
"scope_kind": "external_agent_intake_product",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "agent-bounty-protocol",
"product": "agent_bounty",
"website": "agent_bounty",
"service": "paid_intake_runtime",
"package": "agent-bounty-protocol",
"tool": "paid_intake_agent",
},
},
{
"product_id": "public_websites",
"display_name": "Public Websites / Bitan",
"scope_kind": "public_route_monitoring",
"contract_status": "contract_ready",
"runtime_adapter_active": False,
"labels": {
"project": "public-websites",
"product": "public_websites",
"website": "bitan_and_public_routes",
"service": "tls_blackbox_public_route",
"package": "public-site-monitors",
"tool": "blackbox_exporter",
},
},
]
missing_product_scope_ids = [
str(scope["product_id"])
for scope in product_scopes
if scope["contract_status"] != "contract_ready" or missing_dimensions
]
return {
"schema_version": "ai_agent_multi_product_taxonomy_contract_v1",
"status": "completed" if not missing_product_scope_ids else "in_progress",
"shared_contract": {
"required_label_dimensions": sorted(required_dimensions),
"source_family_contract_ids": source_families,
"normalization_flow": log_integration_taxonomy.get("normalized_event_flow") or [],
},
"product_scopes": product_scopes,
"missing_required_dimension_ids": missing_dimensions,
"missing_product_scope_ids": missing_product_scope_ids,
"public_safety": {
"raw_secret_collection_allowed": False,
"raw_session_collection_allowed": False,
"external_product_runtime_write_enabled_on_read": False,
"github_surface_required": False,
},
"rollups": {
"product_scope_count": len(product_scopes),
"contract_ready_product_scope_count": sum(
1 for scope in product_scopes if scope["contract_status"] == "contract_ready"
),
"runtime_adapter_active_count": sum(
1 for scope in product_scopes if scope["runtime_adapter_active"] is True
),
"required_dimension_count": len(required_dimensions),
"missing_required_dimension_count": len(missing_dimensions),
"source_family_contract_count": len(source_families),
},
}
def _build_work_item_progress(
*,
trace_ledger: Mapping[str, Any],
log_integration_taxonomy: Mapping[str, Any],
log_controlled_writeback_executor: Mapping[str, Any],
log_controlled_writeback_consumer: Mapping[str, Any],
agent_decision_wiring: Mapping[str, Any],
learning_loop: Mapping[str, Any],
alert_noise_reduction: Mapping[str, Any],
host_sustained_load_automation: Mapping[str, Any],
ui_productization: Mapping[str, Any],
multi_product_taxonomy: Mapping[str, Any],
db_read_status: str,
partial_query_failures: Iterable[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Build ordered work items that the UI and agent can keep advancing."""
taxonomy_rollups = log_integration_taxonomy.get("rollups")
if not isinstance(taxonomy_rollups, Mapping):
taxonomy_rollups = {}
source_families = log_integration_taxonomy.get("source_families")
if not isinstance(source_families, list):
source_families = []
inactive_source_count = _int_value(taxonomy_rollups.get("inactive_source_family_count"))
active_source_count = _int_value(taxonomy_rollups.get("active_source_family_count"))
classified_event_total = _int_value(taxonomy_rollups.get("classified_event_total"))
missing_required = trace_ledger.get("missing_required_stage_ids")
if not isinstance(missing_required, list):
missing_required = []
decision_rollups = agent_decision_wiring.get("rollups")
if not isinstance(decision_rollups, Mapping):
decision_rollups = {}
decision_wiring_missing = _int_value(decision_rollups.get("required_stage_missing_count"))
p1a_completed = inactive_source_count == 0
p1b_completed = (
agent_decision_wiring.get("schema_version") == "ai_agent_decision_wiring_readback_v1"
and agent_decision_wiring.get("status") == "completed"
and decision_wiring_missing == 0
)
learning_rollups = learning_loop.get("rollups")
if not isinstance(learning_rollups, Mapping):
learning_rollups = {}
learning_loop_missing = _int_value(learning_rollups.get("required_stage_missing_count"))
p1c_completed = (
learning_loop.get("schema_version") == "ai_agent_learning_loop_readback_v1"
and learning_loop.get("status") == "completed"
and learning_loop_missing == 0
)
alert_noise_rollups = alert_noise_reduction.get("rollups")
if not isinstance(alert_noise_rollups, Mapping):
alert_noise_rollups = {}
alert_noise_missing = _int_value(alert_noise_rollups.get("required_stage_missing_count"))
p1d_completed = (
alert_noise_reduction.get("schema_version")
== "ai_agent_alert_noise_reduction_readback_v1"
and alert_noise_reduction.get("status") == "completed"
and alert_noise_missing == 0
)
host_load_rollups = host_sustained_load_automation.get("rollups")
if not isinstance(host_load_rollups, Mapping):
host_load_rollups = {}
host_load_ready = (
host_sustained_load_automation.get("schema_version")
== "host_sustained_load_controlled_automation_readback_v1"
and host_sustained_load_automation.get("status") == "completed"
and _int_value(host_load_rollups.get("required_asset_count"))
== _int_value(host_load_rollups.get("ready_asset_count"))
and _int_value(host_load_rollups.get("controlled_apply_class_count")) >= 1
)
log_executor_rollups = log_controlled_writeback_executor.get("rollups")
if not isinstance(log_executor_rollups, Mapping):
log_executor_rollups = {}
log_executor_blockers = log_controlled_writeback_executor.get("active_blockers")
if not isinstance(log_executor_blockers, list):
log_executor_blockers = []
log_executor_ready = (
log_controlled_writeback_executor.get("schema_version")
== "ai_agent_log_controlled_writeback_executor_readback_v1"
and log_controlled_writeback_executor.get("status")
== "controlled_writeback_executor_ready"
and log_executor_rollups.get("controlled_executor_dispatch_ready") is True
and not log_executor_blockers
)
log_consumer_rollups = log_controlled_writeback_consumer.get("rollups")
if not isinstance(log_consumer_rollups, Mapping):
log_consumer_rollups = {}
log_consumer_blockers = log_controlled_writeback_consumer.get("active_blockers")
if not isinstance(log_consumer_blockers, list):
log_consumer_blockers = []
log_consumer_ready = (
log_controlled_writeback_consumer.get("schema_version")
== "ai_agent_log_controlled_writeback_consumer_readback_v1"
and log_controlled_writeback_consumer.get("status")
== "controlled_writeback_consumer_readback_ready"
and log_consumer_rollups.get("controlled_consumer_readback_ready") is True
and not log_consumer_blockers
)
ui_rollups = ui_productization.get("rollups")
if not isinstance(ui_rollups, Mapping):
ui_rollups = {}
ui_surface_missing = _int_value(ui_rollups.get("required_surface_missing_count"))
p2a_completed = (
ui_productization.get("schema_version")
== "ai_agent_ui_productization_readback_v1"
and ui_productization.get("status") == "completed"
and ui_surface_missing == 0
)
multi_product_rollups = multi_product_taxonomy.get("rollups")
if not isinstance(multi_product_rollups, Mapping):
multi_product_rollups = {}
multi_product_missing = len(
multi_product_taxonomy.get("missing_product_scope_ids")
if isinstance(multi_product_taxonomy.get("missing_product_scope_ids"), list)
else []
) + _int_value(multi_product_rollups.get("missing_required_dimension_count"))
p2b_completed = (
multi_product_taxonomy.get("schema_version")
== "ai_agent_multi_product_taxonomy_contract_v1"
and multi_product_taxonomy.get("status") == "completed"
and multi_product_missing == 0
)
auxiliary_only_partial = (
db_read_status == "partial"
and _has_only_auxiliary_runtime_receipt_failures(partial_query_failures)
)
consumer_receipt_fallback_complete = bool(
db_read_status == "unavailable"
and log_consumer_ready
and inactive_source_count == 0
and classified_event_total > 0
)
runtime_readback_complete = bool(
db_read_status == "ok"
or auxiliary_only_partial
or consumer_receipt_fallback_complete
)
runtime_truth_complete = runtime_readback_complete and classified_event_total > 0
deployed_readback_complete = (
runtime_readback_complete
and trace_ledger.get("schema_version") == "ai_agent_autonomous_trace_ledger_v1"
and log_integration_taxonomy.get("schema_version") == "ai_agent_log_integration_taxonomy_v1"
and classified_event_total > 0
)
ordered_items = [
{
"work_item_id": "P0-A-runtime-truth",
"priority": "P0-A",
"title": "Controlled apply runtime truth readback",
"status": (
"completed"
if runtime_truth_complete
else "blocked"
if db_read_status in {"not_queried", "unavailable"}
else "in_progress"
),
"exit_criteria": "production API reports db_read_status=ok, auxiliary-only partial, or live consumer receipt fallback with classified receipts",
"blocker": None
if runtime_truth_complete
else (
"runtime_receipt_db_readback_unavailable"
if db_read_status in {"not_queried", "unavailable"}
else "runtime_receipt_partial_query_failures"
if db_read_status == "partial"
else "live_log_classified_event_total_zero"
),
"db_read_status": db_read_status,
"auxiliary_query_degradation_only": auxiliary_only_partial,
"consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
"classified_event_total": classified_event_total,
},
{
"work_item_id": "P0-B-trace-ledger",
"priority": "P0-B",
"title": "Trace ledger for MCP/log/executor/verifier/KM/PlayBook/Telegram",
"status": "completed" if not missing_required else "in_progress",
"exit_criteria": "trace_ledger exposes required closed-loop stages and missing_required_stage_ids",
},
{
"work_item_id": "P0-C-log-taxonomy",
"priority": "P0-C",
"title": "Project/product/site/service/package/tool log taxonomy",
"status": "completed",
"exit_criteria": "log_integration_taxonomy lists source families, labels, and public-safety policy",
"active_source_family_count": active_source_count,
"classified_event_total": classified_event_total,
},
{
"work_item_id": "P0-D-ui-visibility",
"priority": "P0-D",
"title": "AwoooP UI shows automation loop and log integration progress",
"status": "completed",
"exit_criteria": "AwoooP, Approvals, Runs, and Work Items show trace/log taxonomy panel",
},
{
"work_item_id": "P0-E-verification-deploy",
"priority": "P0-E",
"title": "Focused verification and production deploy marker readback",
"status": "completed" if deployed_readback_complete else "in_progress",
"exit_criteria": "deploy marker includes this code and production API exposes trace_ledger/log_integration_taxonomy, including live consumer receipt fallback during DB timeout",
"blocker": None if deployed_readback_complete else "waiting_for_successful_gitea_cd_deploy_marker",
"consumer_receipt_fallback_complete": consumer_receipt_fallback_complete,
},
{
"work_item_id": "P1-A-ingestion-coverage",
"priority": "P1-A",
"title": "Collector and sanitizer coverage for all source families",
"status": "completed" if p1a_completed else "in_progress",
"exit_criteria": "all source families have active sanitized classified events",
"remaining_source_family_count": inactive_source_count,
},
{
"work_item_id": "P1-B-agent-decision-wiring",
"priority": "P1-B",
"title": "RAG retrieval to PlayBook select/repair/check-mode/apply/verifier",
"status": "completed" if p1b_completed else "in_progress" if p1a_completed else "pending",
"exit_criteria": "AI Agent consumes labeled evidence and emits target selector, dry-run, apply, verifier, rollback",
"remaining_decision_wiring_stage_count": decision_wiring_missing,
},
{
"work_item_id": "P1-C-learning-loop",
"priority": "P1-C",
"title": "KM / PlayBook trust learning loop",
"status": "completed" if p1c_completed else "in_progress" if p1b_completed else "pending",
"exit_criteria": "verified execution updates KM entries, trust delta, similar-case clusters, and repair candidates",
"remaining_learning_loop_stage_count": learning_loop_missing,
},
{
"work_item_id": "P1-D-alert-noise-reduction",
"priority": "P1-D",
"title": "Alert grouping and AI controlled workflow routing",
"status": "completed" if p1d_completed else "in_progress" if p1c_completed else "pending",
"exit_criteria": "repeated alerts are clustered, deduped, routed to controlled automation, and no longer default to manual handling",
"remaining_alert_noise_stage_count": alert_noise_missing,
},
{
"work_item_id": "P1-D2-host-sustained-load-controlled-automation",
"priority": "P1-D2",
"title": "CPU sustained-load alerts classify and run AI controlled remediation",
"status": "completed" if host_load_ready else "in_progress" if p1d_completed else "pending",
"exit_criteria": "HostLoadAverageSustainedHigh routes to classifier, dry-run/check-mode, controlled apply packet, verifier, and KM/PlayBook writeback",
"controlled_action_class_count": _int_value(
host_load_rollups.get("controlled_apply_class_count")
),
"ready_asset_count": _int_value(host_load_rollups.get("ready_asset_count")),
"required_asset_count": _int_value(host_load_rollups.get("required_asset_count")),
},
{
"work_item_id": "P1-E-log-controlled-writeback-executor",
"priority": "P1-E",
"title": "LOG feedback executor queue for KM / RAG / MCP / PlayBook",
"status": "completed" if log_executor_ready else "in_progress" if host_load_ready else "pending",
"exit_criteria": "executor readback exposes ready batches, target selectors, source diffs, rollback, verifier, and next-action queue",
"remaining_executor_batch_count": max(
0,
_int_value(log_executor_rollups.get("execution_batch_count"))
- _int_value(log_executor_rollups.get("ready_execution_batch_count")),
),
"active_blocker_count": len(log_executor_blockers),
},
{
"work_item_id": "P1-F-log-controlled-writeback-consumer",
"priority": "P1-F",
"title": "LOG metadata receipts consumable by KM / RAG / MCP / PlayBook / AI Agent",
"status": "completed" if log_consumer_ready else "in_progress" if log_executor_ready else "pending",
"exit_criteria": "runtime-control exposes ready consumer bindings for all LOG metadata writeback targets",
"remaining_consumer_binding_count": max(
0,
_int_value(log_consumer_rollups.get("target_count"))
- _int_value(log_consumer_rollups.get("ready_target_count")),
),
"active_blocker_count": len(log_consumer_blockers),
},
{
"work_item_id": "P2-A-ui-ux-productization",
"priority": "P2-A",
"title": "Professional product UI replacing text-heavy surfaces",
"status": "completed" if p2a_completed else "in_progress" if log_consumer_ready else "pending",
"exit_criteria": "AI automation status is shown as dense dashboard controls, filters, counters, and action rails",
"remaining_ui_surface_count": ui_surface_missing,
},
{
"work_item_id": "P2-B-multi-product-expansion",
"priority": "P2-B",
"title": "Reuse taxonomy across AWOOOI products/projects",
"status": "completed" if p2b_completed else "in_progress" if p2a_completed else "pending",
"exit_criteria": "StockPlatform, VibeWork, MOMO, AwoooGo, and other products report the same log taxonomy contract",
"remaining_product_scope_count": multi_product_missing,
},
]
source_family_items = []
for source in source_families:
if not isinstance(source, Mapping):
continue
total = _int_value(source.get("total"))
source_family_items.append({
"work_item_id": f"P1-A-source-{source.get('source_family_id')}",
"priority": "P1-A",
"source_family_id": source.get("source_family_id"),
"title": f"Ingest and label {source.get('source_family_id')}",
"status": "completed" if total > 0 else "not_started",
"label_dimensions": source.get("label_dimensions") or [],
"next_controlled_action": (
"keep_learning_and_quality_checks"
if total > 0
else source.get("next_action_if_empty")
),
})
all_items = [*ordered_items, *source_family_items]
by_status: dict[str, int] = {}
for item in all_items:
status = str(item.get("status") or "unknown")
by_status[status] = by_status.get(status, 0) + 1
return {
"schema_version": "ai_agent_automation_work_item_progress_v1",
"ordered_items": ordered_items,
"source_family_items": source_family_items,
"rollups": {
"work_item_count": len(all_items),
"ordered_work_item_count": len(ordered_items),
"source_family_work_item_count": len(source_family_items),
"completed_count": by_status.get("completed", 0),
"in_progress_count": by_status.get("in_progress", 0),
"pending_count": by_status.get("pending", 0),
"blocked_count": by_status.get("blocked", 0),
"not_started_count": by_status.get("not_started", 0),
"by_status": by_status,
},
}
def _first_operation(
rows: Iterable[Mapping[str, Any]],
operation_type: str,
*,
statuses: set[str] | None = None,
) -> dict[str, Any] | None:
for row in rows:
if (
str(row.get("operation_type") or "") == operation_type
and (
statuses is None
or str(row.get("status") or "") in statuses
)
):
return dict(row)
return None
def _operation_by_id(
rows: Iterable[Mapping[str, Any]],
op_id: Any,
) -> dict[str, Any] | None:
needle = str(op_id or "")
if not needle:
return None
for row in rows:
if str(row.get("op_id") or "") == needle:
return dict(row)
return None
def _missing_runtime_operation_chain_ref_ids(
rows: Iterable[Mapping[str, Any] | Any],
) -> list[str]:
"""Return exact apply/check/candidate receipt ids missing from a bounded read."""
operation_rows = [_row_mapping(row) for row in rows]
latest_observed_apply = _first_operation(
operation_rows,
"ansible_apply_executed",
)
latest_apply = _first_operation(
operation_rows,
"ansible_apply_executed",
statuses={"success", "failed"},
) or latest_observed_apply
if latest_apply is None:
return []
check_mode_op_id = str(
latest_apply.get("check_mode_op_id")
or latest_apply.get("parent_op_id")
or ""
)
latest_check = _operation_by_id(operation_rows, check_mode_op_id)
candidate_op_id = str(
(latest_check or {}).get("parent_op_id")
or (latest_check or {}).get("source_candidate_op_id")
or latest_apply.get("source_candidate_op_id")
or ""
)
present_ids = {
str(row.get("op_id") or "")
for row in operation_rows
if str(row.get("op_id") or "")
}
refs = [
str(latest_apply.get("op_id") or ""),
check_mode_op_id,
candidate_op_id,
]
return list(dict.fromkeys(ref for ref in refs if ref and ref not in present_ids))
def _merge_runtime_operation_rows(
primary_rows: Iterable[Mapping[str, Any] | Any],
supplemental_rows: Iterable[Mapping[str, Any] | Any],
) -> list[dict[str, Any]]:
merged: list[dict[str, Any]] = []
seen_ids: set[str] = set()
for raw_row in (*list(primary_rows), *list(supplemental_rows)):
row = _row_mapping(raw_row)
op_id = str(row.get("op_id") or "")
if op_id and op_id in seen_ids:
continue
if op_id:
seen_ids.add(op_id)
merged.append(row)
return merged
def _runtime_stage_receipt_has_required_proof(receipt: Mapping[str, Any]) -> bool:
"""Reject learning receipts that do not prove the durable write."""
stage_id = str(receipt.get("stage_id") or "")
if stage_id not in {
"km_playbook_writeback",
"rag_writeback",
"playbook_trust",
}:
return True
detail = receipt.get("detail")
if not isinstance(detail, Mapping):
return False
if detail.get("durable_write_acknowledged") is not True:
return False
if stage_id == "km_playbook_writeback":
return bool(
detail.get("schema_version") == "ansible_km_writeback_v1"
and str(detail.get("knowledge_entry_id") or "")
and str(detail.get("canonical_playbook_id") or "")
and str(detail.get("km_row_version") or "")
and detail.get("repository_write_acknowledged") is True
and detail.get("repository_readback_verified") is True
and detail.get("raw_log_payload_stored") is False
and detail.get("secret_value_stored") is False
and str(detail.get("writer_source_sha") or "")
)
if stage_id == "rag_writeback":
primary_embedding = bool(
detail.get("embedding_persisted") is True
and str(detail.get("knowledge_entry_id") or "")
)
chunk_fallback = bool(
detail.get("chunk_index_verified") is True
and _int_value(detail.get("rag_chunk_count")) > 0
and str(detail.get("source_id") or "")
)
return primary_embedding or chunk_fallback
return bool(
detail.get("schema_version")
== "ansible_playbook_trust_writeback_v2"
and detail.get("identity_schema_version")
== "ansible_playbook_identity_v1"
and str(detail.get("identity_fingerprint") or "")
and str(detail.get("canonical_playbook_id") or "")
and _int_value(detail.get("playbook_row_version")) > 0
and str(detail.get("playbook_row_updated_at") or "")
and str(detail.get("playbook_row_fingerprint") or "")
and detail.get("trust_score") is not None
and _int_value(detail.get("trust_observation_count")) > 0
and detail.get("learning_recorded") is True
and detail.get("trust_updated") is True
and detail.get("repository_write_acknowledged") is True
and detail.get("repository_readback_verified") is True
and detail.get("operation_receipt_readback_verified") is True
and detail.get("raw_log_payload_stored") is False
and detail.get("secret_value_stored") is False
and str(detail.get("writer_source_sha") or "")
)
def _runtime_stage_receipt_map(
rows: Iterable[Mapping[str, Any] | Any],
*,
expected_run_id: str,
) -> dict[str, dict[str, Any]]:
receipts: dict[str, dict[str, Any]] = {}
if not expected_run_id:
return receipts
for raw_row in rows:
row = _row_mapping(raw_row)
if str(row.get("automation_run_id") or "") != expected_run_id:
continue
raw_receipts = row.get("runtime_stage_receipts")
if isinstance(raw_receipts, str):
try:
raw_receipts = json.loads(raw_receipts)
except json.JSONDecodeError:
continue
if not isinstance(raw_receipts, list):
continue
for raw_receipt in raw_receipts:
if not isinstance(raw_receipt, Mapping):
continue
receipt = dict(raw_receipt)
stage_id = str(receipt.get("stage_id") or "")
if (
stage_id not in AI_AUTOMATION_REQUIRED_LOOP_STAGES
or receipt.get("schema_version")
!= AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION
or str(receipt.get("automation_run_id") or "")
!= expected_run_id
or receipt.get("durable_receipt") is not True
or not str(receipt.get("evidence_ref") or "")
or not _runtime_stage_receipt_has_required_proof(receipt)
):
continue
receipts[stage_id] = receipt
return receipts
def _stage_status(row: Mapping[str, Any] | None, *, fallback_status: str | None = None) -> str:
if row is None:
return fallback_status or "missing"
return str(row.get("status") or row.get("result_status") or fallback_status or "present")
def _loop_stage(
*,
stage_id: str,
receipt_source: str,
present: bool,
status: str,
ref_id: str | None,
run_id: str | None,
expected_run_id: str | None,
writes_runtime_state: bool,
next_action_if_missing: str,
) -> dict[str, Any]:
run_id_present = bool(run_id)
run_id_matches_expected = bool(
present
and run_id_present
and expected_run_id
and str(run_id) == str(expected_run_id)
)
return {
"stage_id": stage_id,
"receipt_source": receipt_source,
"present": present,
"status": status,
"ref_id": ref_id,
"run_id": run_id,
"run_id_present": run_id_present,
"run_id_matches_expected": run_id_matches_expected,
"writes_runtime_state": writes_runtime_state,
"next_action_if_missing": (
None if run_id_matches_expected else next_action_if_missing
),
}
def _autonomous_execution_loop_ledger(
*,
project_id: str,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
verifier_latest_rows: Iterable[Mapping[str, Any] | Any],
km_latest_rows: Iterable[Mapping[str, Any] | Any],
telegram_latest_rows: Iterable[Mapping[str, Any] | Any],
auto_repair_latest_rows: Iterable[Mapping[str, Any] | Any],
latest_flow_closure: Mapping[str, Any],
latest_failure_classification: Mapping[str, Any],
controlled_retry_package: Mapping[str, Any],
) -> dict[str, Any]:
"""Build the operation-id ledger that proves whether the runtime loop closed."""
operation_rows = [_row_mapping(row) for row in operation_latest_rows]
verifier_rows = [_row_mapping(row) for row in verifier_latest_rows]
km_rows = [_row_mapping(row) for row in km_latest_rows]
telegram_rows = [_row_mapping(row) for row in telegram_latest_rows]
auto_repair_rows = [_row_mapping(row) for row in auto_repair_latest_rows]
latest_terminal_apply = _first_operation(
operation_rows,
"ansible_apply_executed",
statuses={"success", "failed"},
)
latest_inflight_apply = _first_operation(
operation_rows,
"ansible_apply_executed",
statuses={"pending"},
)
latest_apply = latest_terminal_apply or latest_inflight_apply
latest_check = None
latest_candidate = None
if latest_apply is not None:
latest_check = _operation_by_id(
operation_rows,
latest_apply.get("check_mode_op_id") or latest_apply.get("parent_op_id"),
)
else:
latest_check = _first_operation(operation_rows, "ansible_check_mode_executed")
source_candidate_op_id = None
if latest_check is not None:
source_candidate_op_id = latest_check.get("parent_op_id") or latest_check.get("source_candidate_op_id")
if latest_apply is not None and not source_candidate_op_id:
source_candidate_op_id = latest_apply.get("source_candidate_op_id")
latest_candidate = _operation_by_id(operation_rows, source_candidate_op_id)
if latest_candidate is None and latest_apply is None and latest_check is None:
latest_candidate = _first_operation(operation_rows, "ansible_candidate_matched")
anchor = latest_apply or latest_check or latest_candidate or {}
apply_op_id = str((latest_apply or {}).get("op_id") or "")
check_mode_op_id = str(
(latest_check or {}).get("op_id")
or (latest_apply or {}).get("check_mode_op_id")
or (latest_apply or {}).get("parent_op_id")
or ""
)
candidate_op_id = str(
(latest_candidate or {}).get("op_id")
or source_candidate_op_id
or ""
)
incident_id = str(anchor.get("incident_id") or "")
catalog_id = str(anchor.get("catalog_id") or "")
playbook_path = str(anchor.get("playbook_path") or "")
verifier = next(
(
row
for row in verifier_rows
if apply_op_id and str(row.get("apply_op_id") or "") == apply_op_id
),
None,
)
km_path_type = f"ansible_apply_receipt:{apply_op_id[:8]}" if apply_op_id else ""
km = next(
(
row
for row in km_rows
if (
km_path_type
and str(row.get("path_type") or "") == km_path_type
)
or (
incident_id
and str(row.get("related_incident_id") or "") == incident_id
)
),
None,
)
telegram = next(
(
row
for row in telegram_rows
if str(row.get("send_status") or "") == "sent"
and str(row.get("action") or "") == "controlled_apply_result"
and (
not incident_id
or str(row.get("incident_id") or "") == incident_id
)
),
None,
)
auto_repair = next(
(
row
for row in auto_repair_rows
if apply_op_id
and apply_op_id
in str(row.get("executed_steps_text") or row.get("executed_steps") or "")
),
None,
)
automation_run_id = str(
candidate_op_id
or (latest_check or {}).get("automation_run_id")
or (latest_apply or {}).get("automation_run_id")
or ""
)
same_run_stage_receipt_by_id = _runtime_stage_receipt_map(
operation_rows,
expected_run_id=automation_run_id,
)
candidate_present = latest_candidate is not None
check_present = latest_check is not None
apply_present = latest_apply is not None
auto_repair_present = auto_repair is not None
verifier_present = verifier is not None
km_present = km is not None
telegram_present = telegram is not None
stages = [
_loop_stage(
stage_id="candidate",
receipt_source="automation_operation_log:ansible_candidate_matched",
present=candidate_present,
status=_stage_status(latest_candidate, fallback_status="inferred_from_check_mode")
if candidate_present
else "missing",
ref_id=candidate_op_id or None,
run_id=candidate_op_id or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=False,
next_action_if_missing="candidate_backfill_worker_enqueue_allowlisted_playbook",
),
_loop_stage(
stage_id="check_mode",
receipt_source="automation_operation_log:ansible_check_mode_executed",
present=check_present,
status=_stage_status(latest_check, fallback_status="inferred_from_apply_parent")
if check_present
else "missing",
ref_id=check_mode_op_id or None,
run_id=str((latest_check or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=False,
next_action_if_missing="ansible_check_mode_worker_claims_candidate",
),
_loop_stage(
stage_id="controlled_apply",
receipt_source="automation_operation_log:ansible_apply_executed",
present=apply_present,
status=_stage_status(latest_apply),
ref_id=apply_op_id or None,
run_id=str((latest_apply or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=True,
next_action_if_missing="controlled_apply_worker_waits_for_check_mode_success",
),
_loop_stage(
stage_id="auto_repair_execution_receipt",
receipt_source="auto_repair_executions:ansible_controlled_apply",
present=auto_repair_present,
status=str((auto_repair or {}).get("result_status") or "missing"),
ref_id=str((auto_repair or {}).get("id") or "") or None,
run_id=str((auto_repair or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=True,
next_action_if_missing="receipt_backfill_records_auto_repair_execution",
),
_loop_stage(
stage_id="post_apply_verifier",
receipt_source="incident_evidence.post_execution_state",
present=verifier_present,
status=str((verifier or {}).get("verification_result") or "missing"),
ref_id=str((verifier or {}).get("id") or "") or None,
run_id=str((verifier or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=True,
next_action_if_missing="post_apply_verifier_writes_incident_evidence",
),
_loop_stage(
stage_id="km_playbook_writeback",
receipt_source="knowledge_entries:ansible_apply_receipt",
present=km_present,
status=str((km or {}).get("status") or "missing"),
ref_id=str((km or {}).get("id") or "") or None,
run_id=str((km or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=True,
next_action_if_missing="hermes_writes_km_playbook_trust_candidate",
),
_loop_stage(
stage_id="telegram_receipt",
receipt_source="awooop_outbound_message:controlled_apply_result",
present=telegram_present,
status=str((telegram or {}).get("send_status") or "missing"),
ref_id=str((telegram or {}).get("message_id") or "") or None,
run_id=str((telegram or {}).get("automation_run_id") or "") or None,
expected_run_id=automation_run_id or None,
writes_runtime_state=True,
next_action_if_missing="live_apply_gateway_sends_controlled_apply_result_receipt",
),
]
missing_stage_ids = [
str(stage["stage_id"])
for stage in stages
if stage["run_id_matches_expected"] is not True
]
run_id_mismatch_stage_ids = [
str(stage["stage_id"])
for stage in stages
if stage["present"] is True
and stage["run_id_matches_expected"] is not True
]
same_run_correlation = bool(
automation_run_id
and not missing_stage_ids
and not run_id_mismatch_stage_ids
)
closed = bool(
apply_op_id
and auto_repair_present
and same_run_correlation
and latest_flow_closure.get("closed") is True
)
classification = str(latest_failure_classification.get("classification") or "")
if not candidate_present and not check_present and not apply_present:
execution_state = "waiting_for_candidate"
next_executor_action = "candidate_backfill_worker_waits_for_matching_incident"
elif not apply_present:
execution_state = "executor_in_progress_or_waiting"
next_executor_action = "continue_candidate_to_check_mode_to_apply"
elif closed and classification == "latest_controlled_apply_closed_success":
execution_state = "closed_success"
next_executor_action = "keep_receipt_chain_closed"
elif closed:
execution_state = "closed_failed_apply_repair_ready"
next_executor_action = str(
controlled_retry_package.get("next_ai_action")
or "run_no_write_check_mode_replay"
)
elif "telegram_receipt" in missing_stage_ids:
execution_state = "open_waiting_for_live_gateway_receipt"
next_executor_action = "do_not_fake_send_backfill_wait_for_live_apply_gateway"
else:
execution_state = "open_missing_internal_receipts"
next_executor_action = "backfill_missing_auto_repair_verifier_km_receipts"
return {
"schema_version": "ai_agent_autonomous_execution_loop_ledger_v1",
"project_id": project_id,
"operation_id": apply_op_id or check_mode_op_id or candidate_op_id or None,
"completion_anchor": (
"latest_terminal_apply"
if latest_apply is not None
and str(latest_apply.get("status") or "") in {"success", "failed"}
else "latest_observed_operation"
),
"latest_inflight_apply": (
{
"op_id": latest_inflight_apply.get("op_id"),
"status": latest_inflight_apply.get("status"),
"automation_run_id": latest_inflight_apply.get("automation_run_id"),
"incident_id": latest_inflight_apply.get("incident_id"),
"created_at": latest_inflight_apply.get("created_at"),
}
if latest_inflight_apply is not None
else None
),
"automation_run_id": automation_run_id or None,
"root_candidate_op_id": candidate_op_id or None,
"check_mode_op_id": check_mode_op_id or None,
"apply_op_id": apply_op_id or None,
"incident_id": incident_id or None,
"catalog_id": catalog_id or None,
"playbook_path": playbook_path or None,
"execution_state": execution_state,
"closed": closed,
"same_run_correlation": same_run_correlation,
"run_id_mismatch_stage_ids": run_id_mismatch_stage_ids,
"missing_stage_ids": missing_stage_ids,
"next_executor_action": next_executor_action,
"same_run_stage_receipts": [
same_run_stage_receipt_by_id[stage_id]
for stage_id in AI_AUTOMATION_REQUIRED_LOOP_STAGES
if stage_id in same_run_stage_receipt_by_id
],
"stages": stages,
"safety_contract": {
"writes_on_read": False,
"backfill_may_write_auto_repair_verifier_km": True,
"backfill_may_send_telegram": False,
"live_apply_may_send_telegram_gateway_receipt": True,
"reads_raw_sessions": False,
"reads_secret_values": False,
},
}
def _latest_flow_closure(
*,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
verifier_latest_rows: Iterable[Mapping[str, Any] | Any],
km_latest_rows: Iterable[Mapping[str, Any] | Any],
telegram_latest_rows: Iterable[Mapping[str, Any] | Any],
) -> dict[str, Any]:
operation_rows = [_row_mapping(row) for row in operation_latest_rows]
verifier_rows = [_row_mapping(row) for row in verifier_latest_rows]
km_rows = [_row_mapping(row) for row in km_latest_rows]
telegram_rows = [_row_mapping(row) for row in telegram_latest_rows]
latest_apply = next(
(
row
for row in operation_rows
if str(row.get("operation_type") or "") == "ansible_apply_executed"
),
None,
)
if latest_apply is None:
return {
"apply_op_id": None,
"automation_run_id": None,
"incident_id": None,
"has_post_apply_verifier": False,
"has_km_writeback": False,
"has_telegram_receipt": False,
"same_run_correlation": False,
"run_id_mismatch_stage_ids": [],
"closed": False,
"missing": [
"ansible_apply_executed",
"post_apply_verifier",
"km_writeback",
"telegram_receipt",
],
}
apply_op_id = str(latest_apply.get("op_id") or "")
automation_run_id = str(latest_apply.get("automation_run_id") or "")
incident_id = str(latest_apply.get("incident_id") or "")
km_path_type = f"ansible_apply_receipt:{apply_op_id[:8]}" if apply_op_id else ""
verifier = next(
(
row
for row in verifier_rows
if str(row.get("apply_op_id") or "") == apply_op_id
),
None,
)
km = next(
(
row
for row in km_rows
if str(row.get("path_type") or "") == km_path_type
or (
incident_id
and str(row.get("related_incident_id") or "") == incident_id
)
),
None,
)
telegram = next(
(
row
for row in telegram_rows
if str(row.get("send_status") or "") == "sent"
and str(row.get("action") or "") == "controlled_apply_result"
and (
not incident_id
or str(row.get("incident_id") or "") == incident_id
)
),
None,
)
correlated_receipts = {
"post_apply_verifier": verifier,
"km_writeback": km,
"telegram_receipt": telegram,
}
receipt_run_matches = {
stage_id: bool(
automation_run_id
and row is not None
and str(row.get("automation_run_id") or "") == automation_run_id
)
for stage_id, row in correlated_receipts.items()
}
has_verifier = receipt_run_matches["post_apply_verifier"]
has_km = receipt_run_matches["km_writeback"]
has_telegram = receipt_run_matches["telegram_receipt"]
run_id_mismatch_stage_ids = [
stage_id
for stage_id, row in correlated_receipts.items()
if row is not None and not receipt_run_matches[stage_id]
]
missing = [
name
for name, present in (
("post_apply_verifier", has_verifier),
("km_writeback", has_km),
("telegram_receipt", has_telegram),
)
if not present
]
if not automation_run_id:
missing.insert(0, "automation_run_id")
run_id_mismatch_stage_ids.insert(0, "controlled_apply")
same_run_correlation = bool(
automation_run_id
and not run_id_mismatch_stage_ids
and has_verifier
and has_km
and has_telegram
)
return {
"apply_op_id": apply_op_id or None,
"automation_run_id": automation_run_id or None,
"incident_id": incident_id or None,
"has_post_apply_verifier": has_verifier,
"has_km_writeback": has_km,
"has_telegram_receipt": has_telegram,
"same_run_correlation": same_run_correlation,
"run_id_mismatch_stage_ids": run_id_mismatch_stage_ids,
"closed": not missing and same_run_correlation,
"missing": missing,
}
def _latest_execution_capability_lifecycle(
*,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
operation_summary: Mapping[str, Any],
) -> dict[str, Any]:
"""Correlate issuance, apply, and terminal lease receipts for one run."""
rows = [_row_mapping(row) for row in operation_latest_rows]
latest_apply = _first_operation(
rows,
"ansible_apply_executed",
statuses={"success", "failed"},
)
automation_run_id = str((latest_apply or {}).get("automation_run_id") or "")
issuance = next(
(
row
for row in rows
if str(row.get("operation_type") or "")
== "ansible_executor_capability_issued"
and str(row.get("automation_run_id") or "") == automation_run_id
and str(row.get("status") or "") == "success"
),
None,
)
capability_op_id = str((issuance or {}).get("op_id") or "")
terminal = next(
(
row
for row in rows
if str(row.get("operation_type") or "")
in {
"ansible_executor_capability_revoked",
"ansible_executor_capability_expired",
}
and str(row.get("automation_run_id") or "") == automation_run_id
and str(row.get("parent_op_id") or "") == capability_op_id
and str(row.get("status") or "") == "success"
),
None,
)
apply_capability_op_id = str(
(latest_apply or {}).get("capability_op_id") or ""
)
missing: list[str] = []
if latest_apply is None:
missing.append("ansible_apply_executed")
if not automation_run_id:
missing.append("automation_run_id")
if issuance is None:
missing.append("ansible_executor_capability_issued")
if not apply_capability_op_id:
missing.append("apply_capability_op_id")
elif capability_op_id and apply_capability_op_id != capability_op_id:
missing.append("apply_capability_op_id_mismatch")
if terminal is None:
missing.append("capability_terminal_receipt")
if issuance is not None and not issuance.get("capability_issued_at"):
missing.append("capability_issued_at")
if issuance is not None and not issuance.get("capability_expires_at"):
missing.append("capability_expires_at")
terminal_operation_type = str(
(terminal or {}).get("operation_type") or ""
)
operation_counts = {
operation_type: _int_value(
(operation_summary.get(operation_type) or {}).get("total")
)
for operation_type in (
"ansible_executor_capability_issued",
"ansible_executor_capability_revoked",
"ansible_executor_capability_expired",
)
}
return {
"schema_version": "ansible_execution_capability_lifecycle_v1",
"automation_run_id": automation_run_id or None,
"apply_op_id": str((latest_apply or {}).get("op_id") or "") or None,
"capability_op_id": capability_op_id or None,
"apply_capability_op_id": apply_capability_op_id or None,
"terminal_operation_type": terminal_operation_type or None,
"terminal_status": (
str((terminal or {}).get("capability_terminal_status") or "")
or None
),
"issued_at": (issuance or {}).get("capability_issued_at"),
"expires_at": (issuance or {}).get("capability_expires_at"),
"same_run_correlation": bool(
automation_run_id
and issuance is not None
and terminal is not None
and apply_capability_op_id == capability_op_id
),
"closed": not missing,
"missing": missing,
"operation_counts": operation_counts,
"semantic_operation_fallback_supported": True,
"native_schema_operation_types_required": False,
"writes_on_read": False,
}
def _latest_failure_classification(
*,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
verifier_latest_rows: Iterable[Mapping[str, Any] | Any],
latest_flow_closure: Mapping[str, Any],
) -> dict[str, Any]:
"""Classify the newest controlled apply outcome without exposing command output."""
operation_rows = [_row_mapping(row) for row in operation_latest_rows]
verifier_rows = [_row_mapping(row) for row in verifier_latest_rows]
latest_apply = next(
(
row
for row in operation_rows
if str(row.get("operation_type") or "") == "ansible_apply_executed"
),
None,
)
if latest_apply is None:
return {
"schema_version": "ai_agent_executor_failure_classification_v1",
"classification": "no_controlled_apply_observed",
"action": "wait_for_controlled_apply_receipt",
"target_selector": {},
"evidence": {
"latest_flow_closed": False,
"output_tail_in_readback": False,
"unredacted_output_required": False,
},
}
apply_op_id = str(latest_apply.get("op_id") or "")
incident_id = str(latest_apply.get("incident_id") or "")
returncode = _int_value(latest_apply.get("returncode"))
verifier = next(
(
row
for row in verifier_rows
if apply_op_id and str(row.get("apply_op_id") or "") == apply_op_id
),
{},
)
verification_result = str(verifier.get("verification_result") or "").lower()
latest_flow_closed = latest_flow_closure.get("closed") is True
if returncode == 0 and verification_result in {"success", ""} and latest_flow_closed:
classification = "latest_controlled_apply_closed_success"
action = "keep_receipt_chain_closed"
elif returncode == 0:
classification = "controlled_apply_success_receipt_gap"
action = "backfill_missing_verifier_km_or_telegram_receipt"
elif latest_flow_closed:
classification = "closed_failed_apply_requires_ai_repair"
action = "queue_check_mode_replay_and_playbook_repair_candidate"
else:
classification = "failed_apply_receipt_gap_requires_backfill_then_repair"
action = "backfill_missing_receipts_then_queue_repair_candidate"
return {
"schema_version": "ai_agent_executor_failure_classification_v1",
"classification": classification,
"action": action,
"target_selector": {
"incident_id": incident_id or None,
"apply_op_id": apply_op_id or None,
"parent_op_id": latest_apply.get("parent_op_id"),
"catalog_id": latest_apply.get("catalog_id"),
"playbook_path": latest_apply.get("playbook_path"),
"execution_mode": latest_apply.get("execution_mode"),
},
"evidence": {
"operation_status": latest_apply.get("status"),
"returncode": latest_apply.get("returncode"),
"verification_result": verification_result or None,
"latest_flow_closed": latest_flow_closed,
"has_post_apply_verifier": latest_flow_closure.get("has_post_apply_verifier") is True,
"has_km_writeback": latest_flow_closure.get("has_km_writeback") is True,
"has_telegram_receipt": latest_flow_closure.get("has_telegram_receipt") is True,
"output_tail_in_readback": False,
"unredacted_output_required": False,
},
"safe_next_steps": [
"run_no_write_check_mode_replay",
"extract_sanitized_failed_task_summary",
"write_km_playbook_repair_candidate",
"retry_controlled_apply_only_after_check_mode_passes",
],
}
def _controlled_retry_package(classification: Mapping[str, Any]) -> dict[str, Any]:
"""Build the next no-write repair package from the public failure classification."""
target_selector = classification.get("target_selector")
if not isinstance(target_selector, Mapping):
target_selector = {}
apply_op_id = str(target_selector.get("apply_op_id") or "")
repair_required = classification.get("classification") in {
"closed_failed_apply_requires_ai_repair",
"failed_apply_receipt_gap_requires_backfill_then_repair",
}
return {
"schema_version": "ai_agent_controlled_retry_package_v1",
"package_id": (
f"ansible_retry:{apply_op_id[:8]}"
if repair_required and apply_op_id
else None
),
"status": (
"ready_for_no_write_check_mode_replay"
if repair_required
else "not_required_for_latest_apply"
),
"target_selector": dict(target_selector),
"source_of_truth": {
"catalog_id": target_selector.get("catalog_id"),
"playbook_path": target_selector.get("playbook_path"),
"source_diff_required_before_retry": True,
"failed_task_summary_required": True,
},
"preflight": {
"no_write_check_mode_replay_required": repair_required,
"reuse_parent_check_mode_op_id": target_selector.get("parent_op_id"),
"unredacted_output_required": False,
"secret_value_collection_allowed": False,
},
"apply_gate": {
"controlled_apply_retry_allowed_now": False,
"opens_legacy_runner": False,
"requires_check_mode_success_before_apply": repair_required,
},
"rollback": {
"rollback_candidate_required": repair_required,
"destructive_rollback_allowed": False,
"rollback_plan_source": "playbook_repair_candidate_after_failed_task_summary",
},
"post_apply": {
"post_apply_verifier_required": repair_required,
"km_playbook_trust_writeback_required": repair_required,
"telegram_receipt_required": repair_required,
},
"next_ai_action": (
"run_no_write_check_mode_replay"
if repair_required
else "keep_latest_apply_receipts"
),
}
def classify_deploy_control_plane_observation(
*,
run_status: str,
is_latest_deploy_intent: bool,
active_task_container_count: int,
production_marker_hit: bool,
latest_flow_closed: bool,
runner_capacity_ok: bool,
runner_forbidden_label_count: int,
) -> dict[str, Any]:
"""Classify CD/run noise into an internal PlayBook decision."""
normalized_status = str(run_status or "unknown").strip().lower()
has_active_task = active_task_container_count > 0
runner_lane_safe = runner_capacity_ok and runner_forbidden_label_count == 0
production_truth_ok = production_marker_hit and latest_flow_closed
if not is_latest_deploy_intent:
classification = "superseded_run_skip"
action = "skip_cd_work_and_attach_to_superseded_intent"
elif production_truth_ok and normalized_status == "success":
classification = "deploy_succeeded_marker_hit"
action = "close_deploy_intent_and_write_receipts"
elif normalized_status == "running" and has_active_task and runner_lane_safe:
classification = "running_with_controlled_task"
action = "continue_observing_without_restarting_runner"
elif normalized_status == "running" and not has_active_task and production_truth_ok:
classification = "running_no_container_stale_ui"
action = "treat_gitea_spinner_as_stale_and_keep_production_truth"
elif normalized_status == "failure" and production_truth_ok:
classification = "failed_run_superseded_by_marker_hit"
action = "record_non_blocking_failure_and_keep_current_marker"
elif normalized_status == "failure":
classification = "real_failure_requires_playbook_repair"
action = "open_cd_repair_playbook_with_target_selector_and_verifier"
elif not runner_lane_safe:
classification = "runner_lane_guardrail_violation"
action = "fail_closed_runner_lane_and_open_repair_playbook"
else:
classification = "waiting_for_controlled_observation"
action = "wait_for_mcp_observation_or_deploy_intent_update"
return {
"schema_version": "ai_agent_deploy_control_plane_decision_v1",
"classification": classification,
"action": action,
"inputs": {
"run_status": normalized_status,
"is_latest_deploy_intent": is_latest_deploy_intent,
"active_task_container_count": max(0, active_task_container_count),
"production_marker_hit": production_marker_hit,
"latest_flow_closed": latest_flow_closed,
"runner_capacity_ok": runner_capacity_ok,
"runner_forbidden_label_count": max(0, runner_forbidden_label_count),
},
"internal_writeback": {
"mcp_event_type": "deploy_run_observation",
"rag_context_required": True,
"km_writeback_required": True,
"playbook_route_required": True,
"log_projection_required": True,
"telegram_receipt_required": classification in {
"deploy_succeeded_marker_hit",
"real_failure_requires_playbook_repair",
"runner_lane_guardrail_violation",
},
},
"safety_boundary": {
"reads_raw_sessions": False,
"reads_secret_values": False,
"opens_legacy_runner": False,
"uses_force_push": False,
"writes_runtime_state": classification in {
"deploy_succeeded_marker_hit",
"real_failure_requires_playbook_repair",
"runner_lane_guardrail_violation",
},
},
}
def _control_plane_integration() -> dict[str, Any]:
classifier_examples = [
classify_deploy_control_plane_observation(
run_status="success",
is_latest_deploy_intent=True,
active_task_container_count=0,
production_marker_hit=True,
latest_flow_closed=True,
runner_capacity_ok=True,
runner_forbidden_label_count=0,
),
classify_deploy_control_plane_observation(
run_status="running",
is_latest_deploy_intent=True,
active_task_container_count=0,
production_marker_hit=True,
latest_flow_closed=True,
runner_capacity_ok=True,
runner_forbidden_label_count=0,
),
classify_deploy_control_plane_observation(
run_status="failure",
is_latest_deploy_intent=True,
active_task_container_count=0,
production_marker_hit=False,
latest_flow_closed=False,
runner_capacity_ok=True,
runner_forbidden_label_count=0,
),
]
return {
"schema_version": "ai_agent_autonomous_runtime_internal_loop_v1",
"status": "mcp_rag_km_playbook_log_control_loop_declared",
"purpose": (
"把 Gitea run、runner lane、production marker、browser smoke 與 executor receipt "
"先收斂成內部事件,再由 PlayBook decision 推進或跳過。"
),
"mcp_sensors": [
{
"sensor_id": "gitea_actions_run_observer",
"normalized_event": "RunObservation",
"raw_secret_access_allowed": False,
},
{
"sensor_id": "controlled_runner_lane_observer",
"normalized_event": "RunnerLaneState",
"raw_runner_token_access_allowed": False,
},
{
"sensor_id": "production_marker_observer",
"normalized_event": "ProductionTruthSnapshot",
"raw_session_access_allowed": False,
},
{
"sensor_id": "browser_smoke_observer",
"normalized_event": "FrontendTruthSnapshot",
"raw_conversation_access_allowed": False,
},
],
"rag_context_queries": [
"runner_pressure_buildkit_stockplatform_collision",
"controlled_cd_lane_capacity_label_guardrails",
"autonomous_runtime_marker_receipt_contract",
],
"playbook_decision_classes": [
"deploy_succeeded_marker_hit",
"running_with_controlled_task",
"running_no_container_stale_ui",
"superseded_run_skip",
"failed_run_superseded_by_marker_hit",
"real_failure_requires_playbook_repair",
"runner_lane_guardrail_violation",
],
"km_writeback_contract": {
"knowledge_entry_path_type": "deploy_control_plane_decision:<deploy_intent_id>",
"required_refs": [
"deploy_intent_id",
"target_sha",
"gitea_run_id",
"production_marker",
"latest_flow_closure",
"runner_lane_state",
],
"stores_raw_logs": False,
"stores_secret_values": False,
},
"log_projection_contract": {
"timeline_event_type": "ai_agent_deploy_control_plane_decision",
"logbook_projection": "summary_only_after_verifier",
"raw_html_or_long_log_allowed": False,
},
"classifier_examples": classifier_examples,
"rollups": {
"mcp_sensor_count": 4,
"rag_context_query_count": 3,
"playbook_decision_class_count": 7,
"classifier_example_count": len(classifier_examples),
},
}
def _runtime_truthy(value: Any) -> bool:
if isinstance(value, bool):
return value
return str(value or "").strip().lower() == "true"
def _build_autonomous_single_writer_runtime_readback(
*,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
loop_ledger: Mapping[str, Any],
) -> dict[str, Any]:
rows = [_row_mapping(row) for row in operation_latest_rows]
automation_run_id = str(loop_ledger.get("automation_run_id") or "")
run_rows = [
row
for row in rows
if str(row.get("automation_run_id") or "") == automation_run_id
]
def _operation(operation_type: str) -> dict[str, Any]:
return next(
(
row
for row in run_rows
if str(row.get("operation_type") or "") == operation_type
),
{},
)
candidate = _operation("ansible_candidate_matched")
check_mode = _operation("ansible_check_mode_executed")
controlled_apply = _operation("ansible_apply_executed")
writer_rows = [candidate, check_mode, controlled_apply]
router_source_sha = str(candidate.get("router_source_sha") or "").lower()
controls = {
"controlled_queue_decision_path": (
candidate.get("decision_path") == "repair_candidate_controlled_queue"
),
"single_writer_executor": bool(
all(writer_rows)
and all(
row.get("single_writer_executor")
== "awoooi-ansible-executor-broker"
for row in writer_rows
)
),
"candidate_idempotency_key_present": bool(
str(candidate.get("candidate_idempotency_key") or "")
),
"target_selector_present": _runtime_truthy(
candidate.get("target_selector_present")
),
"source_truth_diff_required": _runtime_truthy(
candidate.get("source_truth_diff_required")
),
"check_mode_receipt_present": bool(
check_mode
and str(check_mode.get("status") or "") == "success"
),
"apply_idempotency_key_present": bool(
str(controlled_apply.get("apply_idempotency_key") or "")
),
"same_run_apply_chain": bool(
automation_run_id
and candidate
and check_mode
and controlled_apply
and loop_ledger.get("same_run_correlation") is True
),
"router_source_sha_present": bool(router_source_sha),
}
blockers = [
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
]
return {
"schema_version": "awoooi_autonomous_single_writer_runtime_v1",
"status": "runtime_evidence_ready" if not blockers else "in_progress",
"automation_run_id": automation_run_id or None,
"candidate_op_id": str(candidate.get("op_id") or "") or None,
"check_mode_op_id": str(check_mode.get("op_id") or "") or None,
"apply_op_id": str(controlled_apply.get("op_id") or "") or None,
"router_source_sha": router_source_sha or None,
"runtime_evidence_ready": not blockers,
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
}
def _build_independent_post_verifier_runtime_readback(
*,
verifier_latest_rows: Iterable[Mapping[str, Any] | Any],
loop_ledger: Mapping[str, Any],
) -> dict[str, Any]:
rows = [_row_mapping(row) for row in verifier_latest_rows]
automation_run_id = str(loop_ledger.get("automation_run_id") or "")
apply_op_id = str(loop_ledger.get("apply_op_id") or "")
verifier = next(
(
row
for row in rows
if apply_op_id
and str(row.get("apply_op_id") or "") == apply_op_id
),
{},
)
required_count = _int_value(verifier.get("required_postcondition_count"))
passed_count = _int_value(verifier.get("passed_postcondition_count"))
verifier_source_sha = str(
verifier.get("verifier_source_sha") or ""
).lower()
controls = {
"same_run_verifier_receipt": bool(
verifier
and automation_run_id
and str(verifier.get("automation_run_id") or "")
== automation_run_id
),
"verification_result_success": (
verifier.get("verification_result") == "success"
),
"asset_post_verifier_schema": (
verifier.get("verifier_schema_version")
== "awoooi_ansible_asset_post_verifier_v1"
),
"asset_specific_verifier": (
verifier.get("verifier")
== "asset_specific_read_only_host_postconditions"
),
"independent_host_runtime_source": (
verifier.get("independent_source")
== "broker_ssh_host_runtime_readback"
),
"executor_returncode_not_trusted": (
str(verifier.get("executor_returncode_trusted") or "").lower()
== "false"
),
"all_postconditions_passed": _runtime_truthy(
verifier.get("all_postconditions_passed")
),
"postcondition_counts_complete": bool(
required_count > 0 and passed_count == required_count
),
"raw_output_not_stored": (
str(verifier.get("raw_output_stored") or "").lower() == "false"
),
"apply_terminal_verifier_passed": _runtime_truthy(
verifier.get("apply_terminal_verifier_passed")
),
"verifier_source_sha_present": bool(verifier_source_sha),
}
blockers = [
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
]
return {
"schema_version": "awoooi_independent_post_verifier_runtime_v1",
"status": "runtime_evidence_ready" if not blockers else "in_progress",
"automation_run_id": automation_run_id or None,
"apply_op_id": apply_op_id or None,
"verifier_evidence_id": str(verifier.get("id") or "") or None,
"catalog_id": verifier.get("catalog_id"),
"verifier_source_sha": verifier_source_sha or None,
"required_postcondition_count": required_count,
"passed_postcondition_count": passed_count,
"runtime_evidence_ready": not blockers,
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
}
def _build_canonical_learning_runtime_readback(
*,
operation_latest_rows: Iterable[Mapping[str, Any] | Any],
loop_ledger: Mapping[str, Any],
) -> dict[str, Any]:
rows = [_row_mapping(row) for row in operation_latest_rows]
automation_run_id = str(loop_ledger.get("automation_run_id") or "")
apply_op_id = str(loop_ledger.get("apply_op_id") or "")
catalog_id = str(loop_ledger.get("catalog_id") or "")
playbook_path = str(loop_ledger.get("playbook_path") or "")
learning_operation = next(
(
row
for row in rows
if str(row.get("operation_type") or "")
== "ansible_learning_writeback_recorded"
and str(row.get("parent_op_id") or "") == apply_op_id
and str(row.get("automation_run_id") or "")
== automation_run_id
),
{},
)
stage_receipts = {
str(receipt.get("stage_id") or ""): receipt
for receipt in loop_ledger.get("same_run_stage_receipts") or []
if isinstance(receipt, Mapping)
}
km_receipt = stage_receipts.get("km_playbook_writeback") or {}
trust_receipt = stage_receipts.get("playbook_trust") or {}
km_detail = km_receipt.get("detail")
if not isinstance(km_detail, Mapping):
km_detail = {}
trust_detail = trust_receipt.get("detail")
if not isinstance(trust_detail, Mapping):
trust_detail = {}
canonical_playbook_id = str(
trust_detail.get("canonical_playbook_id") or ""
)
km_playbook_id = str(km_detail.get("canonical_playbook_id") or "")
writer_source_sha = str(
trust_detail.get("writer_source_sha") or ""
).lower()
km_writer_source_sha = str(
km_detail.get("writer_source_sha") or ""
).lower()
controls = {
"same_run_learning_operation": bool(learning_operation),
"learning_operation_success": (
learning_operation.get("status") == "success"
),
"learning_parent_apply_matches": bool(
apply_op_id
and str(learning_operation.get("parent_op_id") or "")
== apply_op_id
),
"km_writeback_receipt_v1": (
km_detail.get("schema_version") == "ansible_km_writeback_v1"
),
"km_repository_readback_verified": bool(
km_detail.get("repository_write_acknowledged") is True
and km_detail.get("repository_readback_verified") is True
and km_detail.get("durable_write_acknowledged") is True
),
"km_row_version_present": bool(
str(km_detail.get("knowledge_entry_id") or "")
and str(km_detail.get("km_row_version") or "")
),
"playbook_trust_receipt_v2": (
trust_detail.get("schema_version")
== "ansible_playbook_trust_writeback_v2"
),
"canonical_playbook_identity_resolved": bool(
trust_detail.get("identity_schema_version")
== "ansible_playbook_identity_v1"
and str(trust_detail.get("identity_fingerprint") or "")
and canonical_playbook_id
and trust_detail.get("catalog_id") == catalog_id
and trust_detail.get("playbook_path") == playbook_path
),
"km_playbook_identity_matches": bool(
canonical_playbook_id
and canonical_playbook_id == km_playbook_id
and km_detail.get("catalog_id") == catalog_id
),
"playbook_repository_readback_verified": bool(
trust_detail.get("repository_write_acknowledged") is True
and trust_detail.get("repository_readback_verified") is True
and trust_detail.get("operation_receipt_readback_verified") is True
and trust_detail.get("durable_write_acknowledged") is True
),
"playbook_row_version_present": bool(
_int_value(trust_detail.get("playbook_row_version")) > 0
and str(trust_detail.get("playbook_row_updated_at") or "")
and str(trust_detail.get("playbook_row_fingerprint") or "")
),
"learning_and_trust_acknowledged": bool(
trust_detail.get("learning_recorded") is True
and trust_detail.get("trust_updated") is True
and _int_value(trust_detail.get("trust_observation_count")) > 0
),
"learning_receipts_public_safe": bool(
km_detail.get("raw_log_payload_stored") is False
and km_detail.get("secret_value_stored") is False
and trust_detail.get("raw_log_payload_stored") is False
and trust_detail.get("secret_value_stored") is False
),
"writer_source_sha_same_run": bool(
writer_source_sha
and writer_source_sha == km_writer_source_sha
),
}
blockers = [
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
]
return {
"schema_version": "awoooi_canonical_learning_runtime_v1",
"status": "runtime_evidence_ready" if not blockers else "in_progress",
"automation_run_id": automation_run_id or None,
"apply_op_id": apply_op_id or None,
"learning_operation_id": str(
learning_operation.get("op_id") or ""
)
or None,
"knowledge_entry_id": str(
km_detail.get("knowledge_entry_id") or ""
)
or None,
"canonical_playbook_id": canonical_playbook_id or None,
"catalog_id": catalog_id or None,
"km_row_version": km_detail.get("km_row_version"),
"playbook_row_version": trust_detail.get("playbook_row_version"),
"playbook_row_fingerprint": trust_detail.get(
"playbook_row_fingerprint"
),
"writer_source_sha": writer_source_sha or None,
"runtime_evidence_ready": not blockers,
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
}
def build_runtime_receipt_readback_from_rows(
*,
project_id: str = _DEFAULT_PROJECT_ID,
lookback_hours: int = _DEFAULT_LOOKBACK_HOURS,
db_read_status: str = "ok",
operation_count_rows: Iterable[Mapping[str, Any] | Any] = (),
operation_latest_rows: Iterable[Mapping[str, Any] | Any] = (),
auto_repair_count_rows: Iterable[Mapping[str, Any] | Any] = (),
auto_repair_latest_rows: Iterable[Mapping[str, Any] | Any] = (),
verifier_count_rows: Iterable[Mapping[str, Any] | Any] = (),
verifier_latest_rows: Iterable[Mapping[str, Any] | Any] = (),
km_count_rows: Iterable[Mapping[str, Any] | Any] = (),
km_latest_rows: Iterable[Mapping[str, Any] | Any] = (),
telegram_count_rows: Iterable[Mapping[str, Any] | Any] = (),
telegram_latest_rows: Iterable[Mapping[str, Any] | Any] = (),
mcp_gateway_count_rows: Iterable[Mapping[str, Any] | Any] = (),
legacy_mcp_count_rows: Iterable[Mapping[str, Any] | Any] = (),
service_log_count_rows: Iterable[Mapping[str, Any] | Any] = (),
executor_log_count_rows: Iterable[Mapping[str, Any] | Any] = (),
timeline_count_rows: Iterable[Mapping[str, Any] | Any] = (),
playbook_trust_count_rows: Iterable[Mapping[str, Any] | Any] = (),
alert_operation_count_rows: Iterable[Mapping[str, Any] | Any] = (),
alertmanager_event_count_rows: Iterable[Mapping[str, Any] | Any] = (),
grouped_alert_event_count_rows: Iterable[Mapping[str, Any] | Any] = (),
log_controlled_writeback_consumer: Mapping[str, Any] | None = None,
error_type: str | None = None,
partial_query_failures: Iterable[Mapping[str, Any]] = (),
) -> dict[str, Any]:
"""Build the live executor receipt readback from already-fetched rows."""
operation_count_rows = list(operation_count_rows)
auto_repair_count_rows = list(auto_repair_count_rows)
verifier_count_rows = list(verifier_count_rows)
km_count_rows = list(km_count_rows)
telegram_count_rows = list(telegram_count_rows)
mcp_gateway_count_rows = list(mcp_gateway_count_rows)
legacy_mcp_count_rows = list(legacy_mcp_count_rows)
service_log_count_rows = list(service_log_count_rows)
executor_log_count_rows = list(executor_log_count_rows)
timeline_count_rows = list(timeline_count_rows)
playbook_trust_count_rows = list(playbook_trust_count_rows)
alert_operation_count_rows = list(alert_operation_count_rows)
alertmanager_event_count_rows = list(alertmanager_event_count_rows)
grouped_alert_event_count_rows = list(grouped_alert_event_count_rows)
if not isinstance(log_controlled_writeback_consumer, Mapping):
log_controlled_writeback_consumer = (
_fallback_log_controlled_writeback_consumer_readback()
)
consumer_receipt_fallback_active = _consumer_receipt_fallback_active(
db_read_status=db_read_status,
error_type=error_type,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
if consumer_receipt_fallback_active:
_append_consumer_receipt_fallback_rows(
operation_count_rows=operation_count_rows,
auto_repair_count_rows=auto_repair_count_rows,
verifier_count_rows=verifier_count_rows,
km_count_rows=km_count_rows,
telegram_count_rows=telegram_count_rows,
mcp_gateway_count_rows=mcp_gateway_count_rows,
service_log_count_rows=service_log_count_rows,
executor_log_count_rows=executor_log_count_rows,
timeline_count_rows=timeline_count_rows,
playbook_trust_count_rows=playbook_trust_count_rows,
alert_operation_count_rows=alert_operation_count_rows,
alertmanager_event_count_rows=alertmanager_event_count_rows,
grouped_alert_event_count_rows=grouped_alert_event_count_rows,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
consumer_receipt_fallback = _consumer_receipt_fallback_summary(
active=consumer_receipt_fallback_active,
db_read_status=db_read_status,
error_type=error_type,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
operation_latest = list(operation_latest_rows)
auto_repair_latest = list(auto_repair_latest_rows)
verifier_latest = list(verifier_latest_rows)
km_latest = list(km_latest_rows)
telegram_latest = list(telegram_latest_rows)
operation_summary = _operation_counts(operation_count_rows)
auto_repair_summary = _status_counts(
auto_repair_count_rows,
status_key="result_status",
)
verifier_summary = _status_counts(
verifier_count_rows,
status_key="verification_result",
)
km_summary = _status_counts(km_count_rows, status_key="status")
telegram_summary = _status_counts(telegram_count_rows, status_key="send_status")
mcp_gateway_summary = _status_counts(mcp_gateway_count_rows, status_key="status")
legacy_mcp_summary = _status_counts(legacy_mcp_count_rows, status_key="status")
service_log_summary = _status_counts(service_log_count_rows, status_key="status")
executor_log_summary = _status_counts(executor_log_count_rows, status_key="status")
timeline_summary = _status_counts(timeline_count_rows, status_key="status")
playbook_trust_summary = _status_counts(playbook_trust_count_rows, status_key="status")
alert_operation_summary = _status_counts(
alert_operation_count_rows,
status_key="event_type",
)
alertmanager_event_summary = _status_counts(
alertmanager_event_count_rows,
status_key="stage",
)
grouped_alert_summary = _status_counts(
grouped_alert_event_count_rows,
status_key="status",
)
latest_closure = _latest_flow_closure(
operation_latest_rows=operation_latest,
verifier_latest_rows=verifier_latest,
km_latest_rows=km_latest,
telegram_latest_rows=telegram_latest,
)
capability_lifecycle = _latest_execution_capability_lifecycle(
operation_latest_rows=operation_latest,
operation_summary=operation_summary,
)
latest_failure = _latest_failure_classification(
operation_latest_rows=operation_latest,
verifier_latest_rows=verifier_latest,
latest_flow_closure=latest_closure,
)
retry_package = _controlled_retry_package(latest_failure)
loop_ledger = _autonomous_execution_loop_ledger(
project_id=project_id,
operation_latest_rows=operation_latest,
verifier_latest_rows=verifier_latest,
km_latest_rows=km_latest,
telegram_latest_rows=telegram_latest,
auto_repair_latest_rows=auto_repair_latest,
latest_flow_closure=latest_closure,
latest_failure_classification=latest_failure,
controlled_retry_package=retry_package,
)
single_writer_runtime = _build_autonomous_single_writer_runtime_readback(
operation_latest_rows=operation_latest,
loop_ledger=loop_ledger,
)
independent_post_verifier_runtime = (
_build_independent_post_verifier_runtime_readback(
verifier_latest_rows=verifier_latest,
loop_ledger=loop_ledger,
)
)
canonical_learning_runtime = _build_canonical_learning_runtime_readback(
operation_latest_rows=operation_latest,
loop_ledger=loop_ledger,
)
trace_ledger = _build_trace_ledger(
operation_summary=operation_summary,
auto_repair_summary=auto_repair_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
mcp_gateway_summary=mcp_gateway_summary,
legacy_mcp_summary=legacy_mcp_summary,
service_log_summary=service_log_summary,
executor_log_summary=executor_log_summary,
timeline_summary=timeline_summary,
playbook_trust_summary=playbook_trust_summary,
latest_flow_closure=latest_closure,
loop_ledger=loop_ledger,
)
log_integration_taxonomy = _build_log_integration_taxonomy(
operation_summary=operation_summary,
auto_repair_summary=auto_repair_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
mcp_gateway_summary=mcp_gateway_summary,
legacy_mcp_summary=legacy_mcp_summary,
service_log_summary=service_log_summary,
executor_log_summary=executor_log_summary,
timeline_summary=timeline_summary,
playbook_trust_summary=playbook_trust_summary,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
runtime_receipt_readback_recovery = _build_runtime_receipt_readback_recovery(
db_read_status=db_read_status,
log_integration_taxonomy=log_integration_taxonomy,
error_type=error_type,
partial_query_failures=partial_query_failures,
)
agent_decision_wiring = _build_agent_decision_wiring(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
mcp_gateway_summary=mcp_gateway_summary,
legacy_mcp_summary=legacy_mcp_summary,
service_log_summary=service_log_summary,
timeline_summary=timeline_summary,
playbook_trust_summary=playbook_trust_summary,
log_integration_taxonomy=log_integration_taxonomy,
loop_ledger=loop_ledger,
latest_flow_closure=latest_closure,
)
learning_loop = _build_learning_loop_readback(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
playbook_trust_summary=playbook_trust_summary,
log_integration_taxonomy=log_integration_taxonomy,
agent_decision_wiring=agent_decision_wiring,
latest_flow_closure=latest_closure,
latest_failure_classification=latest_failure,
controlled_retry_package=retry_package,
loop_ledger=loop_ledger,
)
alert_noise_reduction = _build_alert_noise_reduction_readback(
alert_operation_summary=alert_operation_summary,
alertmanager_event_summary=alertmanager_event_summary,
grouped_alert_summary=grouped_alert_summary,
telegram_summary=telegram_summary,
operation_summary=operation_summary,
agent_decision_wiring=agent_decision_wiring,
learning_loop=learning_loop,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
host_sustained_load_automation = (
_build_host_sustained_load_controlled_automation_readback()
)
ui_productization = _build_ui_productization_readback()
multi_product_taxonomy = _build_multi_product_taxonomy_contract(log_integration_taxonomy)
log_controlled_writeback_executor = _load_log_controlled_writeback_executor_readback()
work_item_progress = _build_work_item_progress(
trace_ledger=trace_ledger,
log_integration_taxonomy=log_integration_taxonomy,
log_controlled_writeback_executor=log_controlled_writeback_executor,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
agent_decision_wiring=agent_decision_wiring,
learning_loop=learning_loop,
alert_noise_reduction=alert_noise_reduction,
host_sustained_load_automation=host_sustained_load_automation,
ui_productization=ui_productization,
multi_product_taxonomy=multi_product_taxonomy,
db_read_status=db_read_status,
partial_query_failures=partial_query_failures,
)
apply_summary = operation_summary.get("ansible_apply_executed") or {}
readback = {
"schema_version": _LIVE_READBACK_SCHEMA_VERSION,
"project_id": project_id,
"lookback_hours": max(1, int(lookback_hours or _DEFAULT_LOOKBACK_HOURS)),
"db_read_status": db_read_status,
"partial_query_failures": [dict(item) for item in partial_query_failures],
"consumer_receipt_fallback": consumer_receipt_fallback,
"writes_on_read": False,
"ansible_operations": {
"counts": operation_summary,
"latest": _sanitize_latest_rows(
operation_latest,
allowed_keys=(
"op_id",
"parent_op_id",
"operation_type",
"status",
"actor",
"incident_id",
"catalog_id",
"playbook_path",
"execution_mode",
"source_candidate_op_id",
"check_mode_op_id",
"risk_level",
"controlled_apply_allowed",
"decision_path",
"single_writer_executor",
"candidate_idempotency_key",
"apply_idempotency_key",
"source_truth_diff_required",
"target_selector_present",
"router_source_sha",
"capability_op_id",
"capability_issued_at",
"capability_expires_at",
"capability_terminal_status",
"returncode",
"duration_ms",
"created_at",
),
),
},
"autonomous_single_writer_runtime": single_writer_runtime,
"independent_post_verifier_runtime": independent_post_verifier_runtime,
"canonical_learning_runtime": canonical_learning_runtime,
"auto_repair_execution_receipt": {
**auto_repair_summary,
"latest": _sanitize_latest_rows(
auto_repair_latest,
allowed_keys=(
"id",
"incident_id",
"catalog_id",
"playbook_name",
"result_status",
"triggered_by",
"risk_level",
"execution_time_ms",
"created_at",
),
),
},
"ansible_apply_executed": {
"total": _int_value(apply_summary.get("total")),
"recent": _int_value(apply_summary.get("recent")),
"by_status": apply_summary.get("by_status") or {},
},
"post_apply_verifier": {
**verifier_summary,
"latest": _sanitize_latest_rows(
verifier_latest,
allowed_keys=(
"id",
"incident_id",
"matched_playbook_id",
"verification_result",
"apply_op_id",
"catalog_id",
"playbook_path",
"returncode",
"verifier_schema_version",
"verifier",
"independent_source",
"verifier_source_sha",
"executor_returncode_trusted",
"all_postconditions_passed",
"required_postcondition_count",
"passed_postcondition_count",
"raw_output_stored",
"apply_terminal_verifier_passed",
"apply_status",
"collected_at",
),
),
},
"km_writeback": {
**km_summary,
"latest": _sanitize_latest_rows(
km_latest,
allowed_keys=(
"id",
"title",
"related_incident_id",
"related_playbook_id",
"path_type",
"status",
"created_by",
"created_at",
),
),
},
"telegram_receipt": {
**telegram_summary,
"latest": _sanitize_latest_rows(
telegram_latest,
allowed_keys=(
"message_id",
"run_id",
"message_type",
"send_status",
"provider_message_id",
"incident_id",
"action",
"queued_at",
"sent_at",
),
),
},
"mcp_context": {
"gateway": mcp_gateway_summary,
"legacy": legacy_mcp_summary,
"total": _trace_total(mcp_gateway_summary) + _trace_total(legacy_mcp_summary),
"recent": _trace_recent(mcp_gateway_summary) + _trace_recent(legacy_mcp_summary),
},
"service_log_evidence": service_log_summary,
"executor_log_projection": executor_log_summary,
"timeline_projection": timeline_summary,
"playbook_trust": playbook_trust_summary,
"execution_capability_lifecycle": capability_lifecycle,
"latest_flow_closure": latest_closure,
"latest_failure_classification": latest_failure,
"controlled_retry_package": retry_package,
"autonomous_execution_loop_ledger": loop_ledger,
"trace_ledger": trace_ledger,
"log_integration_taxonomy": log_integration_taxonomy,
"runtime_receipt_readback_recovery": runtime_receipt_readback_recovery,
"log_controlled_writeback_executor": log_controlled_writeback_executor,
"log_controlled_writeback_consumer": dict(log_controlled_writeback_consumer),
"agent_decision_wiring": agent_decision_wiring,
"learning_loop": learning_loop,
"alert_noise_reduction": alert_noise_reduction,
"host_sustained_load_automation": host_sustained_load_automation,
"ui_productization": ui_productization,
"multi_product_taxonomy": multi_product_taxonomy,
"work_item_progress": work_item_progress,
}
if error_type:
readback["error"] = {
"type": error_type,
"message": "runtime receipt DB read failed; see API logs",
}
return readback
def _attach_runtime_receipt_readback(
payload: dict[str, Any],
readback: dict[str, Any],
) -> dict[str, Any]:
payload["runtime_receipt_readback"] = readback
rollups = payload.setdefault("rollups", {})
log_executor = readback.get("log_controlled_writeback_executor")
if not isinstance(log_executor, Mapping):
log_executor = {}
log_executor_rollups = log_executor.get("rollups")
if not isinstance(log_executor_rollups, Mapping):
log_executor_rollups = {}
log_executor_context = log_executor.get("agent_consumption_context")
if not isinstance(log_executor_context, Mapping):
log_executor_context = {}
log_executor_queue = log_executor_context.get("next_action_queue")
if not isinstance(log_executor_queue, list):
log_executor_queue = []
log_executor_current_blocker_queue = log_executor_context.get(
"current_blocker_execution_queue"
)
if not isinstance(log_executor_current_blocker_queue, list):
log_executor_current_blocker_queue = []
log_executor_blockers = log_executor.get("active_blockers")
if not isinstance(log_executor_blockers, list):
log_executor_blockers = []
log_consumer = readback.get("log_controlled_writeback_consumer")
if not isinstance(log_consumer, Mapping):
log_consumer = {}
log_consumer_rollups = log_consumer.get("rollups")
if not isinstance(log_consumer_rollups, Mapping):
log_consumer_rollups = {}
log_consumer_blockers = log_consumer.get("active_blockers")
if not isinstance(log_consumer_blockers, list):
log_consumer_blockers = []
log_consumer_dispatch_ledger_count = _int_value(
log_consumer_rollups.get("dispatch_ledger_row_count")
)
operation_counts = (readback.get("ansible_operations") or {}).get("counts")
if not isinstance(operation_counts, Mapping):
operation_counts = {}
capability_lifecycle = readback.get("execution_capability_lifecycle")
if not isinstance(capability_lifecycle, Mapping):
capability_lifecycle = {}
log_dispatch_summary = (
operation_counts.get(LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE) or {}
)
runtime_recovery = readback.get("runtime_receipt_readback_recovery")
if not isinstance(runtime_recovery, Mapping):
runtime_recovery = {}
rollups.update({
"live_ansible_apply_executed_count": _int_value(
readback.get("ansible_apply_executed", {}).get("total")
),
"live_auto_repair_execution_receipt_count": _int_value(
readback.get("auto_repair_execution_receipt", {}).get("total")
),
"live_post_apply_verifier_count": _int_value(
readback.get("post_apply_verifier", {}).get("total")
),
"live_km_writeback_count": _int_value(
readback.get("km_writeback", {}).get("total")
),
"live_telegram_receipt_count": _int_value(
readback.get("telegram_receipt", {}).get("total")
),
"live_executor_latest_flow_closed_count": (
1
if (readback.get("latest_flow_closure") or {}).get("closed") is True
else 0
),
"live_execution_capability_lifecycle_closed_count": (
1 if capability_lifecycle.get("closed") is True else 0
),
"live_execution_capability_same_run_count": (
1
if capability_lifecycle.get("same_run_correlation") is True
else 0
),
"live_autonomous_execution_loop_closed_count": (
1
if (readback.get("autonomous_execution_loop_ledger") or {}).get("closed") is True
else 0
),
"live_executor_latest_apply_repair_required_count": (
1
if (
(readback.get("latest_failure_classification") or {}).get("classification")
in {
"closed_failed_apply_requires_ai_repair",
"failed_apply_receipt_gap_requires_backfill_then_repair",
}
)
else 0
),
"live_executor_retry_package_ready_count": (
1
if (readback.get("controlled_retry_package") or {}).get("status")
== "ready_for_no_write_check_mode_replay"
else 0
),
"live_mcp_context_count": _int_value(readback.get("mcp_context", {}).get("total")),
"live_service_log_evidence_count": _int_value(
readback.get("service_log_evidence", {}).get("total")
),
"live_executor_log_projection_count": _int_value(
readback.get("executor_log_projection", {}).get("total")
),
"live_timeline_projection_count": _int_value(
readback.get("timeline_projection", {}).get("total")
),
"live_playbook_trust_signal_count": _int_value(
readback.get("playbook_trust", {}).get("total")
),
"live_trace_recorded_stage_count": _int_value(
readback.get("trace_ledger", {}).get("recorded_stage_count")
),
"live_trace_required_missing_count": len(
(readback.get("trace_ledger") or {}).get("missing_required_stage_ids") or []
),
"live_log_source_family_count": _int_value(
((readback.get("log_integration_taxonomy") or {}).get("rollups") or {}).get(
"source_family_count"
)
),
"live_log_active_source_family_count": _int_value(
((readback.get("log_integration_taxonomy") or {}).get("rollups") or {}).get(
"active_source_family_count"
)
),
"live_log_label_dimension_count": _int_value(
((readback.get("log_integration_taxonomy") or {}).get("rollups") or {}).get(
"label_dimension_count"
)
),
"live_log_classified_event_total": _int_value(
((readback.get("log_integration_taxonomy") or {}).get("rollups") or {}).get(
"classified_event_total"
)
),
"live_log_recent_classified_event_total": _int_value(
((readback.get("log_integration_taxonomy") or {}).get("rollups") or {}).get(
"recent_classified_event_total"
)
),
"live_runtime_receipt_recovery_status": runtime_recovery.get("status") or "",
"live_runtime_receipt_safe_next_action_id": runtime_recovery.get(
"safe_next_action_id"
)
or "",
"live_runtime_receipt_safe_next_action_stage": runtime_recovery.get(
"safe_next_action_stage"
)
or "",
"live_log_controlled_writeback_executor_batch_count": _int_value(
log_executor_rollups.get("execution_batch_count")
),
"live_log_controlled_writeback_executor_ready_batch_count": _int_value(
log_executor_rollups.get("ready_execution_batch_count")
),
"live_log_controlled_writeback_executor_ready_count": (
1
if log_executor.get("status") == "controlled_writeback_executor_ready"
else 0
),
"live_log_controlled_writeback_executor_blocker_count": len(
log_executor_blockers
),
"live_log_controlled_writeback_next_action_queue_count": len(
log_executor_queue
),
"live_log_controlled_writeback_current_blocker_queue_count": len(
log_executor_current_blocker_queue
),
"live_log_controlled_writeback_current_blocker_control_path_blocked_count": (
_int_value(
log_executor_rollups.get("current_blocker_control_path_blocked_count")
)
),
"live_log_controlled_writeback_current_blocker_local_recovery_package_count": (
_int_value(
log_executor_rollups.get("current_blocker_local_recovery_package_count")
)
),
"live_log_controlled_writeback_dispatch_count": _int_value(
max(
_int_value(log_dispatch_summary.get("total")),
log_consumer_dispatch_ledger_count,
)
),
"live_log_controlled_writeback_recent_dispatch_count": _int_value(
log_dispatch_summary.get("recent")
),
"live_log_controlled_writeback_consumer_dispatch_ledger_count": (
log_consumer_dispatch_ledger_count
),
"live_log_controlled_writeback_consumer_apply_receipt_count": _int_value(
log_consumer_rollups.get("consumer_apply_receipt_row_count")
),
"live_log_controlled_writeback_consumer_binding_count": _int_value(
log_consumer_rollups.get("consumer_binding_count")
),
"live_log_controlled_writeback_consumer_ready_binding_count": _int_value(
log_consumer_rollups.get("ready_consumer_binding_count")
),
"live_log_controlled_writeback_consumer_ready_target_count": _int_value(
log_consumer_rollups.get("ready_target_count")
),
"live_log_controlled_writeback_consumer_ready_count": (
1
if log_consumer.get("status")
== "controlled_writeback_consumer_readback_ready"
else 0
),
"live_log_controlled_writeback_consumer_blocker_count": len(
log_consumer_blockers
),
"live_log_controlled_writeback_consumer_metadata_only_count": _int_value(
log_consumer_rollups.get("metadata_only_receipt_count")
),
"live_log_controlled_writeback_consumer_verifier_ref_count": _int_value(
log_consumer_rollups.get("post_apply_verifier_ref_count")
),
"live_log_controlled_writeback_target_context_receipt_write_count": (
_int_value(log_consumer_rollups.get("target_context_receipt_write_count"))
),
"live_log_controlled_writeback_runtime_target_write_count": (
1
if log_consumer_rollups.get("runtime_target_write_performed") is True
else 0
),
"live_log_controlled_writeback_km_consumer_binding_count": _int_value(
log_consumer_rollups.get("km_consumer_binding_count")
),
"live_log_controlled_writeback_rag_consumer_binding_count": _int_value(
log_consumer_rollups.get("rag_consumer_binding_count")
),
"live_log_controlled_writeback_playbook_consumer_binding_count": _int_value(
log_consumer_rollups.get("playbook_consumer_binding_count")
),
"live_log_controlled_writeback_mcp_consumer_binding_count": _int_value(
log_consumer_rollups.get("mcp_consumer_binding_count")
),
"live_log_controlled_writeback_verifier_consumer_binding_count": _int_value(
log_consumer_rollups.get("verifier_consumer_binding_count")
),
"live_log_controlled_writeback_ai_agent_consumer_binding_count": _int_value(
log_consumer_rollups.get("ai_agent_consumer_binding_count")
),
"live_agent_decision_wiring_stage_count": _int_value(
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
"stage_count"
)
),
"live_agent_decision_wiring_required_present_count": _int_value(
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
"required_stage_present_count"
)
),
"live_agent_decision_wiring_required_missing_count": _int_value(
((readback.get("agent_decision_wiring") or {}).get("rollups") or {}).get(
"required_stage_missing_count"
)
),
"live_agent_decision_wiring_complete_count": (
1
if (readback.get("agent_decision_wiring") or {}).get("status") == "completed"
else 0
),
"live_learning_loop_stage_count": _int_value(
((readback.get("learning_loop") or {}).get("rollups") or {}).get(
"stage_count"
)
),
"live_learning_loop_required_present_count": _int_value(
((readback.get("learning_loop") or {}).get("rollups") or {}).get(
"required_stage_present_count"
)
),
"live_learning_loop_required_missing_count": _int_value(
((readback.get("learning_loop") or {}).get("rollups") or {}).get(
"required_stage_missing_count"
)
),
"live_learning_loop_complete_count": (
1
if (readback.get("learning_loop") or {}).get("status") == "completed"
else 0
),
"live_learning_loop_similar_case_source_count": _int_value(
((readback.get("learning_loop") or {}).get("rollups") or {}).get(
"similar_case_source_total"
)
),
"live_alert_noise_stage_count": _int_value(
((readback.get("alert_noise_reduction") or {}).get("rollups") or {}).get(
"stage_count"
)
),
"live_alert_noise_required_present_count": _int_value(
((readback.get("alert_noise_reduction") or {}).get("rollups") or {}).get(
"required_stage_present_count"
)
),
"live_alert_noise_required_missing_count": _int_value(
((readback.get("alert_noise_reduction") or {}).get("rollups") or {}).get(
"required_stage_missing_count"
)
),
"live_alert_noise_complete_count": (
1
if (readback.get("alert_noise_reduction") or {}).get("status") == "completed"
else 0
),
"live_alert_noise_suppressed_count": _int_value(
((readback.get("alert_noise_reduction") or {}).get("rollups") or {}).get(
"suppressed_alert_total"
)
),
"live_alert_noise_controlled_route_count": _int_value(
((readback.get("alert_noise_reduction") or {}).get("rollups") or {}).get(
"controlled_route_total"
)
),
"live_host_sustained_load_action_class_count": _int_value(
((readback.get("host_sustained_load_automation") or {}).get("rollups") or {}).get(
"action_class_count"
)
),
"live_host_sustained_load_controlled_apply_class_count": _int_value(
((readback.get("host_sustained_load_automation") or {}).get("rollups") or {}).get(
"controlled_apply_class_count"
)
),
"live_host_sustained_load_ready_asset_count": _int_value(
((readback.get("host_sustained_load_automation") or {}).get("rollups") or {}).get(
"ready_asset_count"
)
),
"live_host_sustained_load_complete_count": (
1
if (readback.get("host_sustained_load_automation") or {}).get("status")
== "completed"
else 0
),
"live_ui_productization_surface_count": _int_value(
((readback.get("ui_productization") or {}).get("rollups") or {}).get(
"surface_count"
)
),
"live_ui_productization_required_present_count": _int_value(
((readback.get("ui_productization") or {}).get("rollups") or {}).get(
"required_surface_present_count"
)
),
"live_ui_productization_required_missing_count": _int_value(
((readback.get("ui_productization") or {}).get("rollups") or {}).get(
"required_surface_missing_count"
)
),
"live_ui_productization_complete_count": (
1
if (readback.get("ui_productization") or {}).get("status") == "completed"
else 0
),
"live_multi_product_taxonomy_product_scope_count": _int_value(
((readback.get("multi_product_taxonomy") or {}).get("rollups") or {}).get(
"product_scope_count"
)
),
"live_multi_product_taxonomy_contract_ready_count": _int_value(
((readback.get("multi_product_taxonomy") or {}).get("rollups") or {}).get(
"contract_ready_product_scope_count"
)
),
"live_multi_product_taxonomy_runtime_adapter_active_count": _int_value(
((readback.get("multi_product_taxonomy") or {}).get("rollups") or {}).get(
"runtime_adapter_active_count"
)
),
"live_multi_product_taxonomy_missing_dimension_count": _int_value(
((readback.get("multi_product_taxonomy") or {}).get("rollups") or {}).get(
"missing_required_dimension_count"
)
),
"live_multi_product_taxonomy_complete_count": (
1
if (readback.get("multi_product_taxonomy") or {}).get("status") == "completed"
else 0
),
"live_work_item_count": _int_value(
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
"work_item_count"
)
),
"live_work_item_completed_count": _int_value(
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
"completed_count"
)
),
"live_work_item_in_progress_count": _int_value(
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
"in_progress_count"
)
),
"live_work_item_pending_count": _int_value(
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
"pending_count"
)
),
"live_work_item_blocked_count": _int_value(
((readback.get("work_item_progress") or {}).get("rollups") or {}).get(
"blocked_count"
)
),
})
loop_ledger = readback.get("autonomous_execution_loop_ledger")
if not isinstance(loop_ledger, Mapping):
loop_ledger = {}
latest_flow = readback.get("latest_flow_closure")
if not isinstance(latest_flow, Mapping):
latest_flow = {}
loop_stages = loop_ledger.get("stages")
if not isinstance(loop_stages, list):
loop_stages = []
trace_ledger = readback.get("trace_ledger")
if not isinstance(trace_ledger, Mapping):
trace_ledger = {}
trace_stages = trace_ledger.get("stages")
if not isinstance(trace_stages, list):
trace_stages = []
loop_stage_by_id = {
str(stage.get("stage_id") or ""): stage
for stage in loop_stages
if isinstance(stage, Mapping) and stage.get("stage_id")
}
trace_stage_by_id = {
str(stage.get("stage_id") or ""): stage
for stage in trace_stages
if isinstance(stage, Mapping) and stage.get("stage_id")
}
same_run_stage_receipts = loop_ledger.get("same_run_stage_receipts")
if not isinstance(same_run_stage_receipts, list):
same_run_stage_receipts = []
same_run_stage_receipt_by_id = {
str(receipt.get("stage_id") or ""): receipt
for receipt in same_run_stage_receipts
if isinstance(receipt, Mapping) and receipt.get("stage_id")
}
latest_flow_closed = latest_flow.get("closed") is True
latest_loop_closed = loop_ledger.get("closed") is True
automation_run_id = str(loop_ledger.get("automation_run_id") or "")
latest_flow_run_id = str(latest_flow.get("automation_run_id") or "")
strict_stage_contracts: list[dict[str, Any]] = []
for stage_id in AI_AUTOMATION_REQUIRED_LOOP_STAGES:
if stage_id in AI_AUTOMATION_EXECUTION_CORRELATED_STAGES:
stage = loop_stage_by_id.get(stage_id, {})
evidence_present = stage.get("present") is True
same_run_proven = stage.get("run_id_matches_expected") is True
evidence_source = "same_run_execution_loop_ledger"
else:
same_run_receipt = same_run_stage_receipt_by_id.get(stage_id, {})
if same_run_receipt:
stage = same_run_receipt
evidence_present = True
same_run_proven = bool(
automation_run_id
and str(same_run_receipt.get("automation_run_id") or "")
== automation_run_id
and same_run_receipt.get("durable_receipt") is True
)
evidence_source = "same_run_runtime_stage_receipt"
else:
stage = trace_stage_by_id.get(stage_id, {})
evidence_present = stage.get("recorded") is True
# Aggregate trace counts prove that a source exists, not that it
# belongs to the latest AutomationRun. Incident closure is the
# only auxiliary stage currently carrying the same run identity.
same_run_proven = bool(
stage_id == "incident_closure"
and evidence_present
and automation_run_id
and latest_flow_run_id == automation_run_id
)
evidence_source = "aggregate_trace_ledger"
completion_eligible = evidence_present and same_run_proven
strict_stage_contracts.append(
{
"stage_id": stage_id,
"evidence_present": evidence_present,
"same_run_correlation_proven": same_run_proven,
"completion_eligible": completion_eligible,
"evidence_source": evidence_source,
"evidence_ref": (
stage.get("evidence_ref")
if evidence_source == "same_run_runtime_stage_receipt"
else None
),
}
)
required_stage_count = len(strict_stage_contracts)
present_stage_count = sum(
1 for stage in strict_stage_contracts if stage["completion_eligible"] is True
)
missing_stage_ids = [
str(stage["stage_id"])
for stage in strict_stage_contracts
if stage["completion_eligible"] is not True
]
uncorrelated_stage_ids = [
str(stage["stage_id"])
for stage in strict_stage_contracts
if stage["evidence_present"] is True
and stage["same_run_correlation_proven"] is not True
]
execution_loop_closed = latest_flow_closed and latest_loop_closed
full_trace_same_run_correlation_proven = not missing_stage_ids
strict_runtime_closed = bool(
execution_loop_closed and full_trace_same_run_correlation_proven
)
runtime_completion_percent = round(
(present_stage_count / required_stage_count) * 100
)
if not strict_runtime_closed:
runtime_completion_percent = min(runtime_completion_percent, 99)
program_status = payload.setdefault("program_status", {})
program_status.update(
{
"status": (
"latest_correlated_runtime_loop_closed"
if strict_runtime_closed
else (
"execution_loop_closed_governance_trace_open"
if execution_loop_closed
else "runtime_closure_in_progress"
)
),
"implementation_completion_percent": runtime_completion_percent,
"completion_basis": "canonical_same_run_governance_and_execution_stage_receipts",
"latest_flow_closed": latest_flow_closed,
"latest_loop_closed": latest_loop_closed,
"execution_loop_closed": execution_loop_closed,
"same_run_correlation": (
loop_ledger.get("same_run_correlation") is True
),
"full_trace_same_run_correlation_proven": (
full_trace_same_run_correlation_proven
),
"automation_run_id": loop_ledger.get("automation_run_id"),
"run_id_mismatch_stage_ids": list(
loop_ledger.get("run_id_mismatch_stage_ids") or []
),
"uncorrelated_stage_ids": uncorrelated_stage_ids,
"can_claim_execution_loop_complete": execution_loop_closed,
"can_claim_autonomous_loop_complete": strict_runtime_closed,
"required_runtime_stage_count": required_stage_count,
"present_runtime_stage_count": present_stage_count,
"missing_runtime_stage_ids": missing_stage_ids,
"metadata_only_counts_as_completion": False,
"historical_aggregate_fallback_allowed": False,
"status_note": (
"只有最新關聯 run 的完整 governance、evidence、execution 與 learning "
"receipts 全部存在且 flow/loop closed"
"才可宣稱 autonomous runtime closuresource、test、UI、CD、metadata"
" 與歷史 aggregate 只算 supporting evidence。"
),
}
)
payload["strict_runtime_completion"] = {
"schema_version": "ai_agent_strict_runtime_completion_v2",
"runtime_contract_schema_version": (
AI_AUTOMATION_RUNTIME_CONTRACT_SCHEMA_VERSION
),
"work_item_id": "AIA-P0-001",
"completion_percent": runtime_completion_percent,
"required_stage_count": required_stage_count,
"present_stage_count": present_stage_count,
"missing_stage_ids": missing_stage_ids,
"latest_flow_closed": latest_flow_closed,
"latest_loop_closed": latest_loop_closed,
"execution_loop_closed": execution_loop_closed,
"same_run_correlation": loop_ledger.get("same_run_correlation") is True,
"full_trace_same_run_correlation_proven": (
full_trace_same_run_correlation_proven
),
"automation_run_id": loop_ledger.get("automation_run_id"),
"run_id_mismatch_stage_ids": list(
loop_ledger.get("run_id_mismatch_stage_ids") or []
),
"uncorrelated_stage_ids": uncorrelated_stage_ids,
"stage_contracts": strict_stage_contracts,
"closed": strict_runtime_closed,
"same_run_correlation_required": True,
"metadata_only_counts_as_completion": False,
"historical_aggregate_fallback_allowed": False,
}
rollups["strict_runtime_required_stage_count"] = required_stage_count
rollups["strict_runtime_present_stage_count"] = present_stage_count
rollups["strict_runtime_missing_stage_count"] = len(missing_stage_ids)
rollups["strict_runtime_completion_percent"] = runtime_completion_percent
rollups["strict_runtime_closed_count"] = 1 if strict_runtime_closed else 0
return payload
def build_ai_agent_autonomous_runtime_control() -> dict[str, Any]:
"""Build the current AI Agent autonomy control-plane readback."""
allowed_risks = _allowed_risk_levels()
report_cadences = [
{
"cadence": "daily",
"display_name": "日報",
"schedule": f"每日 {DAILY_REPORT_HOUR_TAIPEI:02d}:00 台北時間",
"worker": "report_generation_service.run_daily_report_loop",
"telegram_gateway_delivery_enabled": True,
"direct_bot_api_allowed": False,
"receipt_source": "daily_report_sent log + Telegram Gateway result",
},
{
"cadence": "weekly",
"display_name": "週報",
"schedule": (
f"每週五 {WEEKLY_REPORT_HOUR_TAIPEI:02d}:00 台北時間"
if WEEKLY_REPORT_WEEKDAY_TAIPEI == 4
else f"每週 weekday={WEEKLY_REPORT_WEEKDAY_TAIPEI} {WEEKLY_REPORT_HOUR_TAIPEI:02d}:00 台北時間"
),
"worker": "report_generation_service.run_weekly_report_loop",
"telegram_gateway_delivery_enabled": True,
"direct_bot_api_allowed": False,
"receipt_source": "weekly_report_sent log + Telegram Gateway result",
},
{
"cadence": "monthly",
"display_name": "月報",
"schedule": f"每月 {MONTHLY_REPORT_DAY_TAIPEI}{MONTHLY_REPORT_HOUR_TAIPEI:02d}:00 台北時間",
"worker": "report_generation_service.run_monthly_report_loop",
"telegram_gateway_delivery_enabled": True,
"direct_bot_api_allowed": False,
"receipt_source": "monthly_report_sent log + Telegram Gateway result",
},
]
executor_receipts = [
{
"operation_type": "ansible_candidate_matched",
"owner_agent": "Hermes",
"purpose": "把修復候選寫入 executor 可認領佇列",
"writes_runtime_state": False,
},
{
"operation_type": "ansible_check_mode_executed",
"owner_agent": "AwoooP Ansible check-mode worker",
"purpose": "執行 ansible-playbook --check --diff 並留下乾跑收據",
"writes_runtime_state": False,
},
{
"operation_type": "ansible_apply_executed",
"owner_agent": "AwoooP controlled apply worker",
"purpose": "check-mode 通過後,對 allowlisted low / medium / high PlayBook 受控 apply",
"writes_runtime_state": True,
},
{
"operation_type": "incident_evidence.post_execution_state",
"owner_agent": "post_apply_verifier",
"purpose": "apply 後寫入 verifier 結果與 post-execution evidence",
"writes_runtime_state": True,
},
{
"operation_type": "knowledge_entries",
"owner_agent": "Hermes",
"purpose": "把已驗證執行沉澱成 KM / PlayBook trust 候選",
"writes_runtime_state": True,
},
{
"operation_type": LOG_CONTROLLED_WRITEBACK_DISPATCH_OPERATION_TYPE,
"owner_agent": "ai_agent_metadata_writeback_executor",
"purpose": "把 LOG feedback batch 寫入 metadata-only controlled dispatch ledger",
"writes_runtime_state": True,
},
]
hard_blockers = [
"secret_token_private_key_cookie_session_auth_header_cleartext",
"drop_truncate_restore_prune_destructive_database_operation",
"reboot_node_drain_irreversible_firewall_or_host_lockout",
"credentialed_exploit_or_external_active_scan",
"new_paid_provider_cost_ceiling_or_provider_switch_without_replay_shadow_canary",
"force_push_delete_repo_refs_or_visibility_change",
"critical_or_break_glass_route_without_explicit_break_glass_contract",
]
legacy_overrides = [
{
"legacy_area": "report_status_board_no_live_send",
"current_effect": "overridden",
"new_behavior": "日報 / 週報 / 月報透過 Telegram Gateway 排程派送",
},
{
"legacy_area": "report_live_delivery_owner_review_required",
"current_effect": "overridden",
"new_behavior": "報告派送走低/中/高風險自動化政策critical 才 break-glass",
},
{
"legacy_area": "high_risk_owner_review_queue",
"current_effect": "overridden_for_high_non_critical",
"new_behavior": "high 風險允許 controlled applycritical / hard blocker 仍不自動",
},
{
"legacy_area": "telegram_no_send_preview_only",
"current_effect": "overridden",
"new_behavior": "用 Telegram Gateway 實送報告與 actionable receipt不直接暴露 Bot API",
},
]
control_plane_integration = _control_plane_integration()
payload = {
"schema_version": _SCHEMA_VERSION,
"generated_at": datetime.now(UTC).isoformat(),
"program_status": {
"current_task_id": "P2-416-D1N",
"status": "current_directive_control_plane_active",
"runtime_authority": _RUNTIME_AUTHORITY,
"deploy_readback_marker": _DEPLOY_READBACK_MARKER,
"deploy_attempt_note": _DEPLOY_ATTEMPT_NOTE,
"legacy_no_send_no_live_rules_overridden": True,
"implementation_completion_percent": 0,
"status_note": (
"目前有效規則low / medium / high 風險由 AI Agent 在 allowlist、"
"Ansible check-mode、verifier、rollback、KM 與 Telegram receipt 下受控自動處理。"
),
},
"current_policy": {
"low_risk_controlled_apply_allowed": "low" in allowed_risks,
"medium_risk_controlled_apply_allowed": "medium" in allowed_risks,
"high_risk_controlled_apply_allowed": "high" in allowed_risks,
"critical_break_glass_required": True,
"owner_review_required_for_low_medium_high": False,
"direct_bot_api_allowed": False,
"telegram_gateway_required": True,
"post_apply_verifier_required": True,
"km_learning_writeback_required": True,
},
"runtime_switches": {
"ansible_check_mode_worker_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CHECK_MODE_WORKER),
"ansible_controlled_apply_enabled": bool(settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY),
"ansible_controlled_apply_allowed_risk_levels": allowed_risks,
"ansible_check_mode_interval_seconds": settings.AWOOOP_ANSIBLE_CHECK_MODE_INTERVAL_SECONDS,
"ansible_check_mode_batch_limit": settings.AWOOOP_ANSIBLE_CHECK_MODE_BATCH_LIMIT,
"ansible_check_mode_timeout_seconds": settings.AWOOOP_ANSIBLE_CHECK_MODE_TIMEOUT_SECONDS,
"ansible_controlled_apply_timeout_seconds": settings.AWOOOP_ANSIBLE_CONTROLLED_APPLY_TIMEOUT_SECONDS,
},
"agent_roles": [
{
"agent_id": "openclaw",
"role": "仲裁 / hard blocker / replay-shadow-canary gate",
"current_job": "只阻擋真正 critical 與 hard blocker不再用身份保護舊架構",
},
{
"agent_id": "hermes",
"role": "報告 / Telegram digest / KM 與 PlayBook trust writeback",
"current_job": "日週月報、收據摘要與 verifier 後學習沉澱",
},
{
"agent_id": "nemotron",
"role": "市場技術雷達 / no-write replay / challenger scorecard",
"current_job": "用市場與回放數據挑戰 OpenClaw / provider / Agent 組合",
},
{
"agent_id": "awooop_ansible_worker",
"role": "executor",
"current_job": "candidate → check-mode → controlled apply → verifier → KM",
},
{
"agent_id": "telegram_ops",
"role": "Telegram Gateway receipt",
"current_job": "群組報告、actionable receipt、失敗告警不展示敏感值或未脫敏資料",
},
],
"report_delivery": {
"status": "telegram_gateway_delivery_enabled",
"cadences": report_cadences,
},
"controlled_executor": {
"status": "check_mode_then_apply_enabled"
if settings.ENABLE_AWOOOP_ANSIBLE_CONTROLLED_APPLY
else "check_mode_only_by_config",
"operation_receipts": executor_receipts,
"required_flow": [
"allowlisted_candidate",
"ansible_check_mode_success",
"controlled_apply",
"post_apply_verifier",
"auto_repair_execution_receipt",
"km_learning_writeback",
"telegram_receipt_or_alert",
],
},
"control_plane_integration": control_plane_integration,
"legacy_policy_overrides": legacy_overrides,
"hard_blockers": hard_blockers,
"visibility_contract": {
"frontend_displays_runtime_truth": True,
"work_window_transcript_display_allowed": False,
"prompt_body_display_allowed": False,
"internal_reasoning_display_allowed": False,
"sensitive_value_display_allowed": False,
"telegram_unredacted_payload_display_allowed": False,
"lan_topology_redaction_required": True,
},
"rollups": {
"automated_risk_tier_count": sum(1 for risk in ("low", "medium", "high") if risk in allowed_risks),
"hard_blocker_count": len(hard_blockers),
"report_cadence_enabled_count": len(report_cadences),
"telegram_gateway_delivery_enabled_count": sum(
1 for item in report_cadences if item["telegram_gateway_delivery_enabled"]
),
"direct_bot_api_allowed_count": 0,
"controlled_executor_operation_receipt_count": len(executor_receipts),
"runtime_write_receipt_type_count": sum(
1 for item in executor_receipts if item["writes_runtime_state"]
),
"legacy_policy_overridden_count": len(legacy_overrides),
"mcp_sensor_count": control_plane_integration["rollups"]["mcp_sensor_count"],
"rag_context_query_count": control_plane_integration["rollups"]["rag_context_query_count"],
"playbook_decision_class_count": control_plane_integration["rollups"]["playbook_decision_class_count"],
"deploy_control_classifier_example_count": control_plane_integration["rollups"]["classifier_example_count"],
},
}
_attach_runtime_receipt_readback(
payload,
build_runtime_receipt_readback_from_rows(
project_id=_DEFAULT_PROJECT_ID,
db_read_status="not_queried",
),
)
_validate_payload(payload)
return payload
async def load_ai_agent_autonomous_runtime_receipt_readback(
*,
project_id: str = _DEFAULT_PROJECT_ID,
lookback_hours: int = _DEFAULT_LOOKBACK_HOURS,
limit: int = 20,
) -> dict[str, Any]:
"""Read live executor receipts with short in-process DB pool protection."""
normalized_lookback_hours = max(1, int(lookback_hours or _DEFAULT_LOOKBACK_HOURS))
normalized_limit = max(1, int(limit or 20))
cache_key = _runtime_receipt_readback_cache_key(
project_id=project_id,
lookback_hours=normalized_lookback_hours,
limit=normalized_limit,
)
cached = _runtime_receipt_readback_cache_get(cache_key)
if cached is not None:
return cached
lock = _get_runtime_receipt_readback_lock()
lock_acquired = False
try:
await asyncio.wait_for(
lock.acquire(),
timeout=_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS,
)
lock_acquired = True
cached = _runtime_receipt_readback_cache_get(cache_key)
if cached is not None:
return cached
readback = await _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
project_id=project_id,
lookback_hours=normalized_lookback_hours,
limit=normalized_limit,
)
if not _runtime_receipt_readback_is_cacheable(readback):
error = readback.get("error")
live_error_type = (
str(error.get("type") or "")
if isinstance(error, Mapping)
else None
)
stale_readback = _runtime_receipt_readback_cache_get_stale(
cache_key,
fallback_reason="live_runtime_receipt_readback_degraded",
live_db_read_status=str(readback.get("db_read_status") or "unavailable"),
live_error_type=live_error_type,
)
if stale_readback is not None:
return stale_readback
_runtime_receipt_readback_cache_store(cache_key, readback)
return readback
except TimeoutError:
logger.warning(
"ai_agent_autonomous_runtime_receipt_readback_lock_timeout",
project_id=project_id,
timeout_seconds=_RUNTIME_RECEIPT_LOCK_TIMEOUT_SECONDS,
)
stale_readback = _runtime_receipt_readback_cache_get_stale(
cache_key,
fallback_reason="runtime_receipt_readback_lock_timeout",
live_db_read_status="unavailable",
live_error_type="RuntimeReceiptReadbackLockTimeout",
)
if stale_readback is not None:
return stale_readback
return build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=normalized_lookback_hours,
db_read_status="unavailable",
error_type="RuntimeReceiptReadbackLockTimeout",
)
finally:
if lock_acquired:
lock.release()
async def _load_ai_agent_autonomous_runtime_receipt_readback_uncached(
*,
project_id: str = _DEFAULT_PROJECT_ID,
lookback_hours: int = _DEFAULT_LOOKBACK_HOURS,
limit: int = 20,
) -> dict[str, Any]:
"""Read live executor receipts without sending messages or mutating runtime state."""
params = {
"project_id": project_id,
"lookback_hours": max(1, int(lookback_hours or _DEFAULT_LOOKBACK_HOURS)),
"limit": max(1, int(limit or 20)),
}
log_controlled_writeback_consumer: dict[str, Any] | None = None
partial_query_failures: list[dict[str, str]] = []
db_context = get_db_context(project_id)
db_context_entered = False
db_context_stage = "db_context_enter"
try:
db = await asyncio.wait_for(
db_context.__aenter__(),
timeout=_RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS,
)
db_context_entered = True
db_context_stage = "db_queries"
try:
query_deadline = time.monotonic() + min(
_RUNTIME_RECEIPT_QUERY_BUDGET_SECONDS,
max(0.5, _LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS - 0.75),
)
async def _set_statement_timeout() -> None:
await asyncio.wait_for(
db.execute(
text(
"SET LOCAL statement_timeout = "
f"'{int(_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS)}ms'"
)
),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
await _set_statement_timeout()
def _query_budget_exhausted(query_name: str) -> bool:
if time.monotonic() < query_deadline:
return False
partial_query_failures.append({
"query_name": query_name,
"error_type": "RuntimeReceiptQueryBudgetExceeded",
})
logger.warning(
"ai_agent_autonomous_runtime_trace_query_budget_exhausted",
project_id=project_id,
query_name=query_name,
)
return True
async def _safe_rows(
query_name: str,
sql: str,
fallback_sql: str | None = None,
) -> list[Mapping[str, Any]]:
if _query_budget_exhausted(query_name):
return []
try:
result = await asyncio.wait_for(
db.execute(text(sql), params),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
return result.mappings().all()
except Exception as exc: # pragma: no cover - depends on live schema drift
partial_query_failures.append({
"query_name": query_name,
"error_type": type(exc).__name__,
})
logger.warning(
"ai_agent_autonomous_runtime_trace_read_failed",
project_id=project_id,
query_name=query_name,
error_type=type(exc).__name__,
)
rollback = getattr(db, "rollback", None)
if callable(rollback):
try:
await rollback()
await _set_statement_timeout()
except Exception as reset_exc: # pragma: no cover - live DB state
logger.warning(
"ai_agent_autonomous_runtime_trace_reset_failed",
project_id=project_id,
query_name=query_name,
error_type=type(reset_exc).__name__,
)
if fallback_sql:
if _query_budget_exhausted(f"{query_name}_fallback"):
return []
try:
result = await asyncio.wait_for(
db.execute(text(fallback_sql), params),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
return result.mappings().all()
except Exception as fallback_exc: # pragma: no cover - live schema drift
partial_query_failures.append({
"query_name": f"{query_name}_fallback",
"error_type": type(fallback_exc).__name__,
})
logger.warning(
"ai_agent_autonomous_runtime_trace_aux_fallback_failed",
project_id=project_id,
query_name=query_name,
error_type=type(fallback_exc).__name__,
)
rollback = getattr(db, "rollback", None)
if callable(rollback):
try:
await rollback()
await _set_statement_timeout()
except Exception as reset_exc: # pragma: no cover
logger.warning(
"ai_agent_autonomous_runtime_trace_reset_failed",
project_id=project_id,
query_name=f"{query_name}_fallback",
error_type=type(reset_exc).__name__,
)
return []
operation_counts = await _safe_rows(
"operation_counts",
_RUNTIME_OPERATION_COUNTS_SQL,
)
operation_latest = await _safe_rows(
"operation_latest",
_RUNTIME_OPERATION_LATEST_SQL,
)
for chain_pass in range(2):
missing_chain_refs = _missing_runtime_operation_chain_ref_ids(
operation_latest
)
if not missing_chain_refs:
break
padded_refs = [*missing_chain_refs[:3], "", ""][:3]
params.update(
{
"operation_chain_ref_1": padded_refs[0],
"operation_chain_ref_2": padded_refs[1],
"operation_chain_ref_3": padded_refs[2],
}
)
operation_chain_rows = await _safe_rows(
f"operation_chain_{chain_pass + 1}",
_RUNTIME_OPERATION_CHAIN_SQL,
)
if not operation_chain_rows:
break
operation_latest = _merge_runtime_operation_rows(
operation_latest,
operation_chain_rows,
)
auto_repair_counts = await _safe_rows(
"auto_repair_counts",
_RUNTIME_AUTO_REPAIR_COUNTS_SQL,
)
auto_repair_latest = await _safe_rows(
"auto_repair_latest",
_RUNTIME_AUTO_REPAIR_LATEST_SQL,
)
verifier_counts = await _safe_rows(
"verifier_counts",
_RUNTIME_VERIFIER_COUNTS_SQL,
)
verifier_latest = await _safe_rows(
"verifier_latest",
_RUNTIME_VERIFIER_LATEST_SQL,
)
km_counts = await _safe_rows(
"km_counts",
_RUNTIME_KM_COUNTS_SQL,
)
km_latest = await _safe_rows(
"km_latest",
_RUNTIME_KM_LATEST_SQL,
)
telegram_counts = await _safe_rows(
"telegram_counts",
_RUNTIME_TELEGRAM_COUNTS_SQL,
)
telegram_latest = await _safe_rows(
"telegram_latest",
_RUNTIME_TELEGRAM_LATEST_SQL,
)
mcp_gateway_counts = await _safe_rows(
"mcp_gateway_counts",
_RUNTIME_MCP_GATEWAY_COUNTS_SQL,
)
legacy_mcp_counts = await _safe_rows(
"legacy_mcp_counts",
_RUNTIME_LEGACY_MCP_COUNTS_SQL,
)
service_log_counts = await _safe_rows(
"service_log_counts",
_RUNTIME_SERVICE_LOG_COUNTS_SQL,
)
executor_log_counts = await _safe_rows(
"executor_log_counts",
_RUNTIME_EXECUTOR_LOG_COUNTS_SQL,
)
timeline_counts = await _safe_rows(
"timeline_counts",
_RUNTIME_TIMELINE_COUNTS_SQL,
_RUNTIME_TIMELINE_COUNTS_FALLBACK_SQL,
)
playbook_trust_counts = await _safe_rows(
"playbook_trust_counts",
_RUNTIME_PLAYBOOK_TRUST_COUNTS_SQL,
_RUNTIME_PLAYBOOK_TRUST_COUNTS_FALLBACK_SQL,
)
alert_operation_counts = await _safe_rows(
"alert_operation_counts",
_RUNTIME_ALERT_OPERATION_COUNTS_SQL,
)
alertmanager_event_counts = await _safe_rows(
"alertmanager_event_counts",
_RUNTIME_ALERTMANAGER_EVENT_COUNTS_SQL,
)
grouped_alert_event_counts = await _safe_rows(
"grouped_alert_event_counts",
_RUNTIME_GROUPED_ALERT_EVENT_COUNTS_SQL,
)
finally:
if db_context_entered:
db_context_stage = "db_context_exit"
try:
await asyncio.wait_for(
db_context.__aexit__(None, None, None),
timeout=_RUNTIME_RECEIPT_DB_CONTEXT_EXIT_TIMEOUT_SECONDS,
)
except Exception as exit_exc: # pragma: no cover - live DB state
partial_query_failures.append({
"query_name": "db_context_exit",
"error_type": type(exit_exc).__name__,
})
logger.warning(
"ai_agent_autonomous_runtime_trace_db_context_exit_failed",
project_id=project_id,
error_type=type(exit_exc).__name__,
)
except Exception as exc:
error_type = (
"RuntimeReceiptDbContextTimeout"
if isinstance(exc, TimeoutError) and db_context_stage == "db_context_enter"
else type(exc).__name__
)
direct_core_rows = (
await _load_core_runtime_receipt_rows_direct(
project_id=project_id,
lookback_hours=params["lookback_hours"],
)
if error_type == "RuntimeReceiptDbContextTimeout"
else None
)
if direct_core_rows is not None:
log_controlled_writeback_consumer = (
await _load_log_controlled_writeback_consumer_readback(
project_id=project_id,
)
)
return build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=params["lookback_hours"],
db_read_status="ok",
operation_count_rows=direct_core_rows.get("operation_counts", []),
operation_latest_rows=direct_core_rows.get("operation_latest", []),
auto_repair_count_rows=direct_core_rows.get("auto_repair_counts", []),
auto_repair_latest_rows=direct_core_rows.get("auto_repair_latest", []),
verifier_count_rows=direct_core_rows.get("verifier_counts", []),
verifier_latest_rows=direct_core_rows.get("verifier_latest", []),
km_count_rows=direct_core_rows.get("km_counts", []),
km_latest_rows=direct_core_rows.get("km_latest", []),
telegram_count_rows=direct_core_rows.get("telegram_counts", []),
telegram_latest_rows=direct_core_rows.get("telegram_latest", []),
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
logger.warning(
"ai_agent_autonomous_runtime_receipt_readback_failed",
project_id=project_id,
error_type=error_type,
db_context_stage=db_context_stage,
)
log_controlled_writeback_consumer = (
await _load_log_controlled_writeback_consumer_readback(
project_id=project_id,
)
)
return build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=params["lookback_hours"],
db_read_status="unavailable",
error_type=error_type,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
)
log_controlled_writeback_consumer = (
await _load_log_controlled_writeback_consumer_readback(
project_id=project_id,
)
)
return build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=params["lookback_hours"],
db_read_status="partial" if partial_query_failures else "ok",
operation_count_rows=operation_counts,
operation_latest_rows=operation_latest,
auto_repair_count_rows=auto_repair_counts,
auto_repair_latest_rows=auto_repair_latest,
verifier_count_rows=verifier_counts,
verifier_latest_rows=verifier_latest,
km_count_rows=km_counts,
km_latest_rows=km_latest,
telegram_count_rows=telegram_counts,
telegram_latest_rows=telegram_latest,
mcp_gateway_count_rows=mcp_gateway_counts,
legacy_mcp_count_rows=legacy_mcp_counts,
service_log_count_rows=service_log_counts,
executor_log_count_rows=executor_log_counts,
timeline_count_rows=timeline_counts,
playbook_trust_count_rows=playbook_trust_counts,
alert_operation_count_rows=alert_operation_counts,
alertmanager_event_count_rows=alertmanager_event_counts,
grouped_alert_event_count_rows=grouped_alert_event_counts,
log_controlled_writeback_consumer=log_controlled_writeback_consumer,
error_type="partial_query_failures" if partial_query_failures else None,
partial_query_failures=partial_query_failures,
)
async def _load_core_runtime_receipt_rows_direct(
*,
project_id: str,
lookback_hours: int,
) -> dict[str, list[dict[str, Any]]] | None:
"""Read P0 core receipt counters and latest correlated rows directly."""
db_url = settings.DATABASE_URL.replace("postgresql+asyncpg://", "postgresql://")
try:
import asyncpg
conn = await asyncio.wait_for(
asyncpg.connect(db_url),
timeout=_RUNTIME_RECEIPT_DB_CONTEXT_TIMEOUT_SECONDS,
)
try:
await asyncio.wait_for(
conn.execute("SELECT set_config('app.project_id', $1, TRUE)", project_id),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
await asyncio.wait_for(
conn.execute(
"SET statement_timeout = "
f"'{int(_RUNTIME_RECEIPT_STATEMENT_TIMEOUT_MS)}ms'"
),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
async def _fetch(query_name: str, sql: str, *args: Any) -> list[dict[str, Any]]:
try:
records = await asyncio.wait_for(
conn.fetch(sql, *args),
timeout=_RUNTIME_RECEIPT_SINGLE_QUERY_TIMEOUT_SECONDS,
)
return [dict(record) for record in records]
except Exception as exc: # pragma: no cover - live schema drift
logger.warning(
"ai_agent_autonomous_runtime_direct_query_failed",
project_id=project_id,
query_name=query_name,
error_type=type(exc).__name__,
)
return []
operation_latest = await _fetch(
"operation_latest",
_RUNTIME_OPERATION_LATEST_DIRECT_SQL,
20,
)
for chain_pass in range(2):
missing_chain_refs = _missing_runtime_operation_chain_ref_ids(
operation_latest
)
if not missing_chain_refs:
break
padded_refs = [*missing_chain_refs[:3], "", ""][:3]
operation_chain_rows = await _fetch(
f"operation_chain_{chain_pass + 1}",
_RUNTIME_OPERATION_CHAIN_DIRECT_SQL,
*padded_refs,
)
if not operation_chain_rows:
break
operation_latest = _merge_runtime_operation_rows(
operation_latest,
operation_chain_rows,
)
return {
"operation_counts": await _fetch(
"operation_counts",
_RUNTIME_OPERATION_COUNTS_DIRECT_SQL,
lookback_hours,
),
"operation_latest": operation_latest,
"auto_repair_counts": await _fetch(
"auto_repair_counts",
_RUNTIME_AUTO_REPAIR_COUNTS_DIRECT_SQL,
lookback_hours,
),
"auto_repair_latest": await _fetch(
"auto_repair_latest",
_RUNTIME_AUTO_REPAIR_LATEST_DIRECT_SQL,
20,
),
"verifier_counts": await _fetch(
"verifier_counts",
_RUNTIME_VERIFIER_COUNTS_DIRECT_SQL,
lookback_hours,
),
"verifier_latest": await _fetch(
"verifier_latest",
_RUNTIME_VERIFIER_LATEST_DIRECT_SQL,
20,
),
"km_counts": await _fetch(
"km_counts",
_RUNTIME_KM_COUNTS_DIRECT_SQL,
project_id,
lookback_hours,
),
"km_latest": await _fetch(
"km_latest",
_RUNTIME_KM_LATEST_DIRECT_SQL,
project_id,
20,
),
"telegram_counts": await _fetch(
"telegram_counts",
_RUNTIME_TELEGRAM_COUNTS_DIRECT_SQL,
project_id,
lookback_hours,
),
"telegram_latest": await _fetch(
"telegram_latest",
_RUNTIME_TELEGRAM_LATEST_DIRECT_SQL,
project_id,
20,
),
}
finally:
await conn.close()
except Exception as exc: # pragma: no cover - live fallback only
logger.warning(
"ai_agent_autonomous_runtime_direct_core_readback_failed",
project_id=project_id,
error_type=type(exc).__name__,
)
return None
async def build_ai_agent_autonomous_runtime_control_with_live_readback(
*,
project_id: str = _DEFAULT_PROJECT_ID,
lookback_hours: int = _DEFAULT_LOOKBACK_HOURS,
) -> dict[str, Any]:
"""Build the control plane and attach live DB receipt readback."""
payload = build_ai_agent_autonomous_runtime_control()
runtime_readback_task = asyncio.create_task(
load_ai_agent_autonomous_runtime_receipt_readback(
project_id=project_id,
lookback_hours=lookback_hours,
)
)
boundary_readback_task = asyncio.create_task(
load_executor_trust_boundary_readback()
)
try:
readback = await asyncio.wait_for(
runtime_readback_task,
timeout=_LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS,
)
except Exception as exc: # pragma: no cover - live DB pressure
logger.warning(
"ai_agent_autonomous_runtime_live_receipt_timeout",
project_id=project_id,
timeout_seconds=_LIVE_RUNTIME_RECEIPT_TIMEOUT_SECONDS,
error_type=type(exc).__name__,
)
readback = build_runtime_receipt_readback_from_rows(
project_id=project_id,
lookback_hours=lookback_hours,
db_read_status="unavailable",
error_type=type(exc).__name__,
)
try:
boundary_readback = await boundary_readback_task
except Exception as exc: # pragma: no cover - defensive public readback
boundary_readback = build_executor_trust_boundary_readback(
{},
deployed_source_sha="",
error_type=type(exc).__name__,
)
_attach_runtime_receipt_readback(payload, readback)
payload["executor_trust_boundary"] = boundary_readback
_attach_autonomous_single_writer_readback(
payload,
readback=readback,
boundary_readback=boundary_readback,
)
_attach_independent_post_verifier_readback(
payload,
readback=readback,
boundary_readback=boundary_readback,
)
_attach_canonical_learning_readback(
payload,
readback=readback,
boundary_readback=boundary_readback,
)
payload.setdefault("rollups", {})[
"executor_trust_boundary_verified_count"
] = 1 if boundary_readback.get("production_boundary_verified") is True else 0
_validate_payload(payload)
return payload
def _attach_autonomous_single_writer_readback(
payload: dict[str, Any],
*,
readback: Mapping[str, Any],
boundary_readback: Mapping[str, Any],
) -> None:
runtime = readback.get("autonomous_single_writer_runtime")
if not isinstance(runtime, Mapping):
runtime = {}
source_boundary = boundary_readback.get(
"autonomous_single_writer_source_boundary"
)
if not isinstance(source_boundary, Mapping):
source_boundary = {}
strict_runtime = payload.get("strict_runtime_completion")
if not isinstance(strict_runtime, Mapping):
strict_runtime = {}
runtime_controls = runtime.get("controls")
if not isinstance(runtime_controls, Mapping):
runtime_controls = {}
source_controls = source_boundary.get("controls")
if not isinstance(source_controls, Mapping):
source_controls = {}
runtime_run_id = str(runtime.get("automation_run_id") or "")
strict_run_id = str(strict_runtime.get("automation_run_id") or "")
deployed_source_sha = str(
boundary_readback.get("deployed_source_sha") or ""
).lower()
router_source_sha = str(runtime.get("router_source_sha") or "").lower()
controls = {
"strict_runtime_same_run_closed": bool(
strict_runtime.get("closed") is True
and runtime_run_id
and runtime_run_id == strict_run_id
),
"router_source_sha_matches_deployment": bool(
deployed_source_sha
and router_source_sha == deployed_source_sha
),
**{
str(control_id): verified is True
for control_id, verified in runtime_controls.items()
},
**{
str(control_id): verified is True
for control_id, verified in source_controls.items()
},
}
blockers = [
*list(runtime.get("active_blockers") or []),
*list(source_boundary.get("active_blockers") or []),
*[
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
],
]
blockers = list(dict.fromkeys(str(blocker) for blocker in blockers if blocker))
closed = bool(controls and not blockers)
payload["autonomous_single_writer"] = {
"schema_version": "awoooi_autonomous_single_writer_readback_v1",
"work_item_id": "AIA-P0-002",
"status": "verified_ready" if closed else "in_progress",
"closed": closed,
"automation_run_id": runtime_run_id or None,
"router_source_sha": router_source_sha or None,
"deployed_source_sha": deployed_source_sha or None,
"candidate_op_id": runtime.get("candidate_op_id"),
"check_mode_op_id": runtime.get("check_mode_op_id"),
"apply_op_id": runtime.get("apply_op_id"),
"source_receipt_ref": source_boundary.get("receipt_ref"),
"source_verifier": source_boundary.get("verifier"),
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
"source_only_counts_as_completion": False,
}
payload.setdefault("rollups", {})[
"autonomous_single_writer_closed_count"
] = 1 if closed else 0
def _attach_independent_post_verifier_readback(
payload: dict[str, Any],
*,
readback: Mapping[str, Any],
boundary_readback: Mapping[str, Any],
) -> None:
runtime = readback.get("independent_post_verifier_runtime")
if not isinstance(runtime, Mapping):
runtime = {}
source_boundary = boundary_readback.get(
"independent_post_verifier_source_boundary"
)
if not isinstance(source_boundary, Mapping):
source_boundary = {}
strict_runtime = payload.get("strict_runtime_completion")
if not isinstance(strict_runtime, Mapping):
strict_runtime = {}
runtime_controls = runtime.get("controls")
if not isinstance(runtime_controls, Mapping):
runtime_controls = {}
source_controls = source_boundary.get("controls")
if not isinstance(source_controls, Mapping):
source_controls = {}
runtime_run_id = str(runtime.get("automation_run_id") or "")
strict_run_id = str(strict_runtime.get("automation_run_id") or "")
deployed_source_sha = str(
boundary_readback.get("deployed_source_sha") or ""
).lower()
verifier_source_sha = str(
runtime.get("verifier_source_sha") or ""
).lower()
controls = {
"strict_runtime_same_run_closed": bool(
strict_runtime.get("closed") is True
and runtime_run_id
and runtime_run_id == strict_run_id
),
"verifier_source_sha_matches_deployment": bool(
deployed_source_sha
and verifier_source_sha == deployed_source_sha
),
**{
str(control_id): verified is True
for control_id, verified in runtime_controls.items()
},
**{
str(control_id): verified is True
for control_id, verified in source_controls.items()
},
}
blockers = [
*list(runtime.get("active_blockers") or []),
*list(source_boundary.get("active_blockers") or []),
*[
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
],
]
blockers = list(dict.fromkeys(str(blocker) for blocker in blockers if blocker))
closed = bool(controls and not blockers)
payload["autonomous_independent_post_verifier"] = {
"schema_version": (
"awoooi_autonomous_independent_post_verifier_readback_v1"
),
"work_item_id": "AIA-P0-003",
"status": "verified_ready" if closed else "in_progress",
"closed": closed,
"automation_run_id": runtime_run_id or None,
"deployed_source_sha": deployed_source_sha or None,
"verifier_source_sha": verifier_source_sha or None,
"apply_op_id": runtime.get("apply_op_id"),
"verifier_evidence_id": runtime.get("verifier_evidence_id"),
"catalog_id": runtime.get("catalog_id"),
"required_postcondition_count": runtime.get(
"required_postcondition_count"
),
"passed_postcondition_count": runtime.get(
"passed_postcondition_count"
),
"source_receipt_ref": source_boundary.get("receipt_ref"),
"source_verifier": source_boundary.get("verifier"),
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
"source_only_counts_as_completion": False,
}
payload.setdefault("rollups", {})[
"autonomous_independent_post_verifier_closed_count"
] = 1 if closed else 0
def _attach_canonical_learning_readback(
payload: dict[str, Any],
*,
readback: Mapping[str, Any],
boundary_readback: Mapping[str, Any],
) -> None:
runtime = readback.get("canonical_learning_runtime")
if not isinstance(runtime, Mapping):
runtime = {}
source_boundary = boundary_readback.get(
"canonical_learning_source_boundary"
)
if not isinstance(source_boundary, Mapping):
source_boundary = {}
strict_runtime = payload.get("strict_runtime_completion")
if not isinstance(strict_runtime, Mapping):
strict_runtime = {}
runtime_controls = runtime.get("controls")
if not isinstance(runtime_controls, Mapping):
runtime_controls = {}
source_controls = source_boundary.get("controls")
if not isinstance(source_controls, Mapping):
source_controls = {}
normalized_source_controls = {
(
f"source_{control_id}"
if str(control_id) in runtime_controls
else str(control_id)
): verified is True
for control_id, verified in source_controls.items()
}
runtime_run_id = str(runtime.get("automation_run_id") or "")
strict_run_id = str(strict_runtime.get("automation_run_id") or "")
deployed_source_sha = str(
boundary_readback.get("deployed_source_sha") or ""
).lower()
writer_source_sha = str(
runtime.get("writer_source_sha") or ""
).lower()
controls = {
"strict_runtime_same_run_closed": bool(
strict_runtime.get("closed") is True
and runtime_run_id
and runtime_run_id == strict_run_id
),
"writer_source_sha_matches_deployment": bool(
deployed_source_sha
and writer_source_sha == deployed_source_sha
),
**{
str(control_id): verified is True
for control_id, verified in runtime_controls.items()
},
**normalized_source_controls,
}
blockers = [
*list(runtime.get("active_blockers") or []),
*list(source_boundary.get("active_blockers") or []),
*[
f"{control_id}_not_verified"
for control_id, verified in controls.items()
if verified is not True
],
]
blockers = list(dict.fromkeys(str(blocker) for blocker in blockers if blocker))
closed = bool(controls and not blockers)
payload["autonomous_canonical_learning_writeback"] = {
"schema_version": (
"awoooi_autonomous_canonical_learning_writeback_v1"
),
"work_item_id": "AIA-P0-004",
"status": "verified_ready" if closed else "in_progress",
"closed": closed,
"automation_run_id": runtime_run_id or None,
"deployed_source_sha": deployed_source_sha or None,
"writer_source_sha": writer_source_sha or None,
"apply_op_id": runtime.get("apply_op_id"),
"learning_operation_id": runtime.get("learning_operation_id"),
"knowledge_entry_id": runtime.get("knowledge_entry_id"),
"canonical_playbook_id": runtime.get("canonical_playbook_id"),
"catalog_id": runtime.get("catalog_id"),
"km_row_version": runtime.get("km_row_version"),
"playbook_row_version": runtime.get("playbook_row_version"),
"playbook_row_fingerprint": runtime.get(
"playbook_row_fingerprint"
),
"source_receipt_ref": source_boundary.get("receipt_ref"),
"source_verifier": source_boundary.get("verifier"),
"controls": controls,
"active_blockers": blockers,
"writes_on_read": False,
"source_only_counts_as_completion": False,
}
payload.setdefault("rollups", {})[
"autonomous_canonical_learning_writeback_closed_count"
] = 1 if closed else 0
_RUNTIME_OPERATION_COUNTS_SQL = """
SELECT
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM automation_operation_log
WHERE coalesce(input ->> 'semantic_operation_type', operation_type) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched'
)
GROUP BY 1, status
ORDER BY 1, status
"""
_RUNTIME_OPERATION_COUNTS_DIRECT_SQL = """
SELECT
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
count(*)::int AS total,
count(*) FILTER (
WHERE created_at >= NOW() - ($1 * INTERVAL '1 hour')
)::int AS recent
FROM automation_operation_log
WHERE coalesce(input ->> 'semantic_operation_type', operation_type) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched'
)
GROUP BY 1, status
ORDER BY 1, status
"""
_RUNTIME_OPERATION_LATEST_SQL = """
WITH latest_apply_chain AS (
SELECT
op_id::text AS apply_op_id,
parent_op_id::text AS check_mode_op_id,
input ->> 'source_candidate_op_id' AS candidate_op_id
FROM automation_operation_log
WHERE operation_type = 'ansible_apply_executed'
AND status IN ('success', 'failed')
ORDER BY created_at DESC
LIMIT 1
)
SELECT
op_id::text AS op_id,
parent_op_id::text AS parent_op_id,
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
actor,
CASE
WHEN coalesce(input ->> 'semantic_operation_type', operation_type)
= 'ansible_candidate_matched'
THEN op_id::text
ELSE input ->> 'automation_run_id'
END AS automation_run_id,
coalesce(incident_id::text, input ->> 'incident_id') AS incident_id,
input ->> 'catalog_id' AS catalog_id,
coalesce(input ->> 'apply_playbook_path', input ->> 'playbook_path') AS playbook_path,
input ->> 'execution_mode' AS execution_mode,
input ->> 'source_candidate_op_id' AS source_candidate_op_id,
input ->> 'check_mode_op_id' AS check_mode_op_id,
input ->> 'risk_level' AS risk_level,
input ->> 'controlled_apply_allowed' AS controlled_apply_allowed,
input ->> 'decision_path' AS decision_path,
input ->> 'single_writer_executor' AS single_writer_executor,
coalesce(
input ->> 'candidate_idempotency_key',
input ->> 'idempotency_key'
) AS candidate_idempotency_key,
input ->> 'apply_idempotency_key' AS apply_idempotency_key,
input #>> '{source_truth_diff,required_before_apply}'
AS source_truth_diff_required,
coalesce(input -> 'target_selector', '{}'::jsonb) <> '{}'::jsonb
AS target_selector_present,
input ->> 'router_source_sha' AS router_source_sha,
coalesce(
input ->> 'capability_op_id',
input #>> '{execution_capability,capability_op_id}'
) AS capability_op_id,
input ->> 'issued_at' AS capability_issued_at,
input ->> 'expires_at' AS capability_expires_at,
input ->> 'terminal_status' AS capability_terminal_status,
input -> 'runtime_stage_receipts' AS runtime_stage_receipts,
coalesce(output ->> 'returncode', dry_run_result ->> 'returncode') AS returncode,
duration_ms,
created_at
FROM automation_operation_log operation_row
LEFT JOIN latest_apply_chain ON TRUE
WHERE operation_row.operation_type IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched',
'remediation_executed',
'km_linked'
)
AND coalesce(
operation_row.input ->> 'semantic_operation_type',
operation_row.operation_type
) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched'
)
ORDER BY
CASE
WHEN operation_row.op_id::text IN (
latest_apply_chain.apply_op_id,
latest_apply_chain.check_mode_op_id,
latest_apply_chain.candidate_op_id
)
OR operation_row.input ->> 'automation_run_id'
= latest_apply_chain.candidate_op_id
THEN 0
WHEN coalesce(
input ->> 'semantic_operation_type',
operation_type
) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed'
) THEN 1
ELSE 2
END,
created_at DESC
LIMIT :limit
"""
_RUNTIME_OPERATION_LATEST_DIRECT_SQL = """
WITH latest_apply_chain AS (
SELECT
op_id::text AS apply_op_id,
parent_op_id::text AS check_mode_op_id,
input ->> 'source_candidate_op_id' AS candidate_op_id
FROM automation_operation_log
WHERE operation_type = 'ansible_apply_executed'
AND status IN ('success', 'failed')
ORDER BY created_at DESC
LIMIT 1
)
SELECT
op_id::text AS op_id,
parent_op_id::text AS parent_op_id,
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
actor,
CASE
WHEN coalesce(input ->> 'semantic_operation_type', operation_type)
= 'ansible_candidate_matched'
THEN op_id::text
ELSE input ->> 'automation_run_id'
END AS automation_run_id,
coalesce(incident_id::text, input ->> 'incident_id') AS incident_id,
input ->> 'catalog_id' AS catalog_id,
coalesce(input ->> 'apply_playbook_path', input ->> 'playbook_path') AS playbook_path,
input ->> 'execution_mode' AS execution_mode,
input ->> 'source_candidate_op_id' AS source_candidate_op_id,
input ->> 'check_mode_op_id' AS check_mode_op_id,
input ->> 'risk_level' AS risk_level,
input ->> 'controlled_apply_allowed' AS controlled_apply_allowed,
input ->> 'decision_path' AS decision_path,
input ->> 'single_writer_executor' AS single_writer_executor,
coalesce(
input ->> 'candidate_idempotency_key',
input ->> 'idempotency_key'
) AS candidate_idempotency_key,
input ->> 'apply_idempotency_key' AS apply_idempotency_key,
input #>> '{source_truth_diff,required_before_apply}'
AS source_truth_diff_required,
coalesce(input -> 'target_selector', '{}'::jsonb) <> '{}'::jsonb
AS target_selector_present,
input ->> 'router_source_sha' AS router_source_sha,
coalesce(
input ->> 'capability_op_id',
input #>> '{execution_capability,capability_op_id}'
) AS capability_op_id,
input ->> 'issued_at' AS capability_issued_at,
input ->> 'expires_at' AS capability_expires_at,
input ->> 'terminal_status' AS capability_terminal_status,
input -> 'runtime_stage_receipts' AS runtime_stage_receipts,
coalesce(output ->> 'returncode', dry_run_result ->> 'returncode') AS returncode,
duration_ms,
created_at
FROM automation_operation_log operation_row
LEFT JOIN latest_apply_chain ON TRUE
WHERE operation_row.operation_type IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched',
'remediation_executed',
'km_linked'
)
AND coalesce(
operation_row.input ->> 'semantic_operation_type',
operation_row.operation_type
) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_learning_writeback_recorded',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired',
'log_controlled_writeback_dispatched'
)
ORDER BY
CASE
WHEN operation_row.op_id::text IN (
latest_apply_chain.apply_op_id,
latest_apply_chain.check_mode_op_id,
latest_apply_chain.candidate_op_id
)
OR operation_row.input ->> 'automation_run_id'
= latest_apply_chain.candidate_op_id
THEN 0
WHEN coalesce(
input ->> 'semantic_operation_type',
operation_type
) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed'
) THEN 1
ELSE 2
END,
created_at DESC
LIMIT $1
"""
_RUNTIME_OPERATION_CHAIN_SQL = """
SELECT
op_id::text AS op_id,
parent_op_id::text AS parent_op_id,
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
actor,
CASE
WHEN coalesce(input ->> 'semantic_operation_type', operation_type)
= 'ansible_candidate_matched'
THEN op_id::text
ELSE input ->> 'automation_run_id'
END AS automation_run_id,
coalesce(incident_id::text, input ->> 'incident_id') AS incident_id,
input ->> 'catalog_id' AS catalog_id,
coalesce(input ->> 'apply_playbook_path', input ->> 'playbook_path') AS playbook_path,
input ->> 'execution_mode' AS execution_mode,
input ->> 'source_candidate_op_id' AS source_candidate_op_id,
input ->> 'check_mode_op_id' AS check_mode_op_id,
input ->> 'risk_level' AS risk_level,
input ->> 'controlled_apply_allowed' AS controlled_apply_allowed,
input ->> 'decision_path' AS decision_path,
input ->> 'single_writer_executor' AS single_writer_executor,
coalesce(
input ->> 'candidate_idempotency_key',
input ->> 'idempotency_key'
) AS candidate_idempotency_key,
input ->> 'apply_idempotency_key' AS apply_idempotency_key,
input #>> '{source_truth_diff,required_before_apply}'
AS source_truth_diff_required,
coalesce(input -> 'target_selector', '{}'::jsonb) <> '{}'::jsonb
AS target_selector_present,
input ->> 'router_source_sha' AS router_source_sha,
coalesce(
input ->> 'capability_op_id',
input #>> '{execution_capability,capability_op_id}'
) AS capability_op_id,
input ->> 'issued_at' AS capability_issued_at,
input ->> 'expires_at' AS capability_expires_at,
input ->> 'terminal_status' AS capability_terminal_status,
input -> 'runtime_stage_receipts' AS runtime_stage_receipts,
coalesce(output ->> 'returncode', dry_run_result ->> 'returncode') AS returncode,
duration_ms,
created_at
FROM automation_operation_log
WHERE op_id::text IN (
:operation_chain_ref_1,
:operation_chain_ref_2,
:operation_chain_ref_3
)
ORDER BY created_at DESC
"""
_RUNTIME_OPERATION_CHAIN_DIRECT_SQL = """
SELECT
op_id::text AS op_id,
parent_op_id::text AS parent_op_id,
coalesce(input ->> 'semantic_operation_type', operation_type)
AS operation_type,
status,
actor,
CASE
WHEN coalesce(input ->> 'semantic_operation_type', operation_type)
= 'ansible_candidate_matched'
THEN op_id::text
ELSE input ->> 'automation_run_id'
END AS automation_run_id,
coalesce(incident_id::text, input ->> 'incident_id') AS incident_id,
input ->> 'catalog_id' AS catalog_id,
coalesce(input ->> 'apply_playbook_path', input ->> 'playbook_path') AS playbook_path,
input ->> 'execution_mode' AS execution_mode,
input ->> 'source_candidate_op_id' AS source_candidate_op_id,
input ->> 'check_mode_op_id' AS check_mode_op_id,
input ->> 'risk_level' AS risk_level,
input ->> 'controlled_apply_allowed' AS controlled_apply_allowed,
input ->> 'decision_path' AS decision_path,
input ->> 'single_writer_executor' AS single_writer_executor,
coalesce(
input ->> 'candidate_idempotency_key',
input ->> 'idempotency_key'
) AS candidate_idempotency_key,
input ->> 'apply_idempotency_key' AS apply_idempotency_key,
input #>> '{source_truth_diff,required_before_apply}'
AS source_truth_diff_required,
coalesce(input -> 'target_selector', '{}'::jsonb) <> '{}'::jsonb
AS target_selector_present,
input ->> 'router_source_sha' AS router_source_sha,
coalesce(
input ->> 'capability_op_id',
input #>> '{execution_capability,capability_op_id}'
) AS capability_op_id,
input ->> 'issued_at' AS capability_issued_at,
input ->> 'expires_at' AS capability_expires_at,
input ->> 'terminal_status' AS capability_terminal_status,
input -> 'runtime_stage_receipts' AS runtime_stage_receipts,
coalesce(output ->> 'returncode', dry_run_result ->> 'returncode') AS returncode,
duration_ms,
created_at
FROM automation_operation_log
WHERE op_id::text IN ($1, $2, $3)
ORDER BY created_at DESC
"""
_RUNTIME_AUTO_REPAIR_COUNTS_SQL = """
SELECT
CASE WHEN success THEN 'success' ELSE 'failed' END AS result_status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM auto_repair_executions
WHERE triggered_by = 'ansible_controlled_apply'
GROUP BY CASE WHEN success THEN 'success' ELSE 'failed' END
ORDER BY result_status
"""
_RUNTIME_AUTO_REPAIR_COUNTS_DIRECT_SQL = """
SELECT
CASE WHEN success THEN 'success' ELSE 'failed' END AS result_status,
count(*)::int AS total,
count(*) FILTER (
WHERE created_at >= NOW() - ($1 * INTERVAL '1 hour')
)::int AS recent
FROM auto_repair_executions
WHERE triggered_by = 'ansible_controlled_apply'
GROUP BY CASE WHEN success THEN 'success' ELSE 'failed' END
ORDER BY result_status
"""
_RUNTIME_AUTO_REPAIR_LATEST_SQL = """
SELECT
id,
incident_id,
playbook_id AS catalog_id,
playbook_name,
CASE WHEN success THEN 'success' ELSE 'failed' END AS result_status,
substring(
executed_steps::text
from 'automation_run_id:([0-9A-Za-z-]+)'
) AS automation_run_id,
executed_steps::text AS executed_steps_text,
triggered_by,
risk_level,
execution_time_ms,
created_at
FROM auto_repair_executions
WHERE triggered_by = 'ansible_controlled_apply'
ORDER BY created_at DESC
LIMIT :limit
"""
_RUNTIME_AUTO_REPAIR_LATEST_DIRECT_SQL = """
SELECT
id,
incident_id,
playbook_id AS catalog_id,
playbook_name,
CASE WHEN success THEN 'success' ELSE 'failed' END AS result_status,
substring(
executed_steps::text
from 'automation_run_id:([0-9A-Za-z-]+)'
) AS automation_run_id,
executed_steps::text AS executed_steps_text,
triggered_by,
risk_level,
execution_time_ms,
created_at
FROM auto_repair_executions
WHERE triggered_by = 'ansible_controlled_apply'
ORDER BY created_at DESC
LIMIT $1
"""
_RUNTIME_VERIFIER_COUNTS_SQL = """
SELECT
coalesce(verification_result, 'missing') AS verification_result,
count(*) AS total,
count(*) FILTER (
WHERE collected_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM incident_evidence
WHERE post_execution_state ->> 'apply_op_id' IS NOT NULL
GROUP BY coalesce(verification_result, 'missing')
ORDER BY verification_result
"""
_RUNTIME_VERIFIER_COUNTS_DIRECT_SQL = """
SELECT
coalesce(verification_result, 'missing') AS verification_result,
count(*)::int AS total,
count(*) FILTER (
WHERE collected_at >= NOW() - ($1 * INTERVAL '1 hour')
)::int AS recent
FROM incident_evidence
WHERE post_execution_state ->> 'apply_op_id' IS NOT NULL
GROUP BY coalesce(verification_result, 'missing')
ORDER BY verification_result
"""
_RUNTIME_VERIFIER_LATEST_SQL = """
SELECT
evidence.id,
evidence.incident_id,
evidence.matched_playbook_id,
coalesce(evidence.verification_result, 'missing')
AS verification_result,
evidence.post_execution_state ->> 'automation_run_id'
AS automation_run_id,
evidence.post_execution_state ->> 'apply_op_id' AS apply_op_id,
evidence.post_execution_state ->> 'catalog_id' AS catalog_id,
evidence.post_execution_state ->> 'playbook_path' AS playbook_path,
evidence.post_execution_state ->> 'returncode' AS returncode,
evidence.post_execution_state ->> 'schema_version'
AS verifier_schema_version,
evidence.post_execution_state ->> 'verifier' AS verifier,
evidence.post_execution_state ->> 'independent_source'
AS independent_source,
evidence.post_execution_state ->> 'verifier_source_sha'
AS verifier_source_sha,
evidence.post_execution_state ->> 'executor_returncode_trusted'
AS executor_returncode_trusted,
evidence.post_execution_state ->> 'all_postconditions_passed'
AS all_postconditions_passed,
evidence.post_execution_state ->> 'required_postcondition_count'
AS required_postcondition_count,
evidence.post_execution_state ->> 'passed_postcondition_count'
AS passed_postcondition_count,
evidence.post_execution_state ->> 'raw_output_stored'
AS raw_output_stored,
apply.output ->> 'independent_post_verifier_passed'
AS apply_terminal_verifier_passed,
apply.status AS apply_status,
evidence.collected_at
FROM incident_evidence evidence
LEFT JOIN automation_operation_log apply
ON apply.op_id = CAST(
evidence.post_execution_state ->> 'apply_op_id' AS uuid
)
WHERE evidence.post_execution_state ->> 'apply_op_id' IS NOT NULL
ORDER BY evidence.collected_at DESC
LIMIT :limit
"""
_RUNTIME_VERIFIER_LATEST_DIRECT_SQL = """
SELECT
evidence.id,
evidence.incident_id,
evidence.matched_playbook_id,
coalesce(evidence.verification_result, 'missing')
AS verification_result,
evidence.post_execution_state ->> 'automation_run_id'
AS automation_run_id,
evidence.post_execution_state ->> 'apply_op_id' AS apply_op_id,
evidence.post_execution_state ->> 'catalog_id' AS catalog_id,
evidence.post_execution_state ->> 'playbook_path' AS playbook_path,
evidence.post_execution_state ->> 'returncode' AS returncode,
evidence.post_execution_state ->> 'schema_version'
AS verifier_schema_version,
evidence.post_execution_state ->> 'verifier' AS verifier,
evidence.post_execution_state ->> 'independent_source'
AS independent_source,
evidence.post_execution_state ->> 'verifier_source_sha'
AS verifier_source_sha,
evidence.post_execution_state ->> 'executor_returncode_trusted'
AS executor_returncode_trusted,
evidence.post_execution_state ->> 'all_postconditions_passed'
AS all_postconditions_passed,
evidence.post_execution_state ->> 'required_postcondition_count'
AS required_postcondition_count,
evidence.post_execution_state ->> 'passed_postcondition_count'
AS passed_postcondition_count,
evidence.post_execution_state ->> 'raw_output_stored'
AS raw_output_stored,
apply.output ->> 'independent_post_verifier_passed'
AS apply_terminal_verifier_passed,
apply.status AS apply_status,
evidence.collected_at
FROM incident_evidence evidence
LEFT JOIN automation_operation_log apply
ON apply.op_id = CAST(
evidence.post_execution_state ->> 'apply_op_id' AS uuid
)
WHERE evidence.post_execution_state ->> 'apply_op_id' IS NOT NULL
ORDER BY evidence.collected_at DESC
LIMIT $1
"""
_RUNTIME_KM_COUNTS_SQL = """
SELECT
status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM knowledge_entries
WHERE project_id = :project_id
AND (
path_type LIKE 'ansible_apply_receipt:%'
OR tags::text LIKE '%ansible_controlled_apply%'
)
GROUP BY status
ORDER BY status
"""
_RUNTIME_KM_COUNTS_DIRECT_SQL = """
SELECT
status,
count(*)::int AS total,
count(*) FILTER (
WHERE created_at >= NOW() - ($2 * INTERVAL '1 hour')
)::int AS recent
FROM knowledge_entries
WHERE project_id = $1
AND (
path_type LIKE 'ansible_apply_receipt:%'
OR tags::text LIKE '%ansible_controlled_apply%'
)
GROUP BY status
ORDER BY status
"""
_RUNTIME_KM_LATEST_SQL = """
SELECT
id,
title,
related_incident_id,
related_playbook_id,
path_type,
status,
created_by,
(
SELECT split_part(tag.value, ':', 2)
FROM json_array_elements_text(tags) AS tag(value)
WHERE tag.value LIKE 'automation_run_id:%'
LIMIT 1
) AS automation_run_id,
created_at
FROM knowledge_entries
WHERE project_id = :project_id
AND (
path_type LIKE 'ansible_apply_receipt:%'
OR tags::text LIKE '%ansible_controlled_apply%'
)
ORDER BY created_at DESC
LIMIT :limit
"""
_RUNTIME_KM_LATEST_DIRECT_SQL = """
SELECT
id,
title,
related_incident_id,
related_playbook_id,
path_type,
status,
created_by,
(
SELECT split_part(tag.value, ':', 2)
FROM json_array_elements_text(tags) AS tag(value)
WHERE tag.value LIKE 'automation_run_id:%'
LIMIT 1
) AS automation_run_id,
created_at
FROM knowledge_entries
WHERE project_id = $1
AND (
path_type LIKE 'ansible_apply_receipt:%'
OR tags::text LIKE '%ansible_controlled_apply%'
)
ORDER BY created_at DESC
LIMIT $2
"""
_RUNTIME_TELEGRAM_COUNTS_SQL = """
SELECT
send_status,
count(*) AS total,
count(*) FILTER (
WHERE queued_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM awooop_outbound_message
WHERE project_id = :project_id
AND channel_type = 'telegram'
AND (
source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result'
OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent'
)
GROUP BY send_status
ORDER BY send_status
"""
_RUNTIME_TELEGRAM_COUNTS_DIRECT_SQL = """
SELECT
send_status,
count(*)::int AS total,
count(*) FILTER (
WHERE queued_at >= NOW() - ($2 * INTERVAL '1 hour')
)::int AS recent
FROM awooop_outbound_message
WHERE project_id = $1
AND channel_type = 'telegram'
AND (
source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result'
OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent'
)
GROUP BY send_status
ORDER BY send_status
"""
_RUNTIME_TELEGRAM_LATEST_SQL = """
SELECT
message_id::text AS message_id,
run_id::text AS outbound_run_id,
COALESCE(
source_envelope ->> 'automation_run_id',
source_envelope #>> '{callback_reply,automation_run_id}',
source_envelope #>> '{source_refs,automation_run_ids,0}'
) AS automation_run_id,
message_type,
send_status,
provider_message_id,
COALESCE(
source_envelope #>> '{callback_reply,incident_id}',
source_envelope #>> '{source_refs,incident_ids,0}'
) AS incident_id,
COALESCE(
source_envelope #>> '{callback_reply,action}',
source_envelope #>> '{alert_notification,status}'
) AS action,
queued_at,
sent_at
FROM awooop_outbound_message
WHERE project_id = :project_id
AND channel_type = 'telegram'
AND (
source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result'
OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent'
)
ORDER BY queued_at DESC
LIMIT :limit
"""
_RUNTIME_TELEGRAM_LATEST_DIRECT_SQL = """
SELECT
message_id::text AS message_id,
run_id::text AS outbound_run_id,
COALESCE(
source_envelope ->> 'automation_run_id',
source_envelope #>> '{callback_reply,automation_run_id}',
source_envelope #>> '{source_refs,automation_run_ids,0}'
) AS automation_run_id,
message_type,
send_status,
provider_message_id,
COALESCE(
source_envelope #>> '{callback_reply,incident_id}',
source_envelope #>> '{source_refs,incident_ids,0}'
) AS incident_id,
COALESCE(
source_envelope #>> '{callback_reply,action}',
source_envelope #>> '{alert_notification,status}'
) AS action,
queued_at,
sent_at
FROM awooop_outbound_message
WHERE project_id = $1
AND channel_type = 'telegram'
AND (
source_envelope #>> '{callback_reply,action}' = 'controlled_apply_result'
OR source_envelope #>> '{alert_notification,status}' = 'alert_notification_sent'
)
ORDER BY queued_at DESC
LIMIT $2
"""
_RUNTIME_MCP_GATEWAY_COUNTS_SQL = """
SELECT
coalesce(result_status, 'unknown') AS status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM awooop_mcp_gateway_audit
WHERE project_id = :project_id
GROUP BY coalesce(result_status, 'unknown')
ORDER BY status
"""
_RUNTIME_LEGACY_MCP_COUNTS_SQL = """
SELECT
CASE
WHEN success IS TRUE THEN 'success'
WHEN success IS FALSE THEN 'failed'
ELSE 'unknown'
END AS status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM mcp_audit_log
GROUP BY
CASE
WHEN success IS TRUE THEN 'success'
WHEN success IS FALSE THEN 'failed'
ELSE 'unknown'
END
ORDER BY status
"""
_RUNTIME_SERVICE_LOG_COUNTS_SQL = """
SELECT
'sanitized_recent_logs' AS status,
count(*) AS total,
count(*) FILTER (
WHERE collected_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM incident_evidence
WHERE recent_logs IS NOT NULL
OR evidence_summary IS NOT NULL
OR mcp_health IS NOT NULL
OR anomaly_context IS NOT NULL
"""
_RUNTIME_EXECUTOR_LOG_COUNTS_SQL = """
SELECT
coalesce(status, 'unknown') AS status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM automation_operation_log
WHERE coalesce(input ->> 'semantic_operation_type', operation_type) IN (
'ansible_candidate_matched',
'ansible_check_mode_executed',
'ansible_apply_executed',
'ansible_rollback_executed',
'ansible_execution_skipped',
'ansible_executor_capability_issued',
'ansible_executor_capability_revoked',
'ansible_executor_capability_expired'
)
AND (
output IS NOT NULL
OR error IS NOT NULL
OR stderr_feed_back IS NOT NULL
OR dry_run_result IS NOT NULL
)
GROUP BY coalesce(status, 'unknown')
ORDER BY status
"""
_RUNTIME_TIMELINE_COUNTS_SQL = """
SELECT
coalesce(status, 'unknown') AS status,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM timeline_events
WHERE event_type IS NOT NULL
OR actor IS NOT NULL
OR actor_role IS NOT NULL
GROUP BY coalesce(status, 'unknown')
ORDER BY status
"""
_RUNTIME_TIMELINE_COUNTS_FALLBACK_SQL = """
SELECT
'timeline_event' AS status,
count(*) AS total,
0 AS recent
FROM timeline_events
"""
_RUNTIME_PLAYBOOK_TRUST_COUNTS_SQL = """
SELECT
CASE
WHEN review_required IS TRUE THEN 'review_required'
WHEN trust_score >= 0.8 THEN 'high_trust'
WHEN trust_score < 0.3 THEN 'low_trust'
WHEN success_count > 0 OR failure_count > 0 THEN 'learning_active'
ELSE 'seeded_not_used'
END AS status,
count(*) AS total,
count(*) FILTER (
WHERE updated_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM playbooks
GROUP BY
CASE
WHEN review_required IS TRUE THEN 'review_required'
WHEN trust_score >= 0.8 THEN 'high_trust'
WHEN trust_score < 0.3 THEN 'low_trust'
WHEN success_count > 0 OR failure_count > 0 THEN 'learning_active'
ELSE 'seeded_not_used'
END
ORDER BY status
"""
_RUNTIME_PLAYBOOK_TRUST_COUNTS_FALLBACK_SQL = """
SELECT
'cataloged' AS status,
count(*) AS total,
0 AS recent
FROM playbooks
"""
_RUNTIME_ALERT_OPERATION_COUNTS_SQL = """
SELECT
event_type,
count(*) AS total,
count(*) FILTER (
WHERE created_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM alert_operation_log
WHERE event_type IN (
'ALERT_RECEIVED',
'AUTO_REPAIR_TRIGGERED',
'EXECUTION_STARTED',
'EXECUTION_COMPLETED',
'NOTIFICATION_CLASSIFIED',
'GUARDRAIL_BLOCKED',
'STATE_GUARD_BLOCKED',
'SILENCED',
'ESCALATED'
)
GROUP BY event_type
ORDER BY event_type
"""
_RUNTIME_ALERTMANAGER_EVENT_COUNTS_SQL = """
SELECT
COALESCE(NULLIF(source_envelope ->> 'stage', ''), 'unknown') AS stage,
count(*) AS total,
count(*) FILTER (
WHERE received_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM awooop_conversation_event
WHERE project_id = :project_id
AND COALESCE(
NULLIF(source_envelope ->> 'provider', ''),
platform_subject_id,
''
) = 'alertmanager'
GROUP BY COALESCE(NULLIF(source_envelope ->> 'stage', ''), 'unknown')
ORDER BY stage
"""
_RUNTIME_GROUPED_ALERT_EVENT_COUNTS_SQL = """
SELECT
'grouped_child_alert' AS status,
count(*) AS total,
count(*) FILTER (
WHERE received_at >= NOW() - (:lookback_hours * INTERVAL '1 hour')
) AS recent
FROM awooop_conversation_event
WHERE project_id = :project_id
AND channel_chat_id LIKE 'alert-group:%'
"""
def _validate_payload(payload: dict[str, Any]) -> None:
if payload.get("schema_version") != _SCHEMA_VERSION:
raise ValueError(f"schema_version must be {_SCHEMA_VERSION}")
status = payload.get("program_status") or {}
if status.get("runtime_authority") != _RUNTIME_AUTHORITY:
raise ValueError(f"runtime_authority must be {_RUNTIME_AUTHORITY}")
if status.get("deploy_readback_marker") != _DEPLOY_READBACK_MARKER:
raise ValueError(f"deploy_readback_marker must be {_DEPLOY_READBACK_MARKER}")
if status.get("deploy_attempt_note") != _DEPLOY_ATTEMPT_NOTE:
raise ValueError(f"deploy_attempt_note must be {_DEPLOY_ATTEMPT_NOTE}")
policy = payload.get("current_policy") or {}
for key in (
"low_risk_controlled_apply_allowed",
"medium_risk_controlled_apply_allowed",
"high_risk_controlled_apply_allowed",
"telegram_gateway_required",
"post_apply_verifier_required",
"km_learning_writeback_required",
):
if policy.get(key) is not True:
raise ValueError(f"current_policy.{key} must be true")
if policy.get("owner_review_required_for_low_medium_high") is not False:
raise ValueError("owner_review_required_for_low_medium_high must be false")
if policy.get("direct_bot_api_allowed") is not False:
raise ValueError("direct_bot_api_allowed must be false")
visibility = payload.get("visibility_contract") or {}
for key in (
"work_window_transcript_display_allowed",
"prompt_body_display_allowed",
"internal_reasoning_display_allowed",
"sensitive_value_display_allowed",
"telegram_unredacted_payload_display_allowed",
):
if visibility.get(key) is not False:
raise ValueError(f"visibility_contract.{key} must remain false")