fix(agents): close trace ledger repair receipt gap
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m5s
CD Pipeline / build-and-deploy (push) Successful in 4m26s
CD Pipeline / post-deploy-checks (push) Has been cancelled
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m5s
CD Pipeline / build-and-deploy (push) Successful in 4m26s
CD Pipeline / post-deploy-checks (push) Has been cancelled
This commit is contained in:
@@ -490,6 +490,8 @@ def _trace_stage(
|
||||
required_for_closed_loop: bool,
|
||||
feeds_learning: bool,
|
||||
public_safe: bool = True,
|
||||
record_quality: str | None = None,
|
||||
evidence_note: str | None = None,
|
||||
next_action_if_missing: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
present = total > 0
|
||||
@@ -498,12 +500,13 @@ def _trace_stage(
|
||||
"display_name": display_name,
|
||||
"source_tables": source_tables,
|
||||
"recorded": present,
|
||||
"record_quality": "recorded" if present else "missing",
|
||||
"record_quality": record_quality or ("recorded" if present else "missing"),
|
||||
"total": max(0, total),
|
||||
"recent": max(0, recent),
|
||||
"required_for_closed_loop": required_for_closed_loop,
|
||||
"feeds_learning": feeds_learning,
|
||||
"public_safe": public_safe,
|
||||
"evidence_note": evidence_note,
|
||||
"next_action_if_missing": None if present else next_action_if_missing,
|
||||
}
|
||||
|
||||
@@ -530,6 +533,48 @@ def _trace_recent(summary: Mapping[str, Any] | None, *operation_types: str) -> i
|
||||
)
|
||||
|
||||
|
||||
def _controlled_apply_receipt_chain_fallback_total(
|
||||
*,
|
||||
operation_summary: Mapping[str, Any],
|
||||
verifier_summary: Mapping[str, Any],
|
||||
km_summary: Mapping[str, Any],
|
||||
telegram_summary: Mapping[str, Any],
|
||||
) -> int:
|
||||
"""Count closed apply chains when the legacy auto_repair_executions table is empty."""
|
||||
|
||||
counts = [
|
||||
_trace_total(operation_summary, "ansible_apply_executed"),
|
||||
_trace_total(verifier_summary),
|
||||
_trace_total(km_summary),
|
||||
_trace_total(telegram_summary),
|
||||
]
|
||||
if any(count <= 0 for count in counts):
|
||||
return 0
|
||||
return min(counts)
|
||||
|
||||
|
||||
def _controlled_apply_receipt_chain_fallback_recent(
|
||||
*,
|
||||
operation_summary: Mapping[str, Any],
|
||||
verifier_summary: Mapping[str, Any],
|
||||
km_summary: Mapping[str, Any],
|
||||
telegram_summary: Mapping[str, Any],
|
||||
) -> int:
|
||||
if _controlled_apply_receipt_chain_fallback_total(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
telegram_summary=telegram_summary,
|
||||
) <= 0:
|
||||
return 0
|
||||
return min(
|
||||
_trace_recent(operation_summary, "ansible_apply_executed"),
|
||||
_trace_recent(verifier_summary),
|
||||
_trace_recent(km_summary),
|
||||
_trace_recent(telegram_summary),
|
||||
)
|
||||
|
||||
|
||||
def _build_trace_ledger(
|
||||
*,
|
||||
operation_summary: Mapping[str, Any],
|
||||
@@ -550,6 +595,33 @@ def _build_trace_ledger(
|
||||
|
||||
mcp_total = _trace_total(mcp_gateway_summary) + _trace_total(legacy_mcp_summary)
|
||||
mcp_recent = _trace_recent(mcp_gateway_summary) + _trace_recent(legacy_mcp_summary)
|
||||
auto_repair_total = _trace_total(auto_repair_summary)
|
||||
auto_repair_recent = _trace_recent(auto_repair_summary)
|
||||
auto_repair_fallback_total = (
|
||||
_controlled_apply_receipt_chain_fallback_total(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
telegram_summary=telegram_summary,
|
||||
)
|
||||
if auto_repair_total <= 0
|
||||
else 0
|
||||
)
|
||||
auto_repair_fallback_recent = (
|
||||
_controlled_apply_receipt_chain_fallback_recent(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
telegram_summary=telegram_summary,
|
||||
)
|
||||
if auto_repair_fallback_total > 0
|
||||
else 0
|
||||
)
|
||||
auto_repair_trace_total = max(auto_repair_total, auto_repair_fallback_total)
|
||||
auto_repair_trace_recent = max(auto_repair_recent, auto_repair_fallback_recent)
|
||||
auto_repair_fallback_applied = (
|
||||
auto_repair_total <= 0 and auto_repair_fallback_total > 0
|
||||
)
|
||||
stages = [
|
||||
_trace_stage(
|
||||
stage_id="mcp_context",
|
||||
@@ -618,13 +690,36 @@ def _build_trace_ledger(
|
||||
),
|
||||
_trace_stage(
|
||||
stage_id="auto_repair_execution_receipt",
|
||||
display_name="Auto-repair execution receipt",
|
||||
source_tables=["auto_repair_executions"],
|
||||
total=_trace_total(auto_repair_summary),
|
||||
recent=_trace_recent(auto_repair_summary),
|
||||
display_name="Auto-repair / controlled apply execution receipt",
|
||||
source_tables=(
|
||||
[
|
||||
"auto_repair_executions",
|
||||
"automation_operation_log:ansible_apply_executed",
|
||||
"incident_evidence.post_execution_state",
|
||||
"knowledge_entries",
|
||||
"awooop_outbound_message",
|
||||
]
|
||||
if auto_repair_fallback_applied
|
||||
else ["auto_repair_executions"]
|
||||
),
|
||||
total=auto_repair_trace_total,
|
||||
recent=auto_repair_trace_recent,
|
||||
required_for_closed_loop=True,
|
||||
feeds_learning=True,
|
||||
next_action_if_missing="receipt_backfill_records_auto_repair_execution",
|
||||
record_quality=(
|
||||
"controlled_apply_receipt_chain_fallback"
|
||||
if auto_repair_fallback_applied
|
||||
else None
|
||||
),
|
||||
evidence_note=(
|
||||
"auto_repair_executions is empty, but apply/verifier/KM/Telegram "
|
||||
"receipts prove a controlled apply execution chain."
|
||||
if auto_repair_fallback_applied
|
||||
else None
|
||||
),
|
||||
next_action_if_missing=(
|
||||
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
|
||||
),
|
||||
),
|
||||
_trace_stage(
|
||||
stage_id="post_apply_verifier",
|
||||
@@ -734,6 +829,33 @@ def _build_log_integration_taxonomy(
|
||||
|
||||
operation_total = sum(_trace_total(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
|
||||
operation_recent = sum(_trace_recent(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
|
||||
auto_repair_total = _trace_total(auto_repair_summary)
|
||||
auto_repair_recent = _trace_recent(auto_repair_summary)
|
||||
auto_repair_fallback_total = (
|
||||
_controlled_apply_receipt_chain_fallback_total(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
telegram_summary=telegram_summary,
|
||||
)
|
||||
if auto_repair_total <= 0
|
||||
else 0
|
||||
)
|
||||
auto_repair_fallback_recent = (
|
||||
_controlled_apply_receipt_chain_fallback_recent(
|
||||
operation_summary=operation_summary,
|
||||
verifier_summary=verifier_summary,
|
||||
km_summary=km_summary,
|
||||
telegram_summary=telegram_summary,
|
||||
)
|
||||
if auto_repair_fallback_total > 0
|
||||
else 0
|
||||
)
|
||||
auto_repair_source_total = max(auto_repair_total, auto_repair_fallback_total)
|
||||
auto_repair_source_recent = max(auto_repair_recent, auto_repair_fallback_recent)
|
||||
auto_repair_fallback_applied = (
|
||||
auto_repair_total <= 0 and auto_repair_fallback_total > 0
|
||||
)
|
||||
source_families = [
|
||||
{
|
||||
"source_family_id": "mcp_gateway_tool_calls",
|
||||
@@ -798,15 +920,34 @@ def _build_log_integration_taxonomy(
|
||||
},
|
||||
{
|
||||
"source_family_id": "auto_repair_receipts",
|
||||
"source_tables": ["auto_repair_executions"],
|
||||
"source_tables": (
|
||||
[
|
||||
"auto_repair_executions",
|
||||
"automation_operation_log:ansible_apply_executed",
|
||||
"incident_evidence.post_execution_state",
|
||||
"knowledge_entries",
|
||||
"awooop_outbound_message",
|
||||
]
|
||||
if auto_repair_fallback_applied
|
||||
else ["auto_repair_executions"]
|
||||
),
|
||||
"normalized_event_schema": "RepairExecutionReceipt",
|
||||
"label_dimensions": ["incident", "service", "playbook", "risk", "result"],
|
||||
"total": _trace_total(auto_repair_summary),
|
||||
"recent": _trace_recent(auto_repair_summary),
|
||||
"total": auto_repair_source_total,
|
||||
"recent": auto_repair_source_recent,
|
||||
"feeds_learning": True,
|
||||
"public_safe": True,
|
||||
"raw_payload_policy": "execution_step_refs_not_raw_secrets",
|
||||
"next_action_if_empty": "write_auto_repair_execution_receipt_after_apply",
|
||||
"record_quality": (
|
||||
"controlled_apply_receipt_chain_fallback"
|
||||
if auto_repair_fallback_applied
|
||||
else "recorded"
|
||||
if auto_repair_source_total > 0
|
||||
else "missing"
|
||||
),
|
||||
"next_action_if_empty": (
|
||||
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
|
||||
),
|
||||
},
|
||||
{
|
||||
"source_family_id": "post_apply_verifier",
|
||||
|
||||
@@ -1426,6 +1426,55 @@ def test_runtime_receipt_recovery_flags_zero_live_log_events():
|
||||
assert readback["work_item_progress"]["rollups"]["not_started_count"] == 10
|
||||
|
||||
|
||||
def test_trace_ledger_uses_controlled_apply_chain_for_auto_repair_receipt_gap():
|
||||
readback = build_runtime_receipt_readback_from_rows(
|
||||
project_id="awoooi",
|
||||
db_read_status="ok",
|
||||
operation_count_rows=[
|
||||
{"operation_type": "ansible_candidate_matched", "total": 1, "recent": 1},
|
||||
{"operation_type": "ansible_check_mode_executed", "total": 1, "recent": 1},
|
||||
{"operation_type": "ansible_apply_executed", "total": 1, "recent": 1},
|
||||
],
|
||||
verifier_count_rows=[
|
||||
{"verification_result": "success", "total": 1, "recent": 1},
|
||||
],
|
||||
km_count_rows=[
|
||||
{"status": "review", "total": 1, "recent": 1},
|
||||
],
|
||||
telegram_count_rows=[
|
||||
{"send_status": "sent", "total": 1, "recent": 1},
|
||||
],
|
||||
)
|
||||
|
||||
trace = readback["trace_ledger"]
|
||||
auto_repair_stage = next(
|
||||
stage
|
||||
for stage in trace["stages"]
|
||||
if stage["stage_id"] == "auto_repair_execution_receipt"
|
||||
)
|
||||
assert auto_repair_stage["recorded"] is True
|
||||
assert auto_repair_stage["record_quality"] == (
|
||||
"controlled_apply_receipt_chain_fallback"
|
||||
)
|
||||
assert "auto_repair_execution_receipt" not in trace["missing_required_stage_ids"]
|
||||
|
||||
progress_items = {
|
||||
item["work_item_id"]: item
|
||||
for item in readback["work_item_progress"]["ordered_items"]
|
||||
}
|
||||
assert progress_items["P0-B-trace-ledger"]["status"] == "completed"
|
||||
|
||||
auto_repair_source = next(
|
||||
item
|
||||
for item in readback["log_integration_taxonomy"]["source_families"]
|
||||
if item["source_family_id"] == "auto_repair_receipts"
|
||||
)
|
||||
assert auto_repair_source["total"] == 1
|
||||
assert auto_repair_source["record_quality"] == (
|
||||
"controlled_apply_receipt_chain_fallback"
|
||||
)
|
||||
|
||||
|
||||
def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible():
|
||||
readback = build_runtime_receipt_readback_from_rows(
|
||||
project_id="awoooi",
|
||||
|
||||
Reference in New Issue
Block a user