fix(agents): close trace ledger repair receipt gap
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 1m5s
CD Pipeline / build-and-deploy (push) Successful in 4m26s
CD Pipeline / post-deploy-checks (push) Has been cancelled

This commit is contained in:
ogt
2026-07-09 22:48:17 +08:00
parent abd7ec0815
commit 0fe3b4b593
2 changed files with 200 additions and 10 deletions

View File

@@ -490,6 +490,8 @@ def _trace_stage(
required_for_closed_loop: bool,
feeds_learning: bool,
public_safe: bool = True,
record_quality: str | None = None,
evidence_note: str | None = None,
next_action_if_missing: str | None = None,
) -> dict[str, Any]:
present = total > 0
@@ -498,12 +500,13 @@ def _trace_stage(
"display_name": display_name,
"source_tables": source_tables,
"recorded": present,
"record_quality": "recorded" if present else "missing",
"record_quality": record_quality or ("recorded" if present else "missing"),
"total": max(0, total),
"recent": max(0, recent),
"required_for_closed_loop": required_for_closed_loop,
"feeds_learning": feeds_learning,
"public_safe": public_safe,
"evidence_note": evidence_note,
"next_action_if_missing": None if present else next_action_if_missing,
}
@@ -530,6 +533,48 @@ def _trace_recent(summary: Mapping[str, Any] | None, *operation_types: str) -> i
)
def _controlled_apply_receipt_chain_fallback_total(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
) -> int:
"""Count closed apply chains when the legacy auto_repair_executions table is empty."""
counts = [
_trace_total(operation_summary, "ansible_apply_executed"),
_trace_total(verifier_summary),
_trace_total(km_summary),
_trace_total(telegram_summary),
]
if any(count <= 0 for count in counts):
return 0
return min(counts)
def _controlled_apply_receipt_chain_fallback_recent(
*,
operation_summary: Mapping[str, Any],
verifier_summary: Mapping[str, Any],
km_summary: Mapping[str, Any],
telegram_summary: Mapping[str, Any],
) -> int:
if _controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
) <= 0:
return 0
return min(
_trace_recent(operation_summary, "ansible_apply_executed"),
_trace_recent(verifier_summary),
_trace_recent(km_summary),
_trace_recent(telegram_summary),
)
def _build_trace_ledger(
*,
operation_summary: Mapping[str, Any],
@@ -550,6 +595,33 @@ def _build_trace_ledger(
mcp_total = _trace_total(mcp_gateway_summary) + _trace_total(legacy_mcp_summary)
mcp_recent = _trace_recent(mcp_gateway_summary) + _trace_recent(legacy_mcp_summary)
auto_repair_total = _trace_total(auto_repair_summary)
auto_repair_recent = _trace_recent(auto_repair_summary)
auto_repair_fallback_total = (
_controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_total <= 0
else 0
)
auto_repair_fallback_recent = (
_controlled_apply_receipt_chain_fallback_recent(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_fallback_total > 0
else 0
)
auto_repair_trace_total = max(auto_repair_total, auto_repair_fallback_total)
auto_repair_trace_recent = max(auto_repair_recent, auto_repair_fallback_recent)
auto_repair_fallback_applied = (
auto_repair_total <= 0 and auto_repair_fallback_total > 0
)
stages = [
_trace_stage(
stage_id="mcp_context",
@@ -618,13 +690,36 @@ def _build_trace_ledger(
),
_trace_stage(
stage_id="auto_repair_execution_receipt",
display_name="Auto-repair execution receipt",
source_tables=["auto_repair_executions"],
total=_trace_total(auto_repair_summary),
recent=_trace_recent(auto_repair_summary),
display_name="Auto-repair / controlled apply execution receipt",
source_tables=(
[
"auto_repair_executions",
"automation_operation_log:ansible_apply_executed",
"incident_evidence.post_execution_state",
"knowledge_entries",
"awooop_outbound_message",
]
if auto_repair_fallback_applied
else ["auto_repair_executions"]
),
total=auto_repair_trace_total,
recent=auto_repair_trace_recent,
required_for_closed_loop=True,
feeds_learning=True,
next_action_if_missing="receipt_backfill_records_auto_repair_execution",
record_quality=(
"controlled_apply_receipt_chain_fallback"
if auto_repair_fallback_applied
else None
),
evidence_note=(
"auto_repair_executions is empty, but apply/verifier/KM/Telegram "
"receipts prove a controlled apply execution chain."
if auto_repair_fallback_applied
else None
),
next_action_if_missing=(
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
),
),
_trace_stage(
stage_id="post_apply_verifier",
@@ -734,6 +829,33 @@ def _build_log_integration_taxonomy(
operation_total = sum(_trace_total(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
operation_recent = sum(_trace_recent(operation_summary, item) for item in _EXECUTOR_OPERATION_TYPES)
auto_repair_total = _trace_total(auto_repair_summary)
auto_repair_recent = _trace_recent(auto_repair_summary)
auto_repair_fallback_total = (
_controlled_apply_receipt_chain_fallback_total(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_total <= 0
else 0
)
auto_repair_fallback_recent = (
_controlled_apply_receipt_chain_fallback_recent(
operation_summary=operation_summary,
verifier_summary=verifier_summary,
km_summary=km_summary,
telegram_summary=telegram_summary,
)
if auto_repair_fallback_total > 0
else 0
)
auto_repair_source_total = max(auto_repair_total, auto_repair_fallback_total)
auto_repair_source_recent = max(auto_repair_recent, auto_repair_fallback_recent)
auto_repair_fallback_applied = (
auto_repair_total <= 0 and auto_repair_fallback_total > 0
)
source_families = [
{
"source_family_id": "mcp_gateway_tool_calls",
@@ -798,15 +920,34 @@ def _build_log_integration_taxonomy(
},
{
"source_family_id": "auto_repair_receipts",
"source_tables": ["auto_repair_executions"],
"source_tables": (
[
"auto_repair_executions",
"automation_operation_log:ansible_apply_executed",
"incident_evidence.post_execution_state",
"knowledge_entries",
"awooop_outbound_message",
]
if auto_repair_fallback_applied
else ["auto_repair_executions"]
),
"normalized_event_schema": "RepairExecutionReceipt",
"label_dimensions": ["incident", "service", "playbook", "risk", "result"],
"total": _trace_total(auto_repair_summary),
"recent": _trace_recent(auto_repair_summary),
"total": auto_repair_source_total,
"recent": auto_repair_source_recent,
"feeds_learning": True,
"public_safe": True,
"raw_payload_policy": "execution_step_refs_not_raw_secrets",
"next_action_if_empty": "write_auto_repair_execution_receipt_after_apply",
"record_quality": (
"controlled_apply_receipt_chain_fallback"
if auto_repair_fallback_applied
else "recorded"
if auto_repair_source_total > 0
else "missing"
),
"next_action_if_empty": (
"record_auto_repair_execution_or_complete_apply_verifier_km_telegram_chain"
),
},
{
"source_family_id": "post_apply_verifier",

View File

@@ -1426,6 +1426,55 @@ def test_runtime_receipt_recovery_flags_zero_live_log_events():
assert readback["work_item_progress"]["rollups"]["not_started_count"] == 10
def test_trace_ledger_uses_controlled_apply_chain_for_auto_repair_receipt_gap():
readback = build_runtime_receipt_readback_from_rows(
project_id="awoooi",
db_read_status="ok",
operation_count_rows=[
{"operation_type": "ansible_candidate_matched", "total": 1, "recent": 1},
{"operation_type": "ansible_check_mode_executed", "total": 1, "recent": 1},
{"operation_type": "ansible_apply_executed", "total": 1, "recent": 1},
],
verifier_count_rows=[
{"verification_result": "success", "total": 1, "recent": 1},
],
km_count_rows=[
{"status": "review", "total": 1, "recent": 1},
],
telegram_count_rows=[
{"send_status": "sent", "total": 1, "recent": 1},
],
)
trace = readback["trace_ledger"]
auto_repair_stage = next(
stage
for stage in trace["stages"]
if stage["stage_id"] == "auto_repair_execution_receipt"
)
assert auto_repair_stage["recorded"] is True
assert auto_repair_stage["record_quality"] == (
"controlled_apply_receipt_chain_fallback"
)
assert "auto_repair_execution_receipt" not in trace["missing_required_stage_ids"]
progress_items = {
item["work_item_id"]: item
for item in readback["work_item_progress"]["ordered_items"]
}
assert progress_items["P0-B-trace-ledger"]["status"] == "completed"
auto_repair_source = next(
item
for item in readback["log_integration_taxonomy"]["source_families"]
if item["source_family_id"] == "auto_repair_receipts"
)
assert auto_repair_source["total"] == 1
assert auto_repair_source["record_quality"] == (
"controlled_apply_receipt_chain_fallback"
)
def test_runtime_receipt_recovery_keeps_partial_live_log_events_visible():
readback = build_runtime_receipt_readback_from_rows(
project_id="awoooi",