wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent): keep closure lanes moving safely

Browse Source
This commit is contained in:
ogt
2026-07-11 16:09:31 +08:00
parent c4a77a1e51
commit bd27d4e759
6 changed files with 1675 additions and 319 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
apps/api/src/jobs/awooop_ansible_candidate_backfill_job.py
View File

@@ -25,6 +25,7 @@ from src.db.base import get_db_context
from src.services.awooop_ansible_audit_service import (
build_ansible_decision_audit_payload,
record_ansible_decision_audit,
verified_ansible_candidate_terminal_sql,
)
from src.services.awooop_ansible_check_mode_service import (
preflight_failed_apply_retry_queue_once,
@@ -63,10 +64,13 @@ async def _fetch_missing_candidate_incidents(
window_hours: int,
scan_limit: int,
) -> list[dict[str, Any]]:
terminal_candidate_sql = verified_ansible_candidate_terminal_sql(
"existing"
)
async with get_db_context(project_id) as db:
await db.execute(text("SET LOCAL statement_timeout = '5000ms'"))
result = await db.execute(
text("""
text(f"""
SELECT
incident_id,
project_id,
@@ -107,13 +111,25 @@ async def _fetch_missing_candidate_incidents(
AND existing.input ->> 'executor' = 'ansible'
AND existing.input ->> 'decision_path' = 'repair_candidate_controlled_queue'
AND coalesce(existing.incident_id::text, existing.input ->> 'incident_id') = incidents.incident_id::text
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log terminal
WHERE terminal.parent_op_id = existing.op_id
AND terminal.operation_type
= 'ansible_execution_skipped'
AND existing.op_id = (
SELECT latest.op_id
FROM automation_operation_log latest
WHERE latest.operation_type
= 'ansible_candidate_matched'
AND latest.created_at >= NOW() - (
:window_hours * INTERVAL '1 hour'
)
AND latest.input ->> 'executor' = 'ansible'
AND latest.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
AND coalesce(
latest.incident_id::text,
latest.input ->> 'incident_id'
) = incidents.incident_id::text
ORDER BY latest.created_at DESC, latest.op_id DESC
LIMIT 1
)
AND NOT ({terminal_candidate_sql})
)
ORDER BY created_at DESC
LIMIT :scan_limit
@@ -843,7 +859,6 @@ async def enqueue_missing_ansible_candidates_once(
)
if stats["failed_apply_retry_priority_tick"]:
logger.info("awooop_ansible_failed_apply_retry_priority_tick", **stats)
return stats
if receipt_backfiller is not None:
receipt_stats = await receipt_backfiller(

173
apps/api/src/services/awooop_ansible_audit_service.py
View File

@@ -9,6 +9,7 @@ from __future__ import annotations
import json
import os
import re
from typing import Any
from uuid import UUID, uuid4
@@ -35,6 +36,152 @@ ANSIBLE_OPERATION_TYPES = frozenset({
"ansible_execution_skipped",
})
def verified_ansible_candidate_terminal_sql(candidate_alias: str) -> str:
"""Return the shared SQL predicate for a safely terminal candidate."""
if re.fullmatch(r"[a-z_][a-z0-9_]*", candidate_alias) is None:
raise ValueError("invalid_candidate_sql_alias")
candidate = candidate_alias
incident_id = (
f"coalesce({candidate}.incident_id::text, "
f"{candidate}.input ->> 'incident_id')"
)
automation_run_id = (
f"coalesce(nullif({candidate}.input ->> 'automation_run_id', ''), "
f"{candidate}.op_id::text)"
)
template = """
EXISTS (
SELECT 1
FROM automation_operation_log direct_terminal
WHERE direct_terminal.parent_op_id = __CANDIDATE__.op_id
AND direct_terminal.operation_type = 'ansible_execution_skipped'
AND direct_terminal.status = 'dry_run'
AND direct_terminal.dry_run_result ->> 'skipped' = 'true'
AND coalesce(
direct_terminal.dry_run_result ->> 'apply_executed',
'false'
) = 'false'
)
OR EXISTS (
SELECT 1
FROM automation_operation_log check_mode
JOIN automation_operation_log apply
ON apply.parent_op_id = check_mode.op_id
AND apply.operation_type = 'ansible_apply_executed'
AND apply.status = 'failed'
JOIN automation_operation_log replay
ON replay.parent_op_id = apply.op_id
AND replay.operation_type = 'ansible_execution_skipped'
AND replay.status IN ('success', 'failed')
AND replay.input ->> 'execution_mode'
= 'controlled_retry_check_mode_replay'
AND replay.output ->> 'runtime_apply_executed' = 'false'
WHERE check_mode.parent_op_id = __CANDIDATE__.op_id
AND check_mode.operation_type = 'ansible_check_mode_executed'
AND check_mode.status = 'success'
AND check_mode.input ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND coalesce(
check_mode.incident_id::text,
check_mode.input ->> 'incident_id'
) = __INCIDENT_ID__
AND apply.input ->> 'automation_run_id' = __AUTOMATION_RUN_ID__
AND coalesce(
apply.incident_id::text,
apply.input ->> 'incident_id'
) = __INCIDENT_ID__
AND replay.input ->> 'automation_run_id' = __AUTOMATION_RUN_ID__
AND coalesce(
replay.incident_id::text,
replay.input ->> 'incident_id'
) = __INCIDENT_ID__
AND replay.output ->> 'terminal_disposition' = CASE
WHEN replay.status = 'success'
THEN 'no_write_replay_passed_waiting_repair_candidate'
ELSE 'no_write_replay_failed_waiting_playbook_or_transport_repair'
END
AND EXISTS (
SELECT 1
FROM incident_evidence verifier
WHERE verifier.incident_id = __INCIDENT_ID__
AND verifier.post_execution_state ->> 'apply_op_id'
= apply.op_id::text
AND verifier.verification_result IN ('failed', 'timeout')
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) receipt(value)
WHERE receipt.value ->> 'stage_id' = 'retry_or_rollback'
AND receipt.value ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND receipt.value #>> '{detail,failed_apply_op_id}'
= apply.op_id::text
AND receipt.value #>> '{detail,retry_check_mode_op_id}'
= replay.op_id::text
AND receipt.value #>> '{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND receipt.value #>> '{detail,verified_no_write_terminal}'
= 'true'
AND receipt.value #>> '{detail,runtime_apply_executed}'
= 'false'
AND receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM jsonb_array_elements(
coalesce(
apply.input -> 'runtime_stage_receipts',
'[]'::jsonb
)
) closure_receipt(value)
WHERE closure_receipt.value ->> 'stage_id'
= 'incident_closure'
AND closure_receipt.value ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
AND closure_receipt.value #>> '{detail,apply_op_id}'
= apply.op_id::text
AND closure_receipt.value #>> '{detail,retry_op_id}'
= replay.op_id::text
AND closure_receipt.value #>> '{detail,terminal_type}'
= replay.output ->> 'terminal_disposition'
AND closure_receipt.value #>>
'{detail,no_write_terminal}' = 'true'
AND closure_receipt.value #>>
'{detail,proposal_executed}' = 'true'
AND closure_receipt.value #>>
'{detail,execution_success}' = 'false'
AND closure_receipt.value #>>
'{detail,telegram_receipt_acknowledged}' = 'true'
AND closure_receipt.value #>>
'{detail,repository_readback_verified}' = 'true'
AND closure_receipt.value ->> 'durable_receipt' = 'true'
)
AND EXISTS (
SELECT 1
FROM alert_operation_log lifecycle
WHERE lifecycle.incident_id = __INCIDENT_ID__
AND lifecycle.event_type::text = 'EXECUTION_COMPLETED'
AND lifecycle.success IS FALSE
AND lifecycle.context ->> 'apply_op_id'
= apply.op_id::text
AND lifecycle.context ->> 'automation_run_id'
= __AUTOMATION_RUN_ID__
)
)
"""
return (
template.replace("__CANDIDATE__", candidate)
.replace("__INCIDENT_ID__", incident_id)
.replace("__AUTOMATION_RUN_ID__", automation_run_id)
)
_CATALOG: tuple[dict[str, Any], ...] = (
{
"catalog_id": "ansible:awoooi-auto-repair-canary",
@@ -730,6 +877,9 @@ async def record_ansible_decision_audit(
except (TypeError, ValueError):
candidate_op_id = str(uuid4())
payload["input"]["automation_run_id"] = candidate_op_id
terminal_candidate_sql = verified_ansible_candidate_terminal_sql(
"candidate"
)
try:
async with get_db_context(str(project_id)) as db:
await db.execute(
@@ -741,7 +891,7 @@ async def record_ansible_decision_audit(
{"idempotency_key": payload["input"]["idempotency_key"]},
)
existing = await db.execute(
text("""
text(f"""
SELECT candidate.op_id
FROM automation_operation_log candidate
WHERE candidate.operation_type = 'ansible_candidate_matched'
@@ -752,13 +902,22 @@ async def record_ansible_decision_audit(
AND candidate.input ->> 'executor' = 'ansible'
AND candidate.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
AND NOT EXISTS (
SELECT 1
FROM automation_operation_log terminal
WHERE terminal.parent_op_id = candidate.op_id
AND terminal.operation_type
= 'ansible_execution_skipped'
AND candidate.op_id = (
SELECT latest.op_id
FROM automation_operation_log latest
WHERE latest.operation_type
= 'ansible_candidate_matched'
AND coalesce(
latest.incident_id::text,
latest.input ->> 'incident_id'
) = :incident_id
AND latest.input ->> 'executor' = 'ansible'
AND latest.input ->> 'decision_path'
= 'repair_candidate_controlled_queue'
ORDER BY latest.created_at DESC, latest.op_id DESC
LIMIT 1
)
AND NOT ({terminal_candidate_sql})
LIMIT 1
"""),
{"incident_id": incident_id},

1287
apps/api/src/services/awooop_ansible_check_mode_service.py
View File

File diff suppressed because it is too large Load Diff