feat(ai): enforce single-writer automation router
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s

This commit is contained in:
ogt
2026-07-11 05:45:48 +08:00
parent 01605b4947
commit f453359e34
20 changed files with 1674 additions and 570 deletions

View File

@@ -1823,26 +1823,9 @@ class DecisionManager:
token.proposal_data["auto_approve_reason"] = auto_decision.reason_detail
await self._save_token(token)
try:
from src.services.awooop_ansible_audit_service import (
record_ansible_decision_audit as _record_ansible_decision_audit,
)
_fire_and_forget(
_record_ansible_decision_audit(
incident=incident,
proposal_data=token.proposal_data,
decision_path="auto_execute",
not_used_reason=(
"auto_execute selected existing executor path; "
"Ansible check-mode is not wired yet"
),
)
)
except Exception as _ansible_audit_err:
logger.debug("ansible_decision_audit_schedule_error", error=str(_ansible_audit_err))
# 觸發自動執行 (非阻塞)
# The active auto-execute path owns the durable controlled-queue
# receipt. Do not write an audit-only candidate here because it
# would race with, and suppress, the claimable broker candidate.
_fire_and_forget(
self._auto_execute(incident, token)
)
@@ -1890,13 +1873,148 @@ class DecisionManager:
return token
async def _auto_execute(self, incident: Incident, token: "DecisionToken") -> None:
"""
ADR-030 Phase 4: 自動執行已批准的操作
"""Route an approved AI decision to the single-writer executor queue."""
僅當 AutoApprovePolicy 判斷可自動執行時呼叫
執行後發 Telegram 結果通知 (統帥要求: 修復結果對應同一告警)
2026-04-09 Claude Sonnet 4.6 Asia/Taipei
proposal_data = token.proposal_data or {}
token.proposal_data = proposal_data
action = str(
proposal_data.get("kubectl_command")
or proposal_data.get("action")
or ""
).strip()
labels = incident.signals[0].labels if incident.signals else {}
host_type = str(labels.get("host_type") or "").lower()
if host_type == "bare_metal" and action.lower().startswith("kubectl"):
token.state = DecisionState.READY
proposal_data.update({
"auto_executed": False,
"direct_runtime_execution_performed": False,
"automation_state": "blocked_with_safe_next_action",
"blocked_reason": (
"host_type=bare_metal but the proposed action uses kubectl; "
"a host-scoped allowlisted PlayBook target is required"
),
"safe_next_action": "queue_host_scoped_playbook_candidate",
})
await self._save_token(token)
logger.warning(
"ai_decision_wrong_domain_blocked_before_queue",
incident_id=incident.incident_id,
host_type=host_type,
action=action[:120],
)
return
risk_value = proposal_data.get("risk_level")
risk_level = str(
getattr(risk_value, "value", risk_value) or ""
).strip().lower()
if risk_level == "critical":
token.state = DecisionState.READY
proposal_data.update({
"auto_executed": False,
"direct_runtime_execution_performed": False,
"automation_state": "critical_break_glass_required",
"blocked_reason": "critical_break_glass_required",
"safe_next_action": "open_bounded_break_glass_contract",
})
await self._save_token(token)
logger.warning(
"ai_decision_critical_break_glass_required",
incident_id=incident.incident_id,
)
return
if not action or proposal_data.get("suggested_action") == "NO_ACTION":
token.state = DecisionState.READY
proposal_data.update({
"auto_executed": False,
"direct_runtime_execution_performed": False,
"automation_state": "verified_no_write_terminal",
"blocked_reason": "no_runtime_action_selected",
})
await self._save_token(token)
return
try:
from src.jobs.awooop_ansible_candidate_backfill_job import (
enqueue_ai_decision_ansible_candidate,
)
handoff = await enqueue_ai_decision_ansible_candidate(
incident=incident,
proposal_data=proposal_data,
project_id=str(getattr(incident, "project_id", None) or "awoooi"),
)
except Exception as exc:
logger.warning(
"ai_decision_controlled_executor_queue_failed",
incident_id=incident.incident_id,
error_type=type(exc).__name__,
)
handoff = {
"schema_version": "ai_decision_controlled_executor_handoff_v1",
"status": "controlled_executor_queue_failed",
"queued": False,
"side_effect_performed": False,
"single_writer_executor": "awoooi-ansible-executor-broker",
"active_blockers": ["controlled_executor_queue_failed"],
}
queued = handoff.get("queued") is True
proposal_data.update({
"auto_executed": False,
"direct_runtime_execution_performed": False,
"approval_execution_invoked": False,
"check_mode_required_before_apply": True,
"single_writer_executor": "awoooi-ansible-executor-broker",
"controlled_executor_handoff": handoff,
"automation_run_id": str(handoff.get("automation_run_id") or ""),
"automation_state": (
"controlled_check_mode_queued"
if queued
else "blocked_with_safe_next_action"
),
"safe_next_action": (
"awooop_ansible_check_mode_worker_claims_candidate"
if queued
else "repair_candidate_backfill_or_playbook_coverage_gap"
),
})
if queued:
token.state = DecisionState.EXECUTING
token.error = None
proposal_data.pop("blocked_reason", None)
else:
token.state = DecisionState.READY
blockers = handoff.get("active_blockers") or [handoff.get("status")]
proposal_data["blocked_reason"] = ",".join(
str(value) for value in blockers if value
)
await self._save_token(token)
logger.info(
"ai_decision_controlled_executor_handoff",
incident_id=incident.incident_id,
status=handoff.get("status"),
queued=queued,
automation_run_id=handoff.get("automation_run_id"),
side_effect_performed=False,
)
async def _legacy_auto_execute_disabled(
self,
incident: Incident,
token: "DecisionToken",
) -> None:
"""
Superseded direct executor retained temporarily for bounded removal.
It is intentionally fail-closed and must never produce a side effect.
"""
raise RuntimeError(
"decision_manager_direct_execution_superseded_by_single_writer_broker"
)
action = token.proposal_data.get("kubectl_command", "") or token.proposal_data.get("action", "")
_alert_labels = incident.signals[0].labels if incident.signals else {}
_host_type_for_domain_guard = (_alert_labels.get("host_type") or "").lower()