feat(ai): enforce single-writer automation router
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m21s
CD Pipeline / build-and-deploy (push) Successful in 6m53s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 49s
This commit is contained in:
@@ -1823,26 +1823,9 @@ class DecisionManager:
|
||||
token.proposal_data["auto_approve_reason"] = auto_decision.reason_detail
|
||||
await self._save_token(token)
|
||||
|
||||
try:
|
||||
from src.services.awooop_ansible_audit_service import (
|
||||
record_ansible_decision_audit as _record_ansible_decision_audit,
|
||||
)
|
||||
|
||||
_fire_and_forget(
|
||||
_record_ansible_decision_audit(
|
||||
incident=incident,
|
||||
proposal_data=token.proposal_data,
|
||||
decision_path="auto_execute",
|
||||
not_used_reason=(
|
||||
"auto_execute selected existing executor path; "
|
||||
"Ansible check-mode is not wired yet"
|
||||
),
|
||||
)
|
||||
)
|
||||
except Exception as _ansible_audit_err:
|
||||
logger.debug("ansible_decision_audit_schedule_error", error=str(_ansible_audit_err))
|
||||
|
||||
# 觸發自動執行 (非阻塞)
|
||||
# The active auto-execute path owns the durable controlled-queue
|
||||
# receipt. Do not write an audit-only candidate here because it
|
||||
# would race with, and suppress, the claimable broker candidate.
|
||||
_fire_and_forget(
|
||||
self._auto_execute(incident, token)
|
||||
)
|
||||
@@ -1890,13 +1873,148 @@ class DecisionManager:
|
||||
return token
|
||||
|
||||
async def _auto_execute(self, incident: Incident, token: "DecisionToken") -> None:
|
||||
"""
|
||||
ADR-030 Phase 4: 自動執行已批准的操作
|
||||
"""Route an approved AI decision to the single-writer executor queue."""
|
||||
|
||||
僅當 AutoApprovePolicy 判斷可自動執行時呼叫
|
||||
執行後發 Telegram 結果通知 (統帥要求: 修復結果對應同一告警)
|
||||
2026-04-09 Claude Sonnet 4.6 Asia/Taipei
|
||||
proposal_data = token.proposal_data or {}
|
||||
token.proposal_data = proposal_data
|
||||
action = str(
|
||||
proposal_data.get("kubectl_command")
|
||||
or proposal_data.get("action")
|
||||
or ""
|
||||
).strip()
|
||||
labels = incident.signals[0].labels if incident.signals else {}
|
||||
host_type = str(labels.get("host_type") or "").lower()
|
||||
|
||||
if host_type == "bare_metal" and action.lower().startswith("kubectl"):
|
||||
token.state = DecisionState.READY
|
||||
proposal_data.update({
|
||||
"auto_executed": False,
|
||||
"direct_runtime_execution_performed": False,
|
||||
"automation_state": "blocked_with_safe_next_action",
|
||||
"blocked_reason": (
|
||||
"host_type=bare_metal but the proposed action uses kubectl; "
|
||||
"a host-scoped allowlisted PlayBook target is required"
|
||||
),
|
||||
"safe_next_action": "queue_host_scoped_playbook_candidate",
|
||||
})
|
||||
await self._save_token(token)
|
||||
logger.warning(
|
||||
"ai_decision_wrong_domain_blocked_before_queue",
|
||||
incident_id=incident.incident_id,
|
||||
host_type=host_type,
|
||||
action=action[:120],
|
||||
)
|
||||
return
|
||||
|
||||
risk_value = proposal_data.get("risk_level")
|
||||
risk_level = str(
|
||||
getattr(risk_value, "value", risk_value) or ""
|
||||
).strip().lower()
|
||||
if risk_level == "critical":
|
||||
token.state = DecisionState.READY
|
||||
proposal_data.update({
|
||||
"auto_executed": False,
|
||||
"direct_runtime_execution_performed": False,
|
||||
"automation_state": "critical_break_glass_required",
|
||||
"blocked_reason": "critical_break_glass_required",
|
||||
"safe_next_action": "open_bounded_break_glass_contract",
|
||||
})
|
||||
await self._save_token(token)
|
||||
logger.warning(
|
||||
"ai_decision_critical_break_glass_required",
|
||||
incident_id=incident.incident_id,
|
||||
)
|
||||
return
|
||||
|
||||
if not action or proposal_data.get("suggested_action") == "NO_ACTION":
|
||||
token.state = DecisionState.READY
|
||||
proposal_data.update({
|
||||
"auto_executed": False,
|
||||
"direct_runtime_execution_performed": False,
|
||||
"automation_state": "verified_no_write_terminal",
|
||||
"blocked_reason": "no_runtime_action_selected",
|
||||
})
|
||||
await self._save_token(token)
|
||||
return
|
||||
|
||||
try:
|
||||
from src.jobs.awooop_ansible_candidate_backfill_job import (
|
||||
enqueue_ai_decision_ansible_candidate,
|
||||
)
|
||||
|
||||
handoff = await enqueue_ai_decision_ansible_candidate(
|
||||
incident=incident,
|
||||
proposal_data=proposal_data,
|
||||
project_id=str(getattr(incident, "project_id", None) or "awoooi"),
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning(
|
||||
"ai_decision_controlled_executor_queue_failed",
|
||||
incident_id=incident.incident_id,
|
||||
error_type=type(exc).__name__,
|
||||
)
|
||||
handoff = {
|
||||
"schema_version": "ai_decision_controlled_executor_handoff_v1",
|
||||
"status": "controlled_executor_queue_failed",
|
||||
"queued": False,
|
||||
"side_effect_performed": False,
|
||||
"single_writer_executor": "awoooi-ansible-executor-broker",
|
||||
"active_blockers": ["controlled_executor_queue_failed"],
|
||||
}
|
||||
|
||||
queued = handoff.get("queued") is True
|
||||
proposal_data.update({
|
||||
"auto_executed": False,
|
||||
"direct_runtime_execution_performed": False,
|
||||
"approval_execution_invoked": False,
|
||||
"check_mode_required_before_apply": True,
|
||||
"single_writer_executor": "awoooi-ansible-executor-broker",
|
||||
"controlled_executor_handoff": handoff,
|
||||
"automation_run_id": str(handoff.get("automation_run_id") or ""),
|
||||
"automation_state": (
|
||||
"controlled_check_mode_queued"
|
||||
if queued
|
||||
else "blocked_with_safe_next_action"
|
||||
),
|
||||
"safe_next_action": (
|
||||
"awooop_ansible_check_mode_worker_claims_candidate"
|
||||
if queued
|
||||
else "repair_candidate_backfill_or_playbook_coverage_gap"
|
||||
),
|
||||
})
|
||||
if queued:
|
||||
token.state = DecisionState.EXECUTING
|
||||
token.error = None
|
||||
proposal_data.pop("blocked_reason", None)
|
||||
else:
|
||||
token.state = DecisionState.READY
|
||||
blockers = handoff.get("active_blockers") or [handoff.get("status")]
|
||||
proposal_data["blocked_reason"] = ",".join(
|
||||
str(value) for value in blockers if value
|
||||
)
|
||||
await self._save_token(token)
|
||||
logger.info(
|
||||
"ai_decision_controlled_executor_handoff",
|
||||
incident_id=incident.incident_id,
|
||||
status=handoff.get("status"),
|
||||
queued=queued,
|
||||
automation_run_id=handoff.get("automation_run_id"),
|
||||
side_effect_performed=False,
|
||||
)
|
||||
|
||||
async def _legacy_auto_execute_disabled(
|
||||
self,
|
||||
incident: Incident,
|
||||
token: "DecisionToken",
|
||||
) -> None:
|
||||
"""
|
||||
Superseded direct executor retained temporarily for bounded removal.
|
||||
|
||||
It is intentionally fail-closed and must never produce a side effect.
|
||||
"""
|
||||
raise RuntimeError(
|
||||
"decision_manager_direct_execution_superseded_by_single_writer_broker"
|
||||
)
|
||||
action = token.proposal_data.get("kubectl_command", "") or token.proposal_data.get("action", "")
|
||||
_alert_labels = incident.signals[0].labels if incident.signals else {}
|
||||
_host_type_for_domain_guard = (_alert_labels.get("host_type") or "").lower()
|
||||
|
||||
Reference in New Issue
Block a user