fix(alerts): close controlled canary lifecycle
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m33s
CD Pipeline / build-and-deploy (push) Successful in 6m43s
CD Pipeline / post-deploy-checks (push) Successful in 1m48s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 50s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m33s
CD Pipeline / build-and-deploy (push) Successful in 6m43s
CD Pipeline / post-deploy-checks (push) Successful in 1m48s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 50s
This commit is contained in:
@@ -21,6 +21,7 @@ from src.services.awooop_ansible_check_mode_service import (
|
||||
_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE,
|
||||
AnsibleCheckModeClaim,
|
||||
AnsibleRunResult,
|
||||
_append_alert_lifecycle_receipt,
|
||||
_append_runtime_stage_receipts_to_apply,
|
||||
_automation_operation_log_incident_id,
|
||||
_build_auto_repair_execution_receipt,
|
||||
@@ -1394,6 +1395,118 @@ def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable()
|
||||
assert payload["input"]["execution_priority"] == 0
|
||||
|
||||
|
||||
def test_ansible_canary_candidate_preserves_approval_and_controlled_apply() -> None:
|
||||
approval_id = "00000000-0000-0000-0000-000000000044"
|
||||
incident = SimpleNamespace(
|
||||
incident_id="INC-CONTROLLED-CANARY",
|
||||
project_id="awoooi",
|
||||
alert_category="infrastructure",
|
||||
notification_type="TYPE-3",
|
||||
severity=SimpleNamespace(value="P2"),
|
||||
affected_services=["awoooi-auto-repair-canary"],
|
||||
signals=[
|
||||
SimpleNamespace(
|
||||
alert_name="AwoooPAutoRepairCanaryT16",
|
||||
labels={
|
||||
"alertname": "AwoooPAutoRepairCanaryT16",
|
||||
"deployment": "awoooi-auto-repair-canary",
|
||||
"namespace": "awoooi-prod",
|
||||
},
|
||||
annotations={},
|
||||
)
|
||||
],
|
||||
)
|
||||
|
||||
payload = build_ansible_decision_audit_payload(
|
||||
incident=incident,
|
||||
proposal_data={
|
||||
"source": "alert_webhook_controlled_router",
|
||||
"risk_level": "medium",
|
||||
"action": "awoooi-auto-repair-canary",
|
||||
"approval_id": approval_id,
|
||||
},
|
||||
decision_path="repair_candidate_controlled_queue",
|
||||
not_used_reason="queue bounded no-traffic canary",
|
||||
)
|
||||
|
||||
assert payload is not None
|
||||
assert payload["input"]["approval_id"] == approval_id
|
||||
candidate = payload["input"]["executor_candidates"][0]
|
||||
assert candidate["catalog_id"] == "ansible:awoooi-auto-repair-canary"
|
||||
assert candidate["inventory_hosts"] == ["host_121"]
|
||||
assert candidate["approval_required"] is False
|
||||
claim = build_ansible_check_mode_claim_input(
|
||||
source_candidate_op_id="00000000-0000-0000-0000-000000000045",
|
||||
candidate_input=payload["input"],
|
||||
)
|
||||
assert claim["approval_id"] == approval_id
|
||||
assert claim["controlled_apply_allowed"] is True
|
||||
assert claim["apply_playbook_path"] == (
|
||||
"infra/ansible/playbooks/awoooi-auto-repair-canary.yml"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_alert_lifecycle_receipt_uses_same_run_and_project_context(
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
from src.repositories import alert_operation_log_repository as repo_module
|
||||
|
||||
captured: dict[str, object] = {}
|
||||
|
||||
class FakeRepository:
|
||||
async def append(self, event_type: str, **kwargs):
|
||||
captured["event_type"] = event_type
|
||||
captured.update(kwargs)
|
||||
return SimpleNamespace(id="receipt-1")
|
||||
|
||||
monkeypatch.setattr(
|
||||
repo_module,
|
||||
"get_alert_operation_log_repository",
|
||||
lambda: FakeRepository(),
|
||||
)
|
||||
claim = AnsibleCheckModeClaim(
|
||||
op_id="00000000-0000-0000-0000-000000000046",
|
||||
source_candidate_op_id="00000000-0000-0000-0000-000000000047",
|
||||
incident_id="INC-CONTROLLED-CANARY",
|
||||
catalog_id="ansible:awoooi-auto-repair-canary",
|
||||
playbook_path="infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
|
||||
apply_playbook_path="infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
|
||||
inventory_hosts=("host_121",),
|
||||
risk_level="low",
|
||||
input_payload={
|
||||
"automation_run_id": "00000000-0000-0000-0000-000000000047",
|
||||
"approval_id": "00000000-0000-0000-0000-000000000048",
|
||||
},
|
||||
)
|
||||
|
||||
written = await _append_alert_lifecycle_receipt(
|
||||
claim,
|
||||
"EXECUTION_STARTED",
|
||||
apply_op_id="00000000-0000-0000-0000-000000000049",
|
||||
success=True,
|
||||
action_detail="controlled_apply_started",
|
||||
project_id="awoooi",
|
||||
)
|
||||
|
||||
assert written is True
|
||||
assert captured["event_type"] == "EXECUTION_STARTED"
|
||||
assert captured["project_id"] == "awoooi"
|
||||
assert captured["approval_id"] == "00000000-0000-0000-0000-000000000048"
|
||||
assert captured["context"]["automation_run_id"] == (
|
||||
"00000000-0000-0000-0000-000000000047"
|
||||
)
|
||||
|
||||
|
||||
def test_controlled_apply_bridges_terminal_projection_and_telegram_receipt() -> None:
|
||||
source = inspect.getsource(run_controlled_apply_for_claim)
|
||||
|
||||
assert '"EXECUTION_STARTED"' in source
|
||||
assert '"EXECUTION_COMPLETED"' in source
|
||||
assert '"TELEGRAM_RESULT_SENT"' in source
|
||||
assert "_finalize_controlled_approval_projection" in source
|
||||
|
||||
|
||||
def test_ansible_decision_audit_payload_exposes_check_mode_safety_flags(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
|
||||
@@ -39,11 +39,11 @@ def test_telegram_alert_ai_automation_matrix_loader_returns_gap_matrix():
|
||||
assert summary["ops_script_direct_send_gap_count"] == 0
|
||||
assert summary["api_direct_send_gap_count"] == 0
|
||||
assert summary["gateway_normalized_callsite_count"] >= 50
|
||||
assert summary["db_or_log_receipt_ready_surface_count"] >= 2
|
||||
assert summary["ai_route_ready_surface_count"] >= 3
|
||||
assert summary["controlled_queue_ready_surface_count"] >= 2
|
||||
assert summary["post_verifier_ready_surface_count"] >= 3
|
||||
assert summary["learning_writeback_ready_surface_count"] >= 3
|
||||
assert summary["db_or_log_receipt_ready_surface_count"] == 9
|
||||
assert summary["ai_route_ready_surface_count"] == 9
|
||||
assert summary["controlled_queue_ready_surface_count"] == 9
|
||||
assert summary["post_verifier_ready_surface_count"] == 9
|
||||
assert summary["learning_writeback_ready_surface_count"] == 9
|
||||
assert summary["manual_default_gap_count"] == 0
|
||||
assert summary["direct_gap_migration_candidate_count"] == 0
|
||||
assert summary["owner_acceptance_candidate_count"] == 0
|
||||
@@ -78,16 +78,34 @@ def test_telegram_alert_ai_automation_matrix_loader_returns_gap_matrix():
|
||||
assert matrix["telegram_direct_bot_api_egress_gap"]["db_or_log_receipt"] == (
|
||||
"ready_no_direct_routes_remaining"
|
||||
)
|
||||
assert matrix["telegram_direct_bot_api_egress_gap"]["controlled_queue"] == (
|
||||
"ready_not_applicable_no_direct_migration_queue_needed"
|
||||
)
|
||||
assert matrix["telegram_direct_bot_api_egress_gap"]["post_verifier"] == (
|
||||
"ready_metadata_inventory_clear"
|
||||
)
|
||||
assert matrix["telegram_egress_controlled_migration_candidates"]["status"] == (
|
||||
"controlled_migration_complete_no_candidates"
|
||||
)
|
||||
assert matrix["telegram_action_required_digest_policy"]["known_item_count"] == 8
|
||||
assert matrix["telegram_action_required_digest_policy"]["db_or_log_receipt"] == (
|
||||
"ready_source_policy_receipt_no_runtime_write"
|
||||
)
|
||||
assert matrix["telegram_egress_controlled_migration_candidates"][
|
||||
"controlled_queue"
|
||||
] == "ready_not_applicable_no_direct_migration_queue_needed"
|
||||
assert matrix["telegram_egress_controlled_migration_candidates"][
|
||||
"post_verifier"
|
||||
] == "ready_owner_acceptance_ledger_clear"
|
||||
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
|
||||
"db_or_log_receipt"
|
||||
] == "ready_awooop_outbound_message_query"
|
||||
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
|
||||
"learning_writeback"
|
||||
] == "ready_delivery_receipt_summary_and_learning_writeback_refs"
|
||||
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
|
||||
"controlled_queue"
|
||||
] == "ready_not_applicable_read_only_query_no_send"
|
||||
assert matrix["telegram_alert_learning_registry_readback"]["status"] == (
|
||||
"registry_readback_present"
|
||||
)
|
||||
|
||||
@@ -1,8 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
from types import SimpleNamespace
|
||||
|
||||
from src.api.v1.webhooks import _auto_repair_action_label
|
||||
from src.api.v1.webhooks import (
|
||||
_auto_repair_action_label,
|
||||
_process_new_alert_background,
|
||||
)
|
||||
|
||||
|
||||
def test_auto_repair_action_label_includes_executed_steps() -> None:
|
||||
@@ -26,3 +30,12 @@ def test_auto_repair_action_label_uses_target_when_steps_missing() -> None:
|
||||
|
||||
assert label == "auto_repair_playbook:PB-TEST api:awoooi-prod"
|
||||
|
||||
|
||||
def test_fallback_candidate_enters_single_writer_controlled_queue() -> None:
|
||||
source = inspect.getsource(_process_new_alert_background)
|
||||
|
||||
assert "repair_candidate_result.candidate_found" in source
|
||||
assert "_controlled_ai_policy_allows(fallback_create.risk_level)" in source
|
||||
assert "_controlled_handoff = await _try_auto_repair_background" in source
|
||||
assert 'primary_responsibility = "AI_CONTROLLED_EXECUTOR"' in source
|
||||
assert '"automation_run_id": (' in source
|
||||
|
||||
Reference in New Issue
Block a user