fix(alerts): close controlled canary lifecycle
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m33s
CD Pipeline / build-and-deploy (push) Successful in 6m43s
CD Pipeline / post-deploy-checks (push) Successful in 1m48s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 50s

This commit is contained in:
ogt
2026-07-11 11:46:08 +08:00
parent 46d2fe0c27
commit 847a6ed0d5
10 changed files with 528 additions and 45 deletions

View File

@@ -21,6 +21,7 @@ from src.services.awooop_ansible_check_mode_service import (
_EXECUTION_CAPABILITY_FALLBACK_OPERATION_TYPE,
AnsibleCheckModeClaim,
AnsibleRunResult,
_append_alert_lifecycle_receipt,
_append_runtime_stage_receipts_to_apply,
_automation_operation_log_incident_id,
_build_auto_repair_execution_receipt,
@@ -1394,6 +1395,118 @@ def test_ansible_decision_audit_payload_marks_repair_candidate_queue_claimable()
assert payload["input"]["execution_priority"] == 0
def test_ansible_canary_candidate_preserves_approval_and_controlled_apply() -> None:
approval_id = "00000000-0000-0000-0000-000000000044"
incident = SimpleNamespace(
incident_id="INC-CONTROLLED-CANARY",
project_id="awoooi",
alert_category="infrastructure",
notification_type="TYPE-3",
severity=SimpleNamespace(value="P2"),
affected_services=["awoooi-auto-repair-canary"],
signals=[
SimpleNamespace(
alert_name="AwoooPAutoRepairCanaryT16",
labels={
"alertname": "AwoooPAutoRepairCanaryT16",
"deployment": "awoooi-auto-repair-canary",
"namespace": "awoooi-prod",
},
annotations={},
)
],
)
payload = build_ansible_decision_audit_payload(
incident=incident,
proposal_data={
"source": "alert_webhook_controlled_router",
"risk_level": "medium",
"action": "awoooi-auto-repair-canary",
"approval_id": approval_id,
},
decision_path="repair_candidate_controlled_queue",
not_used_reason="queue bounded no-traffic canary",
)
assert payload is not None
assert payload["input"]["approval_id"] == approval_id
candidate = payload["input"]["executor_candidates"][0]
assert candidate["catalog_id"] == "ansible:awoooi-auto-repair-canary"
assert candidate["inventory_hosts"] == ["host_121"]
assert candidate["approval_required"] is False
claim = build_ansible_check_mode_claim_input(
source_candidate_op_id="00000000-0000-0000-0000-000000000045",
candidate_input=payload["input"],
)
assert claim["approval_id"] == approval_id
assert claim["controlled_apply_allowed"] is True
assert claim["apply_playbook_path"] == (
"infra/ansible/playbooks/awoooi-auto-repair-canary.yml"
)
@pytest.mark.asyncio
async def test_alert_lifecycle_receipt_uses_same_run_and_project_context(
monkeypatch: pytest.MonkeyPatch,
) -> None:
from src.repositories import alert_operation_log_repository as repo_module
captured: dict[str, object] = {}
class FakeRepository:
async def append(self, event_type: str, **kwargs):
captured["event_type"] = event_type
captured.update(kwargs)
return SimpleNamespace(id="receipt-1")
monkeypatch.setattr(
repo_module,
"get_alert_operation_log_repository",
lambda: FakeRepository(),
)
claim = AnsibleCheckModeClaim(
op_id="00000000-0000-0000-0000-000000000046",
source_candidate_op_id="00000000-0000-0000-0000-000000000047",
incident_id="INC-CONTROLLED-CANARY",
catalog_id="ansible:awoooi-auto-repair-canary",
playbook_path="infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
apply_playbook_path="infra/ansible/playbooks/awoooi-auto-repair-canary.yml",
inventory_hosts=("host_121",),
risk_level="low",
input_payload={
"automation_run_id": "00000000-0000-0000-0000-000000000047",
"approval_id": "00000000-0000-0000-0000-000000000048",
},
)
written = await _append_alert_lifecycle_receipt(
claim,
"EXECUTION_STARTED",
apply_op_id="00000000-0000-0000-0000-000000000049",
success=True,
action_detail="controlled_apply_started",
project_id="awoooi",
)
assert written is True
assert captured["event_type"] == "EXECUTION_STARTED"
assert captured["project_id"] == "awoooi"
assert captured["approval_id"] == "00000000-0000-0000-0000-000000000048"
assert captured["context"]["automation_run_id"] == (
"00000000-0000-0000-0000-000000000047"
)
def test_controlled_apply_bridges_terminal_projection_and_telegram_receipt() -> None:
source = inspect.getsource(run_controlled_apply_for_claim)
assert '"EXECUTION_STARTED"' in source
assert '"EXECUTION_COMPLETED"' in source
assert '"TELEGRAM_RESULT_SENT"' in source
assert "_finalize_controlled_approval_projection" in source
def test_ansible_decision_audit_payload_exposes_check_mode_safety_flags(
monkeypatch,
) -> None:

View File

@@ -39,11 +39,11 @@ def test_telegram_alert_ai_automation_matrix_loader_returns_gap_matrix():
assert summary["ops_script_direct_send_gap_count"] == 0
assert summary["api_direct_send_gap_count"] == 0
assert summary["gateway_normalized_callsite_count"] >= 50
assert summary["db_or_log_receipt_ready_surface_count"] >= 2
assert summary["ai_route_ready_surface_count"] >= 3
assert summary["controlled_queue_ready_surface_count"] >= 2
assert summary["post_verifier_ready_surface_count"] >= 3
assert summary["learning_writeback_ready_surface_count"] >= 3
assert summary["db_or_log_receipt_ready_surface_count"] == 9
assert summary["ai_route_ready_surface_count"] == 9
assert summary["controlled_queue_ready_surface_count"] == 9
assert summary["post_verifier_ready_surface_count"] == 9
assert summary["learning_writeback_ready_surface_count"] == 9
assert summary["manual_default_gap_count"] == 0
assert summary["direct_gap_migration_candidate_count"] == 0
assert summary["owner_acceptance_candidate_count"] == 0
@@ -78,16 +78,34 @@ def test_telegram_alert_ai_automation_matrix_loader_returns_gap_matrix():
assert matrix["telegram_direct_bot_api_egress_gap"]["db_or_log_receipt"] == (
"ready_no_direct_routes_remaining"
)
assert matrix["telegram_direct_bot_api_egress_gap"]["controlled_queue"] == (
"ready_not_applicable_no_direct_migration_queue_needed"
)
assert matrix["telegram_direct_bot_api_egress_gap"]["post_verifier"] == (
"ready_metadata_inventory_clear"
)
assert matrix["telegram_egress_controlled_migration_candidates"]["status"] == (
"controlled_migration_complete_no_candidates"
)
assert matrix["telegram_action_required_digest_policy"]["known_item_count"] == 8
assert matrix["telegram_action_required_digest_policy"]["db_or_log_receipt"] == (
"ready_source_policy_receipt_no_runtime_write"
)
assert matrix["telegram_egress_controlled_migration_candidates"][
"controlled_queue"
] == "ready_not_applicable_no_direct_migration_queue_needed"
assert matrix["telegram_egress_controlled_migration_candidates"][
"post_verifier"
] == "ready_owner_acceptance_ledger_clear"
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
"db_or_log_receipt"
] == "ready_awooop_outbound_message_query"
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
"learning_writeback"
] == "ready_delivery_receipt_summary_and_learning_writeback_refs"
assert matrix["telegram_ai_alert_card_delivery_readback_endpoint"][
"controlled_queue"
] == "ready_not_applicable_read_only_query_no_send"
assert matrix["telegram_alert_learning_registry_readback"]["status"] == (
"registry_readback_present"
)

View File

@@ -1,8 +1,12 @@
from __future__ import annotations
import inspect
from types import SimpleNamespace
from src.api.v1.webhooks import _auto_repair_action_label
from src.api.v1.webhooks import (
_auto_repair_action_label,
_process_new_alert_background,
)
def test_auto_repair_action_label_includes_executed_steps() -> None:
@@ -26,3 +30,12 @@ def test_auto_repair_action_label_uses_target_when_steps_missing() -> None:
assert label == "auto_repair_playbook:PB-TEST api:awoooi-prod"
def test_fallback_candidate_enters_single_writer_controlled_queue() -> None:
source = inspect.getsource(_process_new_alert_background)
assert "repair_candidate_result.candidate_found" in source
assert "_controlled_ai_policy_allows(fallback_create.risk_level)" in source
assert "_controlled_handoff = await _try_auto_repair_background" in source
assert 'primary_responsibility = "AI_CONTROLLED_EXECUTOR"' in source
assert '"automation_run_id": (' in source