575 lines
21 KiB
Python
575 lines
21 KiB
Python
from __future__ import annotations
|
||
|
||
# ruff: noqa: E402
|
||
import json
|
||
import os
|
||
from pathlib import Path
|
||
from types import SimpleNamespace
|
||
|
||
os.environ.setdefault(
|
||
"DATABASE_URL",
|
||
"postgresql+asyncpg://test:test@localhost/test",
|
||
)
|
||
|
||
import pytest
|
||
from fastapi import FastAPI
|
||
from fastapi.testclient import TestClient
|
||
from pydantic import ValidationError
|
||
|
||
from src.api.v1 import webhooks as webhooks_api
|
||
from src.core.config import settings
|
||
from src.services.alert_chain_emergency_ingress import (
|
||
ALERT_CHAIN_AGENT99_ROLE,
|
||
ALERT_CHAIN_CANONICAL_ASSET_ID,
|
||
ALERT_CHAIN_CATALOG_ID,
|
||
ALERT_CHAIN_EXECUTOR,
|
||
AlertChainEmergencyRelayRequest,
|
||
_alert_chain_candidate_queued_message,
|
||
build_alert_chain_emergency_owner,
|
||
process_alert_chain_emergency_relay,
|
||
)
|
||
|
||
REPO_ROOT = Path(__file__).resolve().parents[3]
|
||
|
||
|
||
def payload() -> dict:
|
||
return {
|
||
"schema_version": "agent99_alertmanager_emergency_relay_v1",
|
||
"relay_receipt_id": f"agent99-alertchain-{'a' * 64}",
|
||
"source_host": "192.168.0.110",
|
||
"source_url": "http://192.168.0.110:9093/api/v2/alerts",
|
||
"source_fingerprint": "0123456789abcdef",
|
||
"source_started_at": "2026-07-16T13:40:20+08:00",
|
||
"alertname": "AlertChainBroken_Alertmanager",
|
||
"status": "firing",
|
||
"severity": "critical",
|
||
"target_resource": "alertmanager",
|
||
"namespace": "monitoring",
|
||
"agent99_role": "relay_coordination_only",
|
||
"executor": "host_ansible_executor",
|
||
"catalog_id": "ansible:110-alertmanager-delivery-recovery",
|
||
"runtime_write_performed": False,
|
||
"raw_payload_stored": False,
|
||
}
|
||
|
||
|
||
class FakeApprovalService:
|
||
def __init__(self) -> None:
|
||
self.approval = None
|
||
self.created_requests = []
|
||
self.incremented = 0
|
||
|
||
async def get_approval(self, approval_id):
|
||
if self.approval and str(self.approval.id) == str(approval_id):
|
||
return self.approval
|
||
return None
|
||
|
||
async def create_approval_with_fingerprint(self, *, request, fingerprint):
|
||
self.created_requests.append((request, fingerprint))
|
||
metadata = dict(request.metadata or {})
|
||
approval_id = metadata.pop("preallocated_approval_id")
|
||
self.approval = SimpleNamespace(
|
||
id=approval_id,
|
||
metadata=metadata,
|
||
incident_id=request.incident_id,
|
||
hit_count=1,
|
||
)
|
||
return self.approval
|
||
|
||
async def increment_hit_count(self, _approval_id):
|
||
self.incremented += 1
|
||
self.approval.hit_count += 1
|
||
return self.approval
|
||
|
||
async def update_incident_id(self, _approval_id, incident_id):
|
||
self.approval.incident_id = incident_id
|
||
|
||
|
||
class FakeIncident:
|
||
def __init__(self, incident_id: str) -> None:
|
||
self.incident_id = incident_id
|
||
self.persisted_to_pg = True
|
||
|
||
def model_dump(self, *, mode: str):
|
||
assert mode == "json"
|
||
return {
|
||
"incident_id": self.incident_id,
|
||
"status": "investigating",
|
||
"severity": "P2",
|
||
"affected_services": ["alertmanager"],
|
||
"signals": [
|
||
{
|
||
"alert_name": "AlertChainBroken_Alertmanager",
|
||
"labels": {
|
||
"alertname": "AlertChainBroken_Alertmanager",
|
||
"component": "alertmanager",
|
||
},
|
||
"annotations": {},
|
||
}
|
||
],
|
||
}
|
||
|
||
|
||
class FakeIncidentService:
|
||
def __init__(self) -> None:
|
||
self.incident = None
|
||
|
||
async def get_for_readback(self, incident_id, *, project_id):
|
||
assert project_id == "awoooi"
|
||
if self.incident and self.incident.incident_id == incident_id:
|
||
return self.incident
|
||
return None
|
||
|
||
|
||
def app_client() -> TestClient:
|
||
app = FastAPI()
|
||
app.include_router(webhooks_api.router, prefix="/api/v1")
|
||
return TestClient(app)
|
||
|
||
|
||
def test_relay_model_rejects_raw_content_and_executor_drift() -> None:
|
||
with pytest.raises(ValidationError):
|
||
AlertChainEmergencyRelayRequest.model_validate(
|
||
{**payload(), "raw_log": "must not enter machine ingress"}
|
||
)
|
||
with pytest.raises(ValidationError):
|
||
AlertChainEmergencyRelayRequest.model_validate(
|
||
{**payload(), "executor": "Agent99"}
|
||
)
|
||
with pytest.raises(ValidationError):
|
||
AlertChainEmergencyRelayRequest.model_validate(
|
||
{**payload(), "source_started_at": "2026-07-16T13:40:20"}
|
||
)
|
||
with pytest.raises(ValidationError):
|
||
AlertChainEmergencyRelayRequest.model_validate(
|
||
{**payload(), "source_host": "192.168.0.111"}
|
||
)
|
||
|
||
|
||
def test_agent99_poll_is_read_only_first_hop_and_reduced_authenticated_outbound() -> None:
|
||
source = (
|
||
REPO_ROOT / "agent99-alertmanager-alertchain-poll.ps1"
|
||
).read_text(
|
||
encoding="utf-8"
|
||
)
|
||
first_hop = source.split("$alerts = @(Invoke-RestMethod", 1)[1].split(
|
||
"} catch", 1
|
||
)[0]
|
||
|
||
assert "http://192.168.0.110:9093/api/v2/alerts" in source
|
||
assert '$sourceUri.Host -ne $CanonicalSourceHost' in source
|
||
assert '$sourceUri.Port -ne 9093' in source
|
||
assert '$sourceUri.AbsolutePath -ne "/api/v2/alerts"' in source
|
||
assert "-Method Get" in first_hop
|
||
assert "-Headers" not in first_hop
|
||
assert "token" not in first_hop.lower()
|
||
assert '$state -ne "active"' in source
|
||
assert "AlertChainBroken_Alertmanager" in source
|
||
assert 'Headers @{ "X-Agent99-Relay-Token" = $token }' in source
|
||
assert "agent99_alertmanager_emergency_relay_v1" in source
|
||
assert "source_host = $CanonicalSourceHost" in source
|
||
assert "source_url = $CanonicalSourceUrl" in source
|
||
assert 'agent99_role = "relay_coordination_only"' in source
|
||
assert 'executor = "host_ansible_executor"' in source
|
||
assert 'catalog_id = $CanonicalCatalogId' in source
|
||
assert "candidate_queued" in source
|
||
assert "runtime_write_performed = $false" in source
|
||
assert "raw_payload_stored = $false" in source
|
||
assert "dedupe-receipts" in source
|
||
assert "rawPayloadStored = $false" in source
|
||
assert "Start-AgentSreAlertInbox" not in source
|
||
assert "agent99-run.ps1" not in source
|
||
|
||
listener = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
assert "alertmanager-emergency" not in listener
|
||
assert "AlertmanagerPrefix" not in listener
|
||
|
||
|
||
def test_agent99_install_and_bootstrap_publish_poll_contract() -> None:
|
||
installer = (REPO_ROOT / "agent99-install-sre-alert-relay.ps1").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
bootstrap = (REPO_ROOT / "agent99-bootstrap.ps1").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
|
||
for source in (installer, bootstrap):
|
||
assert "http://192.168.0.110:9093/api/v2/alerts" in source
|
||
assert "pollIntervalSeconds" in source
|
||
assert "relay_coordination_only" in source
|
||
assert "host_ansible_executor" in source
|
||
assert "ansible:110-alertmanager-delivery-recovery" in source
|
||
assert '"alertmanagerPrefix":' not in source
|
||
assert 'Set-AgentProp $config.sreAlertRelay "alertmanagerPrefix"' not in source
|
||
|
||
relay_script = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
task_script = (REPO_ROOT / "agent99-register-tasks.ps1").read_text(
|
||
encoding="utf-8"
|
||
)
|
||
assert "Register-ObjectEvent" in relay_script
|
||
assert "in_process_nonblocking_timer" in relay_script
|
||
assert "$alertChainPollTimer.AutoReset = $false" in relay_script
|
||
assert "$pollData.Timer.Start()" in relay_script
|
||
assert "Unregister-Event -SourceIdentifier $alertChainPollEventId" in relay_script
|
||
assert "Start-Job" not in relay_script
|
||
assert "Wooo-Agent99-AlertChain-Poll" not in task_script
|
||
|
||
for config_name in (
|
||
"agent99.config.example.json",
|
||
"agent99.config.99.example.json",
|
||
):
|
||
config = json.loads((REPO_ROOT / config_name).read_text(encoding="utf-8"))
|
||
relay = config["sreAlertRelay"]
|
||
assert "alertmanagerPrefix" not in relay
|
||
poll = relay["alertChainPoll"]
|
||
assert poll["sourceUrl"] == (
|
||
"http://192.168.0.110:9093/api/v2/alerts"
|
||
)
|
||
assert poll["sourceHost"] == "192.168.0.110"
|
||
assert poll["pollIntervalSeconds"] == 60
|
||
assert poll["agent99Role"] == "relay_coordination_only"
|
||
assert poll["linuxExecutor"] == "host_ansible_executor"
|
||
assert poll["catalogId"] == (
|
||
"ansible:110-alertmanager-delivery-recovery"
|
||
)
|
||
assert poll["firstHopSecretTransmitted"] is False
|
||
assert poll["rawPayloadStored"] is False
|
||
|
||
|
||
def test_owner_identity_is_stable_and_binds_same_run_trace_work_item() -> None:
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
|
||
first = build_alert_chain_emergency_owner(request)
|
||
second = build_alert_chain_emergency_owner(request)
|
||
|
||
assert first == second
|
||
assert first["run_id"] == first["trace_id"]
|
||
assert first["work_item_id"].startswith("alertchain:awoooi:")
|
||
assert first["agent99_role"] == ALERT_CHAIN_AGENT99_ROLE
|
||
assert first["executor"] == ALERT_CHAIN_EXECUTOR
|
||
assert first["catalog_id"] == ALERT_CHAIN_CATALOG_ID
|
||
assert first["canonical_asset_id"] == ALERT_CHAIN_CANONICAL_ASSET_ID
|
||
|
||
|
||
def test_queue_receipt_message_is_compact_truthful_and_actionable() -> None:
|
||
owner = build_alert_chain_emergency_owner(
|
||
AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
)
|
||
|
||
message = _alert_chain_candidate_queued_message(owner=owner)
|
||
|
||
assert "[ALERT-CHAIN | CRITICAL]" in message
|
||
assert "ai_automation_alert_card_v1" in message
|
||
assert owner["incident_id"] in message
|
||
assert owner["run_id"] in message
|
||
assert "Agent99：已完成只讀旁路探測與驗證轉送" in message
|
||
assert "Host Ansible：已排入 check → bounded apply → verify" in message
|
||
assert "未介入此緊急路由" in message
|
||
assert "KM/RAG/MCP/PlayBook" in message
|
||
assert "建議下一步" not in message
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_ingress_persists_and_queues_exact_ansible_candidate() -> None:
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
owner = build_alert_chain_emergency_owner(request)
|
||
approvals = FakeApprovalService()
|
||
incidents = FakeIncidentService()
|
||
enqueue_calls = []
|
||
events = []
|
||
telegram_calls = []
|
||
|
||
async def create_incident(**kwargs):
|
||
assert kwargs["canonical_incident_id"] == owner["incident_id"]
|
||
assert kwargs["alertname"] == "AlertChainBroken_Alertmanager"
|
||
assert kwargs["namespace"] == "monitoring"
|
||
incidents.incident = FakeIncident(owner["incident_id"])
|
||
return owner["incident_id"]
|
||
|
||
async def enqueue(**kwargs):
|
||
enqueue_calls.append(kwargs)
|
||
assert kwargs["automation_run_id"] == owner["run_id"]
|
||
assert kwargs["incident"]["run_id"] == owner["run_id"]
|
||
assert kwargs["incident"]["trace_id"] == owner["trace_id"]
|
||
assert kwargs["incident"]["work_item_id"] == owner["work_item_id"]
|
||
assert kwargs["proposal_data"]["source_recurrence_verified"] is True
|
||
return {
|
||
"status": "controlled_check_mode_queued",
|
||
"automation_run_id": owner["run_id"],
|
||
"trace_id": owner["trace_id"],
|
||
"work_item_id": owner["work_item_id"],
|
||
"queued": True,
|
||
"side_effect_performed": False,
|
||
"active_blockers": [],
|
||
}
|
||
|
||
async def record_event(**kwargs):
|
||
events.append(kwargs)
|
||
|
||
async def notify_telegram(**kwargs):
|
||
telegram_calls.append(kwargs)
|
||
assert kwargs["owner"] == owner
|
||
assert kwargs["project_id"] == "awoooi"
|
||
return {
|
||
"attempted": True,
|
||
"durable_acknowledged": True,
|
||
"delivery_status": "sent",
|
||
"provider_message_id": "4431",
|
||
"destination_alias": "awoooi_sre_war_room",
|
||
}
|
||
|
||
result = await process_alert_chain_emergency_relay(
|
||
request,
|
||
approval_service=approvals,
|
||
incident_service=incidents,
|
||
incident_creator=create_incident,
|
||
candidate_enqueuer=enqueue,
|
||
event_recorder=record_event,
|
||
telegram_notifier=notify_telegram,
|
||
)
|
||
|
||
assert result["status"] == "ansible_candidate_queued_verifier_pending"
|
||
assert result["candidate_persisted"] is True
|
||
assert result["candidate_queued"] is True
|
||
assert result["identity"]["run_id"] == result["identity"]["trace_id"]
|
||
assert result["identity"]["work_item_id"] == owner["work_item_id"]
|
||
assert result["agent99_role"] == "relay_coordination_only"
|
||
assert result["executor"] == "host_ansible_executor"
|
||
assert result["runtime_write_performed"] is False
|
||
assert result["runtime_closure_verified"] is False
|
||
assert result["alert_to_telegram_receipt_verified"] is True
|
||
assert result["telegram_receipt"] == {
|
||
"schema_version": "alert_chain_telegram_queue_receipt_v1",
|
||
"attempted": True,
|
||
"durable_acknowledged": True,
|
||
"delivery_status": "sent",
|
||
"provider_message_id": "4431",
|
||
"destination_alias": "awoooi_sre_war_room",
|
||
"classification": "alert_chain_health",
|
||
"agent99_activity": "read_only_relay_completed",
|
||
"executor_activity": "host_ansible_candidate_queued",
|
||
"model_activity": "not_invoked_deterministic_policy",
|
||
}
|
||
assert len(enqueue_calls) == 1
|
||
assert len(telegram_calls) == 1
|
||
assert events[0]["stage"] == "secondary_ingress_ansible_candidate_queued"
|
||
|
||
repeated = await process_alert_chain_emergency_relay(
|
||
request,
|
||
approval_service=approvals,
|
||
incident_service=incidents,
|
||
incident_creator=create_incident,
|
||
candidate_enqueuer=enqueue,
|
||
event_recorder=record_event,
|
||
telegram_notifier=notify_telegram,
|
||
)
|
||
assert repeated["identity"] == result["identity"]
|
||
assert approvals.incremented == 1
|
||
assert len(telegram_calls) == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_ingress_keeps_persisted_candidate_partial_when_queue_blocks() -> None:
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
owner = build_alert_chain_emergency_owner(request)
|
||
approvals = FakeApprovalService()
|
||
incidents = FakeIncidentService()
|
||
|
||
async def create_incident(**_kwargs):
|
||
incidents.incident = FakeIncident(owner["incident_id"])
|
||
return owner["incident_id"]
|
||
|
||
async def blocked_enqueue(**_kwargs):
|
||
return {
|
||
"status": "pre_decision_evidence_not_verified",
|
||
"automation_run_id": owner["run_id"],
|
||
"trace_id": owner["trace_id"],
|
||
"work_item_id": owner["work_item_id"],
|
||
"queued": False,
|
||
"side_effect_performed": False,
|
||
"active_blockers": ["pre_decision_evidence_not_verified"],
|
||
}
|
||
|
||
async def no_event(**_kwargs):
|
||
return None
|
||
|
||
async def must_not_notify(**_kwargs):
|
||
raise AssertionError("telegram must not send before exact queue ack")
|
||
|
||
result = await process_alert_chain_emergency_relay(
|
||
request,
|
||
approval_service=approvals,
|
||
incident_service=incidents,
|
||
incident_creator=create_incident,
|
||
candidate_enqueuer=blocked_enqueue,
|
||
event_recorder=no_event,
|
||
telegram_notifier=must_not_notify,
|
||
)
|
||
|
||
assert result["candidate_persisted"] is True
|
||
assert result["candidate_queued"] is False
|
||
assert result["telegram_receipt"]["attempted"] is False
|
||
assert result["alert_to_telegram_receipt_verified"] is False
|
||
assert result["runtime_closure_verified"] is False
|
||
assert result["safe_next_action"].startswith("retry_same_relay_receipt")
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_queued_candidate_continues_when_telegram_receipt_is_missing() -> None:
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
owner = build_alert_chain_emergency_owner(request)
|
||
approvals = FakeApprovalService()
|
||
incidents = FakeIncidentService()
|
||
|
||
async def create_incident(**_kwargs):
|
||
incidents.incident = FakeIncident(owner["incident_id"])
|
||
return owner["incident_id"]
|
||
|
||
async def enqueue(**_kwargs):
|
||
return {
|
||
"status": "controlled_check_mode_queued",
|
||
"automation_run_id": owner["run_id"],
|
||
"trace_id": owner["trace_id"],
|
||
"work_item_id": owner["work_item_id"],
|
||
"queued": True,
|
||
"side_effect_performed": False,
|
||
"active_blockers": [],
|
||
}
|
||
|
||
async def record_event(**_kwargs):
|
||
return None
|
||
|
||
async def notify_without_durable_ack(**_kwargs):
|
||
return {
|
||
"attempted": True,
|
||
"durable_acknowledged": False,
|
||
"delivery_status": "durable_mirror_failed",
|
||
}
|
||
|
||
result = await process_alert_chain_emergency_relay(
|
||
request,
|
||
approval_service=approvals,
|
||
incident_service=incidents,
|
||
incident_creator=create_incident,
|
||
candidate_enqueuer=enqueue,
|
||
event_recorder=record_event,
|
||
telegram_notifier=notify_without_durable_ack,
|
||
)
|
||
|
||
assert result["status"] == (
|
||
"ansible_candidate_queued_telegram_receipt_pending"
|
||
)
|
||
assert result["candidate_queued"] is True
|
||
assert result["alert_to_telegram_receipt_verified"] is False
|
||
assert "telegram_durable_queue_receipt_missing" in result["active_blockers"]
|
||
assert result["safe_next_action"] == (
|
||
"ansible_worker_continue_same_run_and_retry_telegram_receipt"
|
||
)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_secondary_ingress_rejects_deterministic_owner_collision() -> None:
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
owner = build_alert_chain_emergency_owner(request)
|
||
approvals = FakeApprovalService()
|
||
approvals.approval = SimpleNamespace(
|
||
id=owner["approval_id"],
|
||
metadata={"schema_version": "untrusted_owner"},
|
||
incident_id=owner["incident_id"],
|
||
hit_count=1,
|
||
)
|
||
enqueue_calls = 0
|
||
|
||
async def must_not_enqueue(**_kwargs):
|
||
nonlocal enqueue_calls
|
||
enqueue_calls += 1
|
||
return {}
|
||
|
||
result = await process_alert_chain_emergency_relay(
|
||
request,
|
||
approval_service=approvals,
|
||
incident_service=FakeIncidentService(),
|
||
candidate_enqueuer=must_not_enqueue,
|
||
)
|
||
|
||
assert result["status"] == "durable_owner_identity_mismatch_no_candidate"
|
||
assert result["candidate_persisted"] is False
|
||
assert result["candidate_queued"] is False
|
||
assert enqueue_calls == 0
|
||
|
||
|
||
def test_endpoint_requires_auth_and_fails_closed_until_queue_ack(monkeypatch) -> None:
|
||
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
|
||
|
||
missing = app_client().post(
|
||
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
||
json=payload(),
|
||
)
|
||
assert missing.status_code == 401
|
||
|
||
async def blocked(_payload):
|
||
return {
|
||
"status": "pre_decision_evidence_not_verified",
|
||
"candidate_persisted": True,
|
||
"candidate_queued": False,
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
webhooks_api,
|
||
"process_alert_chain_emergency_relay",
|
||
blocked,
|
||
)
|
||
response = app_client().post(
|
||
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
||
headers={"X-Agent99-Relay-Token": "expected"},
|
||
json=payload(),
|
||
)
|
||
assert response.status_code == 503
|
||
assert response.json()["detail"] == "pre_decision_evidence_not_verified"
|
||
|
||
|
||
def test_endpoint_returns_public_same_run_receipt(monkeypatch) -> None:
|
||
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
|
||
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
||
owner = build_alert_chain_emergency_owner(request)
|
||
|
||
async def accepted(_payload):
|
||
return {
|
||
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
||
"status": "ansible_candidate_queued_verifier_pending",
|
||
"candidate_persisted": True,
|
||
"candidate_queued": True,
|
||
"identity": {
|
||
"run_id": owner["run_id"],
|
||
"trace_id": owner["trace_id"],
|
||
"work_item_id": owner["work_item_id"],
|
||
},
|
||
"agent99_role": "relay_coordination_only",
|
||
"executor": "host_ansible_executor",
|
||
"catalog_id": ALERT_CHAIN_CATALOG_ID,
|
||
"runtime_write_performed": False,
|
||
"runtime_closure_verified": False,
|
||
}
|
||
|
||
monkeypatch.setattr(
|
||
webhooks_api,
|
||
"process_alert_chain_emergency_relay",
|
||
accepted,
|
||
)
|
||
response = app_client().post(
|
||
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
||
headers={"X-Agent99-Relay-Token": "expected"},
|
||
json=payload(),
|
||
)
|
||
|
||
assert response.status_code == 202
|
||
body = response.json()
|
||
assert body["identity"]["run_id"] == body["identity"]["trace_id"]
|
||
assert body["agent99_role"] == "relay_coordination_only"
|
||
assert body["executor"] == "host_ansible_executor"
|
||
assert body["runtime_write_performed"] is False
|