541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
467 lines
17 KiB
Python
467 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
# ruff: noqa: E402
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
from types import SimpleNamespace
|
|
|
|
os.environ.setdefault(
|
|
"DATABASE_URL",
|
|
"postgresql+asyncpg://test:test@localhost/test",
|
|
)
|
|
|
|
import pytest
|
|
from fastapi import FastAPI
|
|
from fastapi.testclient import TestClient
|
|
from pydantic import ValidationError
|
|
|
|
from src.api.v1 import webhooks as webhooks_api
|
|
from src.core.config import settings
|
|
from src.services.alert_chain_emergency_ingress import (
|
|
ALERT_CHAIN_AGENT99_ROLE,
|
|
ALERT_CHAIN_CANONICAL_ASSET_ID,
|
|
ALERT_CHAIN_CATALOG_ID,
|
|
ALERT_CHAIN_EXECUTOR,
|
|
AlertChainEmergencyRelayRequest,
|
|
build_alert_chain_emergency_owner,
|
|
process_alert_chain_emergency_relay,
|
|
)
|
|
|
|
REPO_ROOT = Path(__file__).resolve().parents[3]
|
|
|
|
|
|
def payload() -> dict:
|
|
return {
|
|
"schema_version": "agent99_alertmanager_emergency_relay_v1",
|
|
"relay_receipt_id": f"agent99-alertchain-{'a' * 64}",
|
|
"source_host": "192.168.0.110",
|
|
"source_url": "http://192.168.0.110:9093/api/v2/alerts",
|
|
"source_fingerprint": "0123456789abcdef",
|
|
"source_started_at": "2026-07-16T13:40:20+08:00",
|
|
"alertname": "AlertChainBroken_Alertmanager",
|
|
"status": "firing",
|
|
"severity": "critical",
|
|
"target_resource": "alertmanager",
|
|
"namespace": "monitoring",
|
|
"agent99_role": "relay_coordination_only",
|
|
"executor": "host_ansible_executor",
|
|
"catalog_id": "ansible:110-alertmanager-delivery-recovery",
|
|
"runtime_write_performed": False,
|
|
"raw_payload_stored": False,
|
|
}
|
|
|
|
|
|
class FakeApprovalService:
|
|
def __init__(self) -> None:
|
|
self.approval = None
|
|
self.created_requests = []
|
|
self.incremented = 0
|
|
|
|
async def get_approval(self, approval_id):
|
|
if self.approval and str(self.approval.id) == str(approval_id):
|
|
return self.approval
|
|
return None
|
|
|
|
async def create_approval_with_fingerprint(self, *, request, fingerprint):
|
|
self.created_requests.append((request, fingerprint))
|
|
metadata = dict(request.metadata or {})
|
|
approval_id = metadata.pop("preallocated_approval_id")
|
|
self.approval = SimpleNamespace(
|
|
id=approval_id,
|
|
metadata=metadata,
|
|
incident_id=request.incident_id,
|
|
hit_count=1,
|
|
)
|
|
return self.approval
|
|
|
|
async def increment_hit_count(self, _approval_id):
|
|
self.incremented += 1
|
|
self.approval.hit_count += 1
|
|
return self.approval
|
|
|
|
async def update_incident_id(self, _approval_id, incident_id):
|
|
self.approval.incident_id = incident_id
|
|
|
|
|
|
class FakeIncident:
|
|
def __init__(self, incident_id: str) -> None:
|
|
self.incident_id = incident_id
|
|
self.persisted_to_pg = True
|
|
|
|
def model_dump(self, *, mode: str):
|
|
assert mode == "json"
|
|
return {
|
|
"incident_id": self.incident_id,
|
|
"status": "investigating",
|
|
"severity": "P2",
|
|
"affected_services": ["alertmanager"],
|
|
"signals": [
|
|
{
|
|
"alert_name": "AlertChainBroken_Alertmanager",
|
|
"labels": {
|
|
"alertname": "AlertChainBroken_Alertmanager",
|
|
"component": "alertmanager",
|
|
},
|
|
"annotations": {},
|
|
}
|
|
],
|
|
}
|
|
|
|
|
|
class FakeIncidentService:
|
|
def __init__(self) -> None:
|
|
self.incident = None
|
|
|
|
async def get_for_readback(self, incident_id, *, project_id):
|
|
assert project_id == "awoooi"
|
|
if self.incident and self.incident.incident_id == incident_id:
|
|
return self.incident
|
|
return None
|
|
|
|
|
|
def app_client() -> TestClient:
|
|
app = FastAPI()
|
|
app.include_router(webhooks_api.router, prefix="/api/v1")
|
|
return TestClient(app)
|
|
|
|
|
|
def test_relay_model_rejects_raw_content_and_executor_drift() -> None:
|
|
with pytest.raises(ValidationError):
|
|
AlertChainEmergencyRelayRequest.model_validate(
|
|
{**payload(), "raw_log": "must not enter machine ingress"}
|
|
)
|
|
with pytest.raises(ValidationError):
|
|
AlertChainEmergencyRelayRequest.model_validate(
|
|
{**payload(), "executor": "Agent99"}
|
|
)
|
|
with pytest.raises(ValidationError):
|
|
AlertChainEmergencyRelayRequest.model_validate(
|
|
{**payload(), "source_started_at": "2026-07-16T13:40:20"}
|
|
)
|
|
with pytest.raises(ValidationError):
|
|
AlertChainEmergencyRelayRequest.model_validate(
|
|
{**payload(), "source_host": "192.168.0.111"}
|
|
)
|
|
|
|
|
|
def test_agent99_poll_is_read_only_first_hop_and_reduced_authenticated_outbound() -> None:
|
|
source = (
|
|
REPO_ROOT / "agent99-alertmanager-alertchain-poll.ps1"
|
|
).read_text(
|
|
encoding="utf-8"
|
|
)
|
|
first_hop = source.split("$alerts = @(Invoke-RestMethod", 1)[1].split(
|
|
"} catch", 1
|
|
)[0]
|
|
|
|
assert "http://192.168.0.110:9093/api/v2/alerts" in source
|
|
assert '$sourceUri.Host -ne $CanonicalSourceHost' in source
|
|
assert '$sourceUri.Port -ne 9093' in source
|
|
assert '$sourceUri.AbsolutePath -ne "/api/v2/alerts"' in source
|
|
assert "-Method Get" in first_hop
|
|
assert "-Headers" not in first_hop
|
|
assert "token" not in first_hop.lower()
|
|
assert '$state -ne "active"' in source
|
|
assert "AlertChainBroken_Alertmanager" in source
|
|
assert 'Headers @{ "X-Agent99-Relay-Token" = $token }' in source
|
|
assert "agent99_alertmanager_emergency_relay_v1" in source
|
|
assert "source_host = $CanonicalSourceHost" in source
|
|
assert "source_url = $CanonicalSourceUrl" in source
|
|
assert 'agent99_role = "relay_coordination_only"' in source
|
|
assert 'executor = "host_ansible_executor"' in source
|
|
assert 'catalog_id = $CanonicalCatalogId' in source
|
|
assert "candidate_queued" in source
|
|
assert "runtime_write_performed = $false" in source
|
|
assert "raw_payload_stored = $false" in source
|
|
assert "dedupe-receipts" in source
|
|
assert "rawPayloadStored = $false" in source
|
|
assert "Start-AgentSreAlertInbox" not in source
|
|
assert "agent99-run.ps1" not in source
|
|
|
|
listener = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "alertmanager-emergency" not in listener
|
|
assert "AlertmanagerPrefix" not in listener
|
|
|
|
|
|
def test_agent99_install_and_bootstrap_publish_poll_contract() -> None:
|
|
installer = (REPO_ROOT / "agent99-install-sre-alert-relay.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
bootstrap = (REPO_ROOT / "agent99-bootstrap.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
for source in (installer, bootstrap):
|
|
assert "http://192.168.0.110:9093/api/v2/alerts" in source
|
|
assert "pollIntervalSeconds" in source
|
|
assert "relay_coordination_only" in source
|
|
assert "host_ansible_executor" in source
|
|
assert "ansible:110-alertmanager-delivery-recovery" in source
|
|
assert '"alertmanagerPrefix":' not in source
|
|
assert 'Set-AgentProp $config.sreAlertRelay "alertmanagerPrefix"' not in source
|
|
|
|
relay_script = (REPO_ROOT / "agent99-sre-alert-relay.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
task_script = (REPO_ROOT / "agent99-register-tasks.ps1").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
assert "Register-ObjectEvent" in relay_script
|
|
assert "in_process_nonblocking_timer" in relay_script
|
|
assert "$alertChainPollTimer.AutoReset = $false" in relay_script
|
|
assert "$pollData.Timer.Start()" in relay_script
|
|
assert "Unregister-Event -SourceIdentifier $alertChainPollEventId" in relay_script
|
|
assert "Start-Job" not in relay_script
|
|
assert "Wooo-Agent99-AlertChain-Poll" not in task_script
|
|
|
|
for config_name in (
|
|
"agent99.config.example.json",
|
|
"agent99.config.99.example.json",
|
|
):
|
|
config = json.loads((REPO_ROOT / config_name).read_text(encoding="utf-8"))
|
|
relay = config["sreAlertRelay"]
|
|
assert "alertmanagerPrefix" not in relay
|
|
poll = relay["alertChainPoll"]
|
|
assert poll["sourceUrl"] == (
|
|
"http://192.168.0.110:9093/api/v2/alerts"
|
|
)
|
|
assert poll["sourceHost"] == "192.168.0.110"
|
|
assert poll["pollIntervalSeconds"] == 60
|
|
assert poll["agent99Role"] == "relay_coordination_only"
|
|
assert poll["linuxExecutor"] == "host_ansible_executor"
|
|
assert poll["catalogId"] == (
|
|
"ansible:110-alertmanager-delivery-recovery"
|
|
)
|
|
assert poll["firstHopSecretTransmitted"] is False
|
|
assert poll["rawPayloadStored"] is False
|
|
|
|
|
|
def test_owner_identity_is_stable_and_binds_same_run_trace_work_item() -> None:
|
|
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
|
|
|
first = build_alert_chain_emergency_owner(request)
|
|
second = build_alert_chain_emergency_owner(request)
|
|
|
|
assert first == second
|
|
assert first["run_id"] == first["trace_id"]
|
|
assert first["work_item_id"].startswith("alertchain:awoooi:")
|
|
assert first["agent99_role"] == ALERT_CHAIN_AGENT99_ROLE
|
|
assert first["executor"] == ALERT_CHAIN_EXECUTOR
|
|
assert first["catalog_id"] == ALERT_CHAIN_CATALOG_ID
|
|
assert first["canonical_asset_id"] == ALERT_CHAIN_CANONICAL_ASSET_ID
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_secondary_ingress_persists_and_queues_exact_ansible_candidate() -> None:
|
|
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
|
owner = build_alert_chain_emergency_owner(request)
|
|
approvals = FakeApprovalService()
|
|
incidents = FakeIncidentService()
|
|
enqueue_calls = []
|
|
events = []
|
|
|
|
async def create_incident(**kwargs):
|
|
assert kwargs["canonical_incident_id"] == owner["incident_id"]
|
|
assert kwargs["alertname"] == "AlertChainBroken_Alertmanager"
|
|
assert kwargs["namespace"] == "monitoring"
|
|
incidents.incident = FakeIncident(owner["incident_id"])
|
|
return owner["incident_id"]
|
|
|
|
async def enqueue(**kwargs):
|
|
enqueue_calls.append(kwargs)
|
|
assert kwargs["automation_run_id"] == owner["run_id"]
|
|
assert kwargs["incident"]["run_id"] == owner["run_id"]
|
|
assert kwargs["incident"]["trace_id"] == owner["trace_id"]
|
|
assert kwargs["incident"]["work_item_id"] == owner["work_item_id"]
|
|
assert kwargs["proposal_data"]["source_recurrence_verified"] is True
|
|
return {
|
|
"status": "controlled_check_mode_queued",
|
|
"automation_run_id": owner["run_id"],
|
|
"trace_id": owner["trace_id"],
|
|
"work_item_id": owner["work_item_id"],
|
|
"queued": True,
|
|
"side_effect_performed": False,
|
|
"active_blockers": [],
|
|
}
|
|
|
|
async def record_event(**kwargs):
|
|
events.append(kwargs)
|
|
|
|
result = await process_alert_chain_emergency_relay(
|
|
request,
|
|
approval_service=approvals,
|
|
incident_service=incidents,
|
|
incident_creator=create_incident,
|
|
candidate_enqueuer=enqueue,
|
|
event_recorder=record_event,
|
|
)
|
|
|
|
assert result["status"] == "ansible_candidate_queued_verifier_pending"
|
|
assert result["candidate_persisted"] is True
|
|
assert result["candidate_queued"] is True
|
|
assert result["identity"]["run_id"] == result["identity"]["trace_id"]
|
|
assert result["identity"]["work_item_id"] == owner["work_item_id"]
|
|
assert result["agent99_role"] == "relay_coordination_only"
|
|
assert result["executor"] == "host_ansible_executor"
|
|
assert result["runtime_write_performed"] is False
|
|
assert result["runtime_closure_verified"] is False
|
|
assert len(enqueue_calls) == 1
|
|
assert events[0]["stage"] == "secondary_ingress_ansible_candidate_queued"
|
|
|
|
repeated = await process_alert_chain_emergency_relay(
|
|
request,
|
|
approval_service=approvals,
|
|
incident_service=incidents,
|
|
incident_creator=create_incident,
|
|
candidate_enqueuer=enqueue,
|
|
event_recorder=record_event,
|
|
)
|
|
assert repeated["identity"] == result["identity"]
|
|
assert approvals.incremented == 1
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_secondary_ingress_keeps_persisted_candidate_partial_when_queue_blocks() -> None:
|
|
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
|
owner = build_alert_chain_emergency_owner(request)
|
|
approvals = FakeApprovalService()
|
|
incidents = FakeIncidentService()
|
|
|
|
async def create_incident(**_kwargs):
|
|
incidents.incident = FakeIncident(owner["incident_id"])
|
|
return owner["incident_id"]
|
|
|
|
async def blocked_enqueue(**_kwargs):
|
|
return {
|
|
"status": "pre_decision_evidence_not_verified",
|
|
"automation_run_id": owner["run_id"],
|
|
"trace_id": owner["trace_id"],
|
|
"work_item_id": owner["work_item_id"],
|
|
"queued": False,
|
|
"side_effect_performed": False,
|
|
"active_blockers": ["pre_decision_evidence_not_verified"],
|
|
}
|
|
|
|
async def no_event(**_kwargs):
|
|
return None
|
|
|
|
result = await process_alert_chain_emergency_relay(
|
|
request,
|
|
approval_service=approvals,
|
|
incident_service=incidents,
|
|
incident_creator=create_incident,
|
|
candidate_enqueuer=blocked_enqueue,
|
|
event_recorder=no_event,
|
|
)
|
|
|
|
assert result["candidate_persisted"] is True
|
|
assert result["candidate_queued"] is False
|
|
assert result["runtime_closure_verified"] is False
|
|
assert result["safe_next_action"].startswith("retry_same_relay_receipt")
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_secondary_ingress_rejects_deterministic_owner_collision() -> None:
|
|
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
|
owner = build_alert_chain_emergency_owner(request)
|
|
approvals = FakeApprovalService()
|
|
approvals.approval = SimpleNamespace(
|
|
id=owner["approval_id"],
|
|
metadata={"schema_version": "untrusted_owner"},
|
|
incident_id=owner["incident_id"],
|
|
hit_count=1,
|
|
)
|
|
enqueue_calls = 0
|
|
|
|
async def must_not_enqueue(**_kwargs):
|
|
nonlocal enqueue_calls
|
|
enqueue_calls += 1
|
|
return {}
|
|
|
|
result = await process_alert_chain_emergency_relay(
|
|
request,
|
|
approval_service=approvals,
|
|
incident_service=FakeIncidentService(),
|
|
candidate_enqueuer=must_not_enqueue,
|
|
)
|
|
|
|
assert result["status"] == "durable_owner_identity_mismatch_no_candidate"
|
|
assert result["candidate_persisted"] is False
|
|
assert result["candidate_queued"] is False
|
|
assert enqueue_calls == 0
|
|
|
|
|
|
def test_endpoint_requires_auth_and_fails_closed_until_queue_ack(monkeypatch) -> None:
|
|
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
|
|
|
|
missing = app_client().post(
|
|
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
|
json=payload(),
|
|
)
|
|
assert missing.status_code == 401
|
|
|
|
async def blocked(_payload):
|
|
return {
|
|
"status": "pre_decision_evidence_not_verified",
|
|
"candidate_persisted": True,
|
|
"candidate_queued": False,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
webhooks_api,
|
|
"process_alert_chain_emergency_relay",
|
|
blocked,
|
|
)
|
|
response = app_client().post(
|
|
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
|
headers={"X-Agent99-Relay-Token": "expected"},
|
|
json=payload(),
|
|
)
|
|
assert response.status_code == 503
|
|
assert response.json()["detail"] == "pre_decision_evidence_not_verified"
|
|
|
|
|
|
def test_endpoint_returns_public_same_run_receipt(monkeypatch) -> None:
|
|
monkeypatch.setattr(settings, "AGENT99_SRE_ALERT_RELAY_TOKEN", "expected")
|
|
request = AlertChainEmergencyRelayRequest.model_validate(payload())
|
|
owner = build_alert_chain_emergency_owner(request)
|
|
|
|
async def accepted(_payload):
|
|
return {
|
|
"schema_version": "alert_chain_emergency_ingress_result_v1",
|
|
"status": "ansible_candidate_queued_verifier_pending",
|
|
"candidate_persisted": True,
|
|
"candidate_queued": True,
|
|
"identity": {
|
|
"run_id": owner["run_id"],
|
|
"trace_id": owner["trace_id"],
|
|
"work_item_id": owner["work_item_id"],
|
|
},
|
|
"agent99_role": "relay_coordination_only",
|
|
"executor": "host_ansible_executor",
|
|
"catalog_id": ALERT_CHAIN_CATALOG_ID,
|
|
"runtime_write_performed": False,
|
|
"runtime_closure_verified": False,
|
|
}
|
|
|
|
monkeypatch.setattr(
|
|
webhooks_api,
|
|
"process_alert_chain_emergency_relay",
|
|
accepted,
|
|
)
|
|
response = app_client().post(
|
|
"/api/v1/webhooks/agent99/alertmanager-emergency",
|
|
headers={"X-Agent99-Relay-Token": "expected"},
|
|
json=payload(),
|
|
)
|
|
|
|
assert response.status_code == 202
|
|
body = response.json()
|
|
assert body["identity"]["run_id"] == body["identity"]["trace_id"]
|
|
assert body["agent99_role"] == "relay_coordination_only"
|
|
assert body["executor"] == "host_ansible_executor"
|
|
assert body["runtime_write_performed"] is False
|