f4cf465676
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m23s
CD Pipeline / build-and-deploy (push) Successful in 14m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m57s
334 lines
12 KiB
Python
334 lines
12 KiB
Python
from __future__ import annotations
|
|
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from src.jobs import agent99_controlled_dispatch_reconciler_job as job
|
|
from src.services.agent99_controlled_dispatch_ledger import (
|
|
build_agent99_dispatch_identity,
|
|
)
|
|
|
|
|
|
class _Ledger:
|
|
def __init__(self, receipt: dict[str, object]) -> None:
|
|
self.receipt = receipt
|
|
self.verifier_calls: list[dict[str, object]] = []
|
|
|
|
async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def]
|
|
return [{"identity": IDENTITY, "receipt": self.receipt}]
|
|
|
|
async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def]
|
|
self.verifier_calls.append(kwargs)
|
|
return {
|
|
**self.receipt,
|
|
"post_verifier_passed": True,
|
|
"receipt_persisted": True,
|
|
"verifier": {
|
|
"status": "success",
|
|
"evidence_refs": kwargs["evidence_refs"],
|
|
},
|
|
}
|
|
|
|
|
|
IDENTITY = build_agent99_dispatch_identity(
|
|
project_id="awoooi",
|
|
incident_id="INC-20260711-11C751",
|
|
source_fingerprint="fp-cold-start",
|
|
route_id="agent99:host_recovery:Recover",
|
|
)
|
|
|
|
|
|
def test_agent99_playbook_identity_is_project_scoped() -> None:
|
|
other = build_agent99_dispatch_identity(
|
|
project_id="other-project",
|
|
incident_id=IDENTITY.incident_id,
|
|
source_fingerprint=IDENTITY.source_fingerprint,
|
|
route_id=IDENTITY.route_id,
|
|
)
|
|
|
|
assert job._playbook_id(IDENTITY) != job._playbook_id(other)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciler_consumes_authenticated_outcome_and_closes_same_run(
|
|
monkeypatch,
|
|
) -> None:
|
|
ledger = _Ledger({
|
|
"status": "dispatch_accepted_verifier_pending",
|
|
"post_verifier_passed": False,
|
|
"dispatch_receipt": {"kind": "host_recovery"},
|
|
})
|
|
finalized: list[dict[str, object]] = []
|
|
|
|
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
|
monkeypatch.setattr(
|
|
job,
|
|
"read_agent99_sre_outcome",
|
|
lambda _run_id: {
|
|
"automationRunId": str(IDENTITY.run_id),
|
|
"traceId": IDENTITY.trace_id,
|
|
"workItemId": IDENTITY.work_item_id,
|
|
"mode": "Recover",
|
|
"controlledApply": True,
|
|
"outcome": {
|
|
"schemaVersion": "agent99_outcome_contract_v1",
|
|
"state": "resolved",
|
|
"transportOk": True,
|
|
"verifierName": "agent99-recover-post-condition-v1",
|
|
"verifierPassed": True,
|
|
"sourceEventResolved": True,
|
|
"verifiedAt": "2026-07-11T20:00:00+08:00",
|
|
},
|
|
},
|
|
)
|
|
|
|
async def finalize(identity, *, mode, evidence_refs): # type: ignore[no-untyped-def]
|
|
finalized.append({
|
|
"identity": identity,
|
|
"mode": mode,
|
|
"evidence_refs": evidence_refs,
|
|
})
|
|
return {"runtime_closure_verified": True}
|
|
|
|
monkeypatch.setattr(job, "_finalize_learning", finalize)
|
|
|
|
result = await job.reconcile_agent99_controlled_dispatches_once()
|
|
|
|
assert result == {
|
|
"scanned": 1,
|
|
"source_not_resolved": 0,
|
|
"status_reconcile_requested": 0,
|
|
"status_reconcile_pending": 0,
|
|
"external_no_write_reconciled": 0,
|
|
"verifier_written": 1,
|
|
"closed": 1,
|
|
}
|
|
assert len(ledger.verifier_calls) == 1
|
|
refs = ledger.verifier_calls[0]["evidence_refs"]
|
|
assert refs["agent99_outcome_receipt_id"].endswith(str(IDENTITY.run_id))
|
|
assert "post_verifier_evidence_ref" in refs
|
|
assert "source_event_evidence_ref" in refs
|
|
assert finalized[0]["identity"].run_id == IDENTITY.run_id
|
|
assert finalized[0]["mode"] == "Recover"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_reconciler_does_not_finalize_without_outcome(monkeypatch) -> None:
|
|
ledger = _Ledger({
|
|
"status": "dispatch_delivery_unknown_reconcile_only",
|
|
"post_verifier_passed": False,
|
|
})
|
|
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
|
monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run_id: None)
|
|
|
|
result = await job.reconcile_agent99_controlled_dispatches_once()
|
|
|
|
assert result == {
|
|
"scanned": 1,
|
|
"source_not_resolved": 0,
|
|
"status_reconcile_requested": 0,
|
|
"status_reconcile_pending": 0,
|
|
"external_no_write_reconciled": 0,
|
|
"verifier_written": 0,
|
|
"closed": 0,
|
|
}
|
|
assert ledger.verifier_calls == []
|
|
|
|
|
|
def test_windows_relay_outcome_readback_is_authenticated_and_public_safe() -> None:
|
|
source = Path(__file__).resolve().parents[3] / "agent99-sre-alert-relay.ps1"
|
|
text = source.read_text(encoding="utf-8")
|
|
|
|
assert '$context.Request.HttpMethod -eq "GET"' in text
|
|
assert "relay_auth_not_configured" in text
|
|
assert 'Headers["X-Agent99-Relay-Token"]' in text
|
|
token_gate = text.index("if (-not $expectedToken)")
|
|
get_branch = text.index('if ($context.Request.HttpMethod -eq "GET")')
|
|
post_branch = text.index('if ($context.Request.HttpMethod -ne "POST")')
|
|
assert token_gate < get_branch < post_branch
|
|
assert "storesRawEvidence = $false" in text
|
|
assert "stdout" not in text[text.index("function Get-AgentOutcomeReadback"):text.index("function Write-AgentRelayEvent")]
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_exact_11c751_legacy_failure_backfills_once_without_recurrence() -> None:
|
|
row = {
|
|
"incident_id": "INC-20260711-11C751",
|
|
"project_id": "awoooi",
|
|
"alertname": "ColdStartGateBlocked",
|
|
"severity": "low",
|
|
"alert_category": "general",
|
|
"notification_type": "legacy_backfill",
|
|
"approval_id": "00000000-0000-0000-0000-000000000751",
|
|
"approval_fingerprint": "legacy-source-fingerprint-11c751",
|
|
"approval_status": "execution_failed",
|
|
}
|
|
calls: list[dict[str, object]] = []
|
|
dispatched_keys: set[tuple[str, str, str]] = set()
|
|
fetch_count = 0
|
|
|
|
async def fetcher(**_kwargs): # type: ignore[no-untyped-def]
|
|
nonlocal fetch_count
|
|
fetch_count += 1
|
|
if fetch_count == 1:
|
|
return [row]
|
|
return [{
|
|
**row,
|
|
"latest_approval_id": "00000000-0000-0000-0000-000000000752",
|
|
"latest_approval_fingerprint": "newer-approval-fingerprint-11c751",
|
|
}]
|
|
|
|
async def dispatcher(**kwargs): # type: ignore[no-untyped-def]
|
|
calls.append(kwargs)
|
|
key = (
|
|
str(kwargs["incident_id"]),
|
|
str(kwargs["fingerprint"]),
|
|
str(kwargs["route_id"]),
|
|
)
|
|
if key in dispatched_keys:
|
|
return {"status": "deduplicated", "dispatchPerformed": False}
|
|
dispatched_keys.add(key)
|
|
return {"status": "dispatched", "dispatchPerformed": True}
|
|
|
|
first = await job.backfill_legacy_agent99_dispatches_once(
|
|
fetcher=fetcher,
|
|
dispatcher=dispatcher,
|
|
)
|
|
second = await job.backfill_legacy_agent99_dispatches_once(
|
|
fetcher=fetcher,
|
|
dispatcher=dispatcher,
|
|
)
|
|
|
|
assert first == {
|
|
"scanned": 1,
|
|
"dispatch_performed": 1,
|
|
"deduplicated": 0,
|
|
"retryable": 0,
|
|
"blocked": 0,
|
|
}
|
|
assert second == {
|
|
"scanned": 1,
|
|
"dispatch_performed": 0,
|
|
"deduplicated": 1,
|
|
"retryable": 0,
|
|
"blocked": 0,
|
|
}
|
|
assert len(dispatched_keys) == 1
|
|
assert calls[0]["incident_id"] == "INC-20260711-11C751"
|
|
assert calls[0]["namespace"] == "host_recovery"
|
|
assert calls[0]["target_resource"] == "cold-start-gate"
|
|
assert calls[0]["route_id"] == "agent99:host_recovery:Recover"
|
|
assert calls[0]["fingerprint"].startswith("legacy-cold-start:")
|
|
assert calls[0]["fingerprint"] == calls[1]["fingerprint"]
|
|
assert calls[0]["approval_id"] == calls[1]["approval_id"]
|
|
assert calls[0]["labels"]["legacy_approval_fingerprint"] == (
|
|
calls[1]["labels"]["legacy_approval_fingerprint"]
|
|
)
|
|
assert calls[0]["labels"]["legacy_executor"] == "ansible"
|
|
assert calls[0]["labels"]["legacy_status"] == "execution_failed"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_learning_checkpoint_resumes_without_replaying_assets(
|
|
monkeypatch,
|
|
) -> None:
|
|
identity = build_agent99_dispatch_identity(
|
|
project_id="awoooi",
|
|
incident_id="INC-20260711-BACKUP",
|
|
source_fingerprint="backup-source-1",
|
|
route_id="agent99:backup_health:BackupCheck",
|
|
approval_id="00000000-0000-0000-0000-000000000751",
|
|
)
|
|
|
|
class CheckpointLedger:
|
|
def __init__(self) -> None:
|
|
self.refs: dict[str, str] = {}
|
|
|
|
async def record_learning_asset_acks(
|
|
self,
|
|
*,
|
|
identity, # type: ignore[no-untyped-def]
|
|
receipt_refs,
|
|
) -> dict[str, object]:
|
|
self.refs.update(receipt_refs)
|
|
return {
|
|
"status": "learning_asset_acks_recorded",
|
|
"receipt_persisted": True,
|
|
"learning_receipt_refs": dict(self.refs),
|
|
"runtime_closure_verified": False,
|
|
}
|
|
|
|
ledger = CheckpointLedger()
|
|
effects = {"playbook": 0, "km": 0, "telegram": 0, "dr": 0}
|
|
terminal_calls = 0
|
|
|
|
async def write_playbook(*_args, **_kwargs): # type: ignore[no-untyped-def]
|
|
effects["playbook"] += 1
|
|
return "playbooks:PB-A99:version:1"
|
|
|
|
async def write_km(*_args, **_kwargs): # type: ignore[no-untyped-def]
|
|
effects["km"] += 1
|
|
return "knowledge_entries:km-1:embedding_verified"
|
|
|
|
async def write_telegram(*_args, **_kwargs): # type: ignore[no-untyped-def]
|
|
effects["telegram"] += 1
|
|
return "telegram_outbound:1:durable_ack"
|
|
|
|
async def write_dr(*_args, **_kwargs): # type: ignore[no-untyped-def]
|
|
effects["dr"] += 1
|
|
return "knowledge_entries:dr-1:dr_scorecard"
|
|
|
|
async def terminal(**_kwargs): # type: ignore[no-untyped-def]
|
|
nonlocal terminal_calls
|
|
terminal_calls += 1
|
|
if terminal_calls == 1:
|
|
return {
|
|
"status": "learning_writeback_persistence_failed",
|
|
"runtime_closure_verified": False,
|
|
}
|
|
return {
|
|
"status": "closed_verified_learning_written",
|
|
"runtime_closure_verified": True,
|
|
}
|
|
|
|
monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger)
|
|
monkeypatch.setattr(job, "_ensure_playbook_trust", write_playbook)
|
|
monkeypatch.setattr(job, "_ensure_km_writeback", write_km)
|
|
monkeypatch.setattr(job, "_ensure_telegram_receipt", write_telegram)
|
|
monkeypatch.setattr(job, "_ensure_dr_scorecard", write_dr)
|
|
monkeypatch.setattr(job, "record_agent99_learning_writeback", terminal)
|
|
|
|
first = await job._finalize_learning(
|
|
identity,
|
|
mode="BackupCheck",
|
|
evidence_refs={},
|
|
)
|
|
second = await job._finalize_learning(
|
|
identity,
|
|
mode="BackupCheck",
|
|
evidence_refs={},
|
|
)
|
|
|
|
assert first["runtime_closure_verified"] is False
|
|
assert second["runtime_closure_verified"] is True
|
|
assert effects == {"playbook": 1, "km": 1, "telegram": 1, "dr": 1}
|
|
assert terminal_calls == 2
|
|
|
|
|
|
def test_legacy_backfill_query_is_bounded_and_requires_failure_evidence() -> None:
|
|
source = Path(job.__file__).read_text(encoding="utf-8")
|
|
fetch = source[
|
|
source.index("async def _fetch_legacy_cold_start_failures"):source.index(
|
|
"def _legacy_source_fingerprint"
|
|
)
|
|
]
|
|
|
|
assert 'LEGACY_COLD_START_INCIDENT_ID = "INC-20260711-11C751"' in source
|
|
assert "lower(approval_state.status::text) = 'execution_failed'" in fetch
|
|
assert "ORDER BY candidate.created_at ASC, candidate.id ASC" in fetch
|
|
assert "legacy.input ->> 'executor' = 'ansible'" in fetch
|
|
assert "LIMIT :limit" in fetch
|
|
assert "statement_timeout = '5000ms'" in fetch
|
|
assert "backfill_legacy_agent99_dispatches_once()" in source
|