diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index e1fc23f66..42de86857 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -5235,6 +5235,7 @@ function Invoke-AgentQueuedCommands { $modeName = if ($command.mode) { [string]$command.mode } else { "" } $controlled = if ($command.PSObject.Properties["controlledApply"]) { [bool]$command.controlledApply } else { $false } $commandSource = if ($command.PSObject.Properties["source"]) { [string]$command.source } else { "queue" } + $requestedBy = if ($command.PSObject.Properties["requestedBy"]) { [string]$command.requestedBy } else { "" } $commandReason = if ($command.PSObject.Properties["reason"]) { [string]$command.reason } else { "operator_instruction" } $commandInstruction = if ($command.PSObject.Properties["instruction"]) { [string]$command.instruction } else { "" } $correlationKey = if ($command.PSObject.Properties["correlationKey"]) { [string]$command.correlationKey } else { "" } @@ -5322,6 +5323,27 @@ function Invoke-AgentQueuedCommands { } else { "" } + $parsedAutomationRunId = [guid]::Empty + $automationRunIdValid = [bool]([guid]::TryParse($automationRunId, [ref]$parsedAutomationRunId)) + $legacyColdStartIdentity = [bool]( + $incidentId -eq "INC-20260711-11C751" -and + $workItemId -eq "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" -and + $sourceFingerprint -match "^legacy-cold-start:[0-9a-f]{64}$" -and + $executionGeneration -eq "1" + ) + $liveColdStartIdentity = [bool]( + $incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$" -and + $workItemId -eq "agent99-dispatch:awoooi:$incidentId`:agent99:host_recovery:Recover" -and + $sourceFingerprint -match "^[0-9a-f]{32,64}$" -and + $executionGeneration -match "^[1-3]$" + ) + $coldStartIdentityEligible = [bool]( + $automationRunIdValid -and + $projectId -eq "awoooi" -and + $dispatchRouteId -eq "agent99:host_recovery:Recover" -and + $lockOwner -eq $automationRunId -and + ($legacyColdStartIdentity -or $liveColdStartIdentity) + ) $sameRunReconcileContractExact = [bool]( $reconcileOnly -and $modeName -eq "Status" -and @@ -5330,15 +5352,13 @@ function Invoke-AgentQueuedCommands { -not $dispatchIdentityIncomplete -and $dispatchControlIdentitySafe -and $canonicalDigestMatches -and - $projectId -eq "awoooi" -and - $workItemId -eq "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" -and + $coldStartIdentityEligible -and $traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$" -and - $incidentId -eq "INC-20260711-11C751" -and - $automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811" -and - $dispatchRouteId -eq "agent99:host_recovery:Recover" -and - $executionGeneration -eq "1" -and - $lockOwner -eq $automationRunId -and - $sourceFingerprint.StartsWith("legacy-cold-start:") -and + $commandSource -eq "agent99-same-run-reconcile" -and + $requestedBy -eq "awoooi-agent99-controlled-dispatch-reconciler" -and + $alertSource -eq "awoooi-production-source-verifier" -and + $alertKind -eq "host_recovery" -and + $alertService -eq "cold-start-gate" -and $sourceEventResolutionRequired -and $sourceEventResolutionPolicy -eq "external_source_receipt_required" -and $sourceEventResolved -eq $true -and @@ -7489,6 +7509,7 @@ $result = [pscustomobject]@{ reconcileQueueId = $ReconcileQueueId runtimeWritePerformed = [bool]( ($recoveryTrigger -and $recoveryTrigger.queued) -or + @($externalHostRecovery | Where-Object { $_.runtimeWritePerformed }).Count -gt 0 -or ($host112Recovery -and $host112Recovery.runtimeWritePerformed) -or ($host188EdgeRecovery -and $host188EdgeRecovery.runtimeWritePerformed) ) diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index ed9ba508d..b35d910eb 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -329,35 +329,39 @@ function Test-AgentSameRunDispatchIdentity { if (-not [guid]::TryParse($runId, [ref]$parsedRunId)) { return [pscustomobject]@{ ok = $false; reason = "identity_run_id_invalid" } } - if ($runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811") { - return [pscustomobject]@{ ok = $false; reason = "identity_run_not_eligible" } - } if ([string](Get-AgentField $Identity "schema_version" "") -ne "agent99_controlled_dispatch_identity_v1") { return [pscustomobject]@{ ok = $false; reason = "identity_schema_invalid" } } if ([string](Get-AgentField $Identity "project_id" "") -ne "awoooi") { return [pscustomobject]@{ ok = $false; reason = "identity_project_not_eligible" } } - if ([string](Get-AgentField $Identity "work_item_id" "") -ne "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start") { - return [pscustomobject]@{ ok = $false; reason = "identity_work_item_not_eligible" } - } if ([string](Get-AgentField $Identity "trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$") { return [pscustomobject]@{ ok = $false; reason = "identity_trace_not_eligible" } } if ([string](Get-AgentField $Identity "route_id" "") -ne "agent99:host_recovery:Recover") { return [pscustomobject]@{ ok = $false; reason = "identity_route_not_eligible" } } - if ([string](Get-AgentField $Identity "incident_id" "") -ne "INC-20260711-11C751") { - return [pscustomobject]@{ ok = $false; reason = "identity_incident_not_eligible" } - } - if ([string](Get-AgentField $Identity "execution_generation" "") -ne "1") { - return [pscustomobject]@{ ok = $false; reason = "identity_generation_advance_forbidden" } - } if ([string](Get-AgentField $Identity "lock_owner" "") -ne $runId) { return [pscustomobject]@{ ok = $false; reason = "identity_lock_owner_mismatch" } } - if (-not ([string](Get-AgentField $Identity "source_fingerprint" "")).StartsWith("legacy-cold-start:")) { - return [pscustomobject]@{ ok = $false; reason = "identity_source_not_legacy_cold_start" } + $incidentId = [string](Get-AgentField $Identity "incident_id" "") + $workItemId = [string](Get-AgentField $Identity "work_item_id" "") + $sourceFingerprint = [string](Get-AgentField $Identity "source_fingerprint" "") + $executionGeneration = [string](Get-AgentField $Identity "execution_generation" "") + $legacyIdentity = [bool]( + $incidentId -eq "INC-20260711-11C751" -and + $workItemId -eq "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" -and + $sourceFingerprint -match "^legacy-cold-start:[0-9a-f]{64}$" -and + $executionGeneration -eq "1" + ) + $liveIdentity = [bool]( + $incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$" -and + $workItemId -eq "agent99-dispatch:awoooi:$incidentId`:agent99:host_recovery:Recover" -and + $sourceFingerprint -match "^[0-9a-f]{32,64}$" -and + $executionGeneration -match "^[1-3]$" + ) + if (-not ($legacyIdentity -or $liveIdentity)) { + return [pscustomobject]@{ ok = $false; reason = "identity_cold_start_scope_not_eligible" } } $canonical = [ordered]@{ approval_id = [string](Get-AgentField $Identity "approval_id" "") @@ -407,6 +411,27 @@ function Convert-AgentStoredDispatchIdentity { } } +function Test-AgentStoredColdStartRecoverEnvelope { + param([object]$Stored) + + if (-not $Stored -or [bool](Get-AgentField $Stored "reconcileOnly" $false)) { + return $false + } + $mode = [string](Get-AgentField $Stored "mode" (Get-AgentField $Stored "suggestedMode" "")) + $source = [string](Get-AgentField $Stored "alertSource" (Get-AgentField $Stored "source" "")) + $kind = [string](Get-AgentField $Stored "alertKind" (Get-AgentField $Stored "kind" "")) + $service = [string](Get-AgentField $Stored "alertService" (Get-AgentField $Stored "service" "")) + $controlled = Get-AgentField $Stored "controlledApply" $null + return [bool]( + $mode -eq "Recover" -and + $source -eq "awoooi-api-alertmanager" -and + $kind -eq "host_recovery" -and + $service -eq "cold-start-gate" -and + $controlled -is [bool] -and + $controlled + ) +} + function Get-AgentExistingRecoverIdentity { param([string]$RunId) @@ -415,32 +440,20 @@ function Get-AgentExistingRecoverIdentity { $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "running-*.json" -File -ErrorAction SilentlyContinue) $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running|same-run-status)-" }) - # The API's durable receipt predates the inbox's final classification. The - # exact legacy Recover envelope was accepted by the relay and then moved to - # ignored by an older inbox build. Admit only that immutable source envelope - # as identity evidence so the externally-resolved incident can run Status; - # never replay Recover or accept a generic ignored/reconcile payload. + # One legacy envelope was moved to ignored by an older inbox build. Current + # runs are admitted only from an immutable cold-start Recover envelope; a + # generic ignored, queue, or reconcile payload can never authorize Status. $acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json" $files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256) $files += @(Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern -File -ErrorAction SilentlyContinue) foreach ($file in $files) { try { $stored = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json - if ([bool](Get-AgentField $stored "reconcileOnly" $false)) { continue } + if (-not (Test-AgentStoredColdStartRecoverEnvelope $stored)) { continue } $storedMode = [string](Get-AgentField $stored "mode" (Get-AgentField $stored "suggestedMode" "")) if ($storedMode -ne "Recover") { continue } $identity = Convert-AgentStoredDispatchIdentity $stored if ([string](Get-AgentField $identity "run_id" "") -ne $RunId) { continue } - if ($file.DirectoryName -eq $AcceptedIgnoredDir) { - if ( - [string](Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId" -or - [string](Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager" -or - [string](Get-AgentField $stored "kind" "") -ne "host_recovery" -or - [string](Get-AgentField $stored "service" "") -ne "cold-start-gate" -or - (Get-AgentField $stored "controlledApply" $null) -isnot [bool] -or - -not [bool](Get-AgentField $stored "controlledApply" $false) - ) { continue } - } $check = Test-AgentSameRunDispatchIdentity $identity if ($check.ok) { return $identity } } catch { diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index 6f4c66155..e766b502f 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -443,7 +443,19 @@ async def _try_auto_repair_background( # Agent99 is the only dispatch owner for host control-plane recovery. # The generic Ansible catalog must not manufacture candidates from # incidental words such as 110, backup, Docker, or container. - route_id = str(handoff.get("route_id") or "") + policy_route_id = str(handoff.get("route_id") or "") + route_id = policy_route_id + if ( + handoff.get("single_writer_executor") == "Agent99" + and durable_agent99_route is not None + ): + route_id = str(durable_agent99_route["route_id"]) + handoff.update({ + "policy_route_id": policy_route_id, + "route_id": route_id, + "agent99_kind": durable_agent99_route["kind"], + "agent99_mode": durable_agent99_route["suggested_mode"], + }) dispatch_result = await bridge_alertmanager_to_agent99( alert_id=source_alert_id, alertname=source_alertname or alert_type, diff --git a/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py index bb43f5faa..55ae37c7a 100644 --- a/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py +++ b/apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py @@ -613,7 +613,7 @@ async def reconcile_agent99_controlled_dispatches_once( else {} ) } - if is_cold_start_same_run_identity(identity): + if is_cold_start_same_run_identity(identity, receipt): source_receipt = await asyncio.to_thread( read_cold_start_source_resolution, ) @@ -637,6 +637,7 @@ async def reconcile_agent99_controlled_dispatches_once( request_agent99_same_run_status_reconcile, identity, source_receipt, + receipt, ) if request_receipt.get("accepted") is True: counters["status_reconcile_requested"] += 1 diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index bb6af49e5..f345d5e37 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -343,6 +343,15 @@ def build_agent99_dispatch_receipt_envelope( else "dispatch_delivery_unknown_reconcile_only" ), "identity": public_identity, + "dispatch_scope": { + "schema_version": "agent99_dispatch_scope_v1", + "kind": str(dispatch_receipt.get("kind") or ""), + "suggested_mode": str(dispatch_receipt.get("suggested_mode") or ""), + "target_resource": str(dispatch_receipt.get("target_resource") or ""), + "controlled_apply_requested": bool( + dispatch_receipt.get("controlled_apply_requested") is True + ), + }, "dispatch_receipt": { "schema_version": str( dispatch_receipt.get("schema_version") @@ -493,6 +502,13 @@ class PostgresAgent99DispatchLedger: "schema_version": "agent99_controlled_dispatch_receipt_v1", "status": "dispatch_reserved", "identity": identity_payload, + "dispatch_scope": { + "schema_version": "agent99_dispatch_scope_v1", + "kind": input_payload["kind"], + "suggested_mode": input_payload["suggested_mode"], + "target_resource": input_payload["target_resource"], + "controlled_apply_requested": input_payload["controlled_apply"], + }, "dispatch_accepted": False, "controlled_apply_requested": bool(input_payload["controlled_apply"]), "controlled_apply_authorized": False, diff --git a/apps/api/src/services/agent99_same_run_reconcile.py b/apps/api/src/services/agent99_same_run_reconcile.py index bc16244fc..4e0142809 100644 --- a/apps/api/src/services/agent99_same_run_reconcile.py +++ b/apps/api/src/services/agent99_same_run_reconcile.py @@ -13,6 +13,7 @@ from __future__ import annotations import json import math +import re import time import urllib.error import urllib.parse @@ -82,16 +83,51 @@ _COLD_START_QUERIES = { } -def is_cold_start_same_run_identity(identity: Agent99DispatchIdentity) -> bool: - """Limit this transport to the existing cold-start Recover identity.""" +def is_cold_start_same_run_identity( + identity: Agent99DispatchIdentity, + receipt: dict[str, Any] | None = None, +) -> bool: + """Admit only a canonical cold-start Recover dispatch to no-write Status.""" - return bool( + legacy_identity = bool( identity.route_id == AGENT99_COLD_START_ROUTE_ID and identity.incident_id == AGENT99_COLD_START_INCIDENT_ID and str(identity.run_id) == AGENT99_COLD_START_RUN_ID and identity.source_fingerprint.startswith("legacy-cold-start:") and identity.execution_generation == "1" ) + if legacy_identity: + return True + + scope = ( + receipt.get("dispatch_scope") + if isinstance(receipt, dict) + and isinstance(receipt.get("dispatch_scope"), dict) + else {} + ) + try: + generation = int(identity.execution_generation) + except ValueError: + return False + expected_work_item_id = ( + f"agent99-dispatch:awoooi:{identity.incident_id}:" + f"{AGENT99_COLD_START_ROUTE_ID}" + ) + return bool( + identity.project_id == "awoooi" + and identity.route_id == AGENT99_COLD_START_ROUTE_ID + and re.fullmatch(r"INC-[0-9]{8}-[0-9A-F]{6}", identity.incident_id) + is not None + and identity.work_item_id == expected_work_item_id + and re.fullmatch(r"[0-9a-f]{32,64}", identity.source_fingerprint) + is not None + and 1 <= generation <= 3 + and scope.get("schema_version") == "agent99_dispatch_scope_v1" + and scope.get("kind") == "host_recovery" + and scope.get("suggested_mode") == "Recover" + and scope.get("target_resource") == "cold-start-gate" + and scope.get("controlled_apply_requested") is True + ) def agent99_same_run_evidence_id(identity: Agent99DispatchIdentity) -> str: @@ -235,6 +271,7 @@ def read_cold_start_source_resolution( def request_agent99_same_run_status_reconcile( identity: Agent99DispatchIdentity, source_receipt: dict[str, Any], + dispatch_receipt: dict[str, Any] | None = None, *, relay_url: str | None = None, relay_token: str | None = None, @@ -242,7 +279,7 @@ def request_agent99_same_run_status_reconcile( ) -> dict[str, Any]: """Ask Agent99 for one idempotent, no-write Status on the existing run.""" - if not is_cold_start_same_run_identity(identity): + if not is_cold_start_same_run_identity(identity, dispatch_receipt): return { "status": "identity_not_eligible_fail_closed", "accepted": False, diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 13cde411d..a0ffeda4c 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -838,6 +838,14 @@ def dispatch_agent99_sre_alert_with_receipt( alert_id = _safe_text(payload.get("id"), max_length=160) kind = _safe_text(payload.get("kind"), max_length=120) + awoooi = ( + payload.get("awoooi") + if isinstance(payload.get("awoooi"), dict) + else {} + ) + target_resource = _safe_text(awoooi.get("targetResource"), max_length=240) + suggested_mode = _safe_text(payload.get("suggestedMode"), max_length=80) + controlled_apply_requested = bool(payload.get("controlledApply") is True) if settings.AGENT99_SRE_ALERT_RELAY_URL: raw_receipt = post_agent99_sre_alert(payload) or {} response_payload: dict[str, Any] = {} @@ -862,6 +870,9 @@ def dispatch_agent99_sre_alert_with_receipt( "transport": "relay", "alert_id": alert_id, "kind": kind, + "target_resource": target_resource, + "suggested_mode": suggested_mode, + "controlled_apply_requested": controlled_apply_requested, "http_status": http_status, "accepted": accepted, "inbox_triggered": inbox_triggered, @@ -884,6 +895,9 @@ def dispatch_agent99_sre_alert_with_receipt( "transport": "file", "alert_id": alert_id, "kind": kind, + "target_resource": target_resource, + "suggested_mode": suggested_mode, + "controlled_apply_requested": controlled_apply_requested, "accepted": True, "inbox_triggered": False, # A file write is not proof that the Agent99 inbox consumed it. @@ -903,6 +917,9 @@ def dispatch_agent99_sre_alert_with_receipt( "transport": "disabled", "alert_id": alert_id, "kind": kind, + "target_resource": target_resource, + "suggested_mode": suggested_mode, + "controlled_apply_requested": controlled_apply_requested, "accepted": False, "inbox_triggered": False, "delivery_certainty": "not_delivered", diff --git a/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py index bbca57c5e..e925d0e55 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py @@ -37,6 +37,10 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() -> identity=identity, dispatch_receipt={ "status": "accepted_inbox_pending", + "kind": "host_recovery", + "suggested_mode": "Recover", + "target_resource": "cold-start-gate", + "controlled_apply_requested": True, "accepted": True, "inbox_triggered": False, "delivery_certainty": "unknown", @@ -48,6 +52,13 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() -> assert envelope["dispatch_promoted"] is False assert envelope["controlled_apply_authorized"] is False assert envelope["retry_policy"]["reconcile_only"] is True + assert envelope["dispatch_scope"] == { + "schema_version": "agent99_dispatch_scope_v1", + "kind": "host_recovery", + "suggested_mode": "Recover", + "target_resource": "cold-start-gate", + "controlled_apply_requested": True, + } @pytest.mark.asyncio diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index 4fae7764b..44d246dbb 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -128,6 +128,44 @@ def test_cold_start_source_queries_include_host112_scope() -> None: assert 'scope="110_120_121_188"' not in query +def test_live_cold_start_identity_requires_canonical_dispatch_scope() -> None: + identity = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-20260715-A1B2C3", + source_fingerprint="a" * 32, + route_id="agent99:host_recovery:Recover", + work_item_id=( + "agent99-dispatch:awoooi:INC-20260715-A1B2C3:" + "agent99:host_recovery:Recover" + ), + ) + scope = { + "dispatch_scope": { + "schema_version": "agent99_dispatch_scope_v1", + "kind": "host_recovery", + "suggested_mode": "Recover", + "target_resource": "cold-start-gate", + "controlled_apply_requested": True, + } + } + + assert reconcile.is_cold_start_same_run_identity(identity, scope) is True + wrong_target = deepcopy(scope) + wrong_target["dispatch_scope"]["target_resource"] = "other-host" + assert reconcile.is_cold_start_same_run_identity(identity, wrong_target) is False + wrong_work_item = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-20260715-A1B2C3", + source_fingerprint="a" * 32, + route_id="agent99:host_recovery:Recover", + work_item_id="agent99-dispatch:awoooi:INC-20260715-A1B2C3:other", + ) + assert ( + reconcile.is_cold_start_same_run_identity(wrong_work_item, scope) is False + ) + assert reconcile.is_cold_start_same_run_identity(identity, {}) is False + + def test_cold_start_source_resolution_allows_warning_only_but_requires_fresh_zero( monkeypatch, ) -> None: @@ -301,6 +339,48 @@ def test_same_run_status_request_rejects_unresolved_source_without_transport( } +def test_live_same_run_request_carries_dispatch_scope_into_identity_gate( + monkeypatch, +) -> None: + identity = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-20260715-D4E5F6", + source_fingerprint="b" * 64, + route_id="agent99:host_recovery:Recover", + ) + dispatch_receipt = { + "dispatch_scope": { + "schema_version": "agent99_dispatch_scope_v1", + "kind": "host_recovery", + "suggested_mode": "Recover", + "target_resource": "cold-start-gate", + "controlled_apply_requested": True, + } + } + monkeypatch.setattr( + reconcile.urllib.request, + "urlopen", + lambda *_args, **_kwargs: pytest.fail("transport must not run"), + ) + + eligible = reconcile.request_agent99_same_run_status_reconcile( + identity, + {"resolved": False, "receipt_ref": None}, + dispatch_receipt, + relay_url="http://agent99.invalid/agent99/sre-alert/", + relay_token="configured", + ) + missing_scope = reconcile.request_agent99_same_run_status_reconcile( + identity, + {"resolved": False, "receipt_ref": None}, + relay_url="http://agent99.invalid/agent99/sre-alert/", + relay_token="configured", + ) + + assert eligible["status"] == "source_not_resolved_fail_closed" + assert missing_scope["status"] == "identity_not_eligible_fail_closed" + + @pytest.mark.parametrize( ("relay_complete", "remove_runtime_field", "invalid_generation_type"), [(False, False, False), (True, True, False), (True, False, True)], @@ -357,14 +437,15 @@ class _Ledger: def __init__(self) -> None: self.verifier_calls: list[dict[str, object]] = [] self.external_calls: list[dict[str, object]] = [] + self.list_receipt: dict[str, object] = { + "status": "dispatch_accepted_verifier_pending", + "post_verifier_passed": False, + } async def list_reconcilable(self, **_kwargs): # type: ignore[no-untyped-def] return [{ "identity": IDENTITY, - "receipt": { - "status": "dispatch_accepted_verifier_pending", - "post_verifier_passed": False, - }, + "receipt": self.list_receipt, }] async def record_verifier(self, **kwargs): # type: ignore[no-untyped-def] @@ -392,7 +473,7 @@ async def test_reconciler_requests_status_only_after_production_source_resolves( monkeypatch, ) -> None: ledger = _Ledger() - requests: list[tuple[object, object]] = [] + requests: list[tuple[object, object, object]] = [] monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: None) monkeypatch.setattr( @@ -403,7 +484,9 @@ async def test_reconciler_requests_status_only_after_production_source_resolves( monkeypatch.setattr( job, "request_agent99_same_run_status_reconcile", - lambda identity, source: requests.append((identity, source)) + lambda identity, source, receipt: requests.append( + (identity, source, receipt) + ) or {"accepted": True}, ) @@ -411,7 +494,11 @@ async def test_reconciler_requests_status_only_after_production_source_resolves( assert result["status_reconcile_requested"] == 1 assert result["verifier_written"] == 0 - assert requests == [(IDENTITY, _source_receipt())] + assert requests == [( + IDENTITY, + _source_receipt(), + ledger.list_receipt, + )] assert ledger.verifier_calls == [] @@ -481,14 +568,16 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity( "identity": other.public_dict(), }, } - requests: list[tuple[object, object]] = [] + requests: list[tuple[object, object, object]] = [] monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome) monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source) monkeypatch.setattr( job, "request_agent99_same_run_status_reconcile", - lambda identity, receipt: requests.append((identity, receipt)) + lambda identity, source, receipt: requests.append( + (identity, source, receipt) + ) or {"accepted": True}, ) @@ -496,7 +585,7 @@ async def test_reconciler_rejects_status_outcome_with_mismatched_identity( assert result["status_reconcile_requested"] == 1 assert result["verifier_written"] == 0 - assert requests == [(IDENTITY, source)] + assert requests == [(IDENTITY, source, ledger.list_receipt)] assert ledger.verifier_calls == [] @@ -543,14 +632,16 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes( source = _source_receipt() outcome = deepcopy(_valid_outcome()) _mutate_path(outcome, path, value) - requests: list[tuple[object, object]] = [] + requests: list[tuple[object, object, object]] = [] monkeypatch.setattr(job, "get_agent99_dispatch_ledger", lambda: ledger) monkeypatch.setattr(job, "read_agent99_sre_outcome", lambda _run: outcome) monkeypatch.setattr(job, "read_cold_start_source_resolution", lambda: source) monkeypatch.setattr( job, "request_agent99_same_run_status_reconcile", - lambda identity, receipt: requests.append((identity, receipt)) + lambda identity, source, receipt: requests.append( + (identity, source, receipt) + ) or {"accepted": True}, ) finalize = AsyncMock(side_effect=AssertionError("learning must not run")) @@ -561,7 +652,7 @@ async def test_reconciler_rejects_synthetic_bad_same_run_outcomes( assert result["status_reconcile_requested"] == 1 assert result["external_no_write_reconciled"] == 0 assert result["verifier_written"] == 0 - assert requests == [(IDENTITY, source)] + assert requests == [(IDENTITY, source, ledger.list_receipt)] assert ledger.external_calls == [] assert ledger.verifier_calls == [] finalize.assert_not_awaited() @@ -738,22 +829,21 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert "agent99_same_run_status_reconcile_v1" in relay assert "function Test-AgentSameRunDispatchIdentity" in relay assert 'route_id" "") -ne "agent99:host_recovery:Recover"' in relay - assert 'incident_id" "") -ne "INC-20260711-11C751"' in relay - assert 'runId -ne "bf30ca01-1080-5725-b2d1-3e1534dfc811"' in relay - assert 'execution_generation" "") -ne "1"' in relay assert 'project_id" "") -ne "awoooi"' in relay - assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay + assert '$incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$"' in relay + assert '$executionGeneration -match "^[1-3]$"' in relay + assert '$sourceFingerprint -match "^[0-9a-f]{32,64}$"' in relay + assert 'function Test-AgentStoredColdStartRecoverEnvelope' in relay assert "function Get-AgentExistingRecoverIdentity" in relay assert '$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"' in relay assert '$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"' in relay assert '$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)' in relay assert 'Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern' in relay assert 'Get-AgentField $awoooi "agent99DispatchIdentity"' in relay - assert 'Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId"' in relay - assert 'Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager"' in relay - assert 'Get-AgentField $stored "kind" "") -ne "host_recovery"' in relay - assert 'Get-AgentField $stored "service" "") -ne "cold-start-gate"' in relay + assert '$source -eq "awoooi-api-alertmanager"' in relay + assert '$kind -eq "host_recovery"' in relay + assert '$service -eq "cold-start-gate"' in relay assert "existing_dispatch_identity_not_found" in relay assert "existing_dispatch_identity_mismatch" in relay assert "function Test-AgentSameRunOutcomeEligible" in relay @@ -782,16 +872,15 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert 'reason = "same_run_reconcile_exact_queue_contract_required"' in control assert "$canonicalDigestMatches" in control assert "$sameRunReconcileContractExact" in control - assert '$incidentId -eq "INC-20260711-11C751"' in control - assert ( - '$automationRunId -eq "bf30ca01-1080-5725-b2d1-3e1534dfc811"' - in control - ) + assert "$automationRunIdValid" in control + assert "$legacyColdStartIdentity" in control + assert "$liveColdStartIdentity" in control + assert "$coldStartIdentityEligible" in control assert '$dispatchRouteId -eq "agent99:host_recovery:Recover"' in control - assert '$executionGeneration -eq "1"' in control assert '$lockOwner -eq $automationRunId' in control assert '$projectId -eq "awoooi"' in control - assert '$workItemId -eq "agent99-dispatch:awoooi:' in control + assert '$commandSource -eq "agent99-same-run-reconcile"' in control + assert '$alertSource -eq "awoooi-production-source-verifier"' in control assert '$traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in control assert '$reconcileEvidenceId -eq $expectedReconcileEvidenceId' in control assert "function Get-AgentEvidenceAtPath" in control @@ -819,7 +908,7 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> / "apps/api/src/jobs/agent99_controlled_dispatch_reconciler_job.py" ).read_text(encoding="utf-8") cold_branch = reconcile_job[ - reconcile_job.index("if is_cold_start_same_run_identity(identity):") : + reconcile_job.index("if is_cold_start_same_run_identity(identity, receipt):") : reconcile_job.index( 'if receipt.get("post_verifier_passed") is not True:' ) diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index 3a69cbe9c..0f6d461ce 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -362,6 +362,9 @@ def test_dispatch_receipt_keeps_relay_acceptance_without_raw_body(monkeypatch) - "transport": "relay", "alert_id": "awoooi-alertmanager-receipt-smoke", "kind": "host_recovery", + "target_resource": "cold-start-gate", + "suggested_mode": "Recover", + "controlled_apply_requested": True, "http_status": 202, "accepted": True, "inbox_triggered": True, diff --git a/apps/api/tests/test_controlled_alert_target_router.py b/apps/api/tests/test_controlled_alert_target_router.py index 9ba973f3e..3496425a0 100644 --- a/apps/api/tests/test_controlled_alert_target_router.py +++ b/apps/api/tests/test_controlled_alert_target_router.py @@ -225,6 +225,13 @@ async def test_webhook_router_never_queues_generic_ansible_for_cold_start( incident_lookup.assert_not_awaited() bridge.assert_awaited_once() assert handoff["single_writer_executor"] == "Agent99" + assert handoff["policy_route_id"] == "agent99_recover_after_owner_review" + assert handoff["route_id"] == "agent99:host_recovery:Recover" + assert bridge.await_args.kwargs["route_id"] == "agent99:host_recovery:Recover" + assert bridge.await_args.kwargs["work_item_id"] == ( + "agent99-dispatch:awoooi:INC-20260711-11C751:" + "agent99:host_recovery:Recover" + ) assert handoff["execution_priority"] == 30 assert handoff["queued"] is True assert handoff["runtime_execution_authorized"] is False