diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index 52d5e6d09..66c0e48ad 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -73,6 +73,7 @@ function Write-AgentRelayQueueReceipt { $awoooi = Get-AgentField $Alert "awoooi" $null $identity = Get-AgentField $awoooi "agent99DispatchIdentity" $null $routing = Get-AgentField $Alert "routing" $null + $typedRoute = Get-AgentField $routing "typedTargetRoute" $null $payload = [pscustomobject]@{ schemaVersion = "agent99_sre_queue_receipt_v1" receiptId = $receiptId @@ -86,6 +87,14 @@ function Write-AgentRelayQueueReceipt { traceId = [string](Get-AgentField $identity "trace_id" (Get-AgentField $routing "traceId" "")) workItemId = [string](Get-AgentField $identity "work_item_id" (Get-AgentField $routing "workItemId" "")) executionGeneration = [string](Get-AgentField $identity "execution_generation" (Get-AgentField $routing "executionGeneration" "")) + dispatchScope = [pscustomobject]@{ + schemaVersion = "agent99_dispatch_scope_v1" + canonicalAssetId = [string](Get-AgentField $typedRoute "canonical_asset_id" "") + typedDomain = [string](Get-AgentField $typedRoute "target_kind" "") + executor = [string](Get-AgentField $typedRoute "executor" "") + verifier = [string](Get-AgentField $typedRoute "verifier" "") + routeId = [string](Get-AgentField $identity "route_id" "") + } generatedAt = (Get-Date -Format o) storesRawPayload = $false } @@ -142,6 +151,7 @@ function Resolve-AgentAlertRoute { } $kindModes = @{ + "agent99_control_plane_health" = "SelfCheck" "provider_freshness_signal" = "ProviderFreshness" "backup_health" = "BackupCheck" "backup_missing" = "BackupCheck" @@ -322,6 +332,162 @@ function Convert-AgentInventoryHostToken { return "" } +function Test-Agent99ControlPlaneHealthEnvelope { + param( + [object]$Alert, + [string]$ModeName, + [bool]$ControlledApply + ) + + if ($ModeName -ne "SelfCheck") { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_selfcheck_only" } + } + if ($ControlledApply) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_write_forbidden" } + } + if (Convert-AgentBool (Get-AgentField $Alert "force" $false)) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_force_forbidden" } + } + + $routing = Get-AgentField $Alert "routing" $null + $typedRoute = Get-AgentField $routing "typedTargetRoute" $null + $actionPolicy = Get-AgentField $routing "actionPolicy" $null + if (-not $routing -or -not $typedRoute -or -not $actionPolicy) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_scope_missing" } + } + if ( + [string](Get-AgentField $routing "schemaVersion" "") -ne "agent99_alert_route_v1" -or + [string](Get-AgentField $typedRoute "schema_version" "") -ne "typed_domain_target_route_v2" -or + [string](Get-AgentField $actionPolicy "schemaVersion" "") -ne "agent99_control_plane_action_policy_v1" + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_schema_invalid" } + } + + $allowedPolicyModes = @(Get-AgentField $actionPolicy "allowedModes" @()) + if ($allowedPolicyModes.Count -ne 1 -or [string]$allowedPolicyModes[0] -ne "SelfCheck") { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_action_policy_invalid" } + } + foreach ($field in @( + "controlledApplyAllowed", + "rebootAllowed", + "vmPowerChangeAllowed", + "arbitraryShellAllowed", + "crossHostFallbackAllowed", + "crossDomainFallbackAllowed" + )) { + $value = Get-AgentField $actionPolicy $field $null + if (-not ($value -is [bool]) -or [bool]$value) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_action_policy_invalid" } + } + } + + $typedDispatchAllowed = Get-AgentField $routing "typedDispatchAllowed" $null + $routingCrossDomain = Get-AgentField $routing "crossDomainFallbackAllowed" $null + $routeCrossDomain = Get-AgentField $typedRoute "cross_domain_fallback_allowed" $null + $controlledApplyAllowed = Get-AgentField $typedRoute "controlled_apply_allowed" $null + if (-not ($typedDispatchAllowed -is [bool]) -or -not [bool]$typedDispatchAllowed) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_dispatch_not_allowed" } + } + if (-not ($controlledApplyAllowed -is [bool]) -or [bool]$controlledApplyAllowed) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_readonly_route_required" } + } + if ( + -not ($routingCrossDomain -is [bool]) -or + [bool]$routingCrossDomain -or + -not ($routeCrossDomain -is [bool]) -or + [bool]$routeCrossDomain + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_cross_domain_forbidden" } + } + + $canonicalAssetId = [string](Get-AgentField $typedRoute "canonical_asset_id" "") + $targetKind = [string](Get-AgentField $typedRoute "target_kind" "") + $executor = [string](Get-AgentField $typedRoute "executor" "") + $verifier = [string](Get-AgentField $typedRoute "verifier" "") + if ( + $canonicalAssetId -ne "host_99" -or + $targetKind -ne "windows_vmware" -or + $executor -ne "Agent99" -or + $verifier -ne "agent99_control_plane_selfcheck_independent_verifier" + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_exact_asset_required" } + } + + $allowedInventoryHosts = @( + @(Get-AgentField $typedRoute "allowed_inventory_hosts" @()) | + ForEach-Object { Convert-AgentInventoryHostToken $_ } | + Where-Object { $_ } | + Select-Object -Unique + ) + $routeHostScope = Convert-AgentInventoryHostToken (Get-AgentField $typedRoute "host" "") + $alertHostText = [string](Get-AgentField $Alert "host" "") + $alertHostScope = Convert-AgentInventoryHostToken $alertHostText + if ( + $allowedInventoryHosts.Count -ne 1 -or + [string]$allowedInventoryHosts[0] -ne "host_99" -or + $routeHostScope -ne "host_99" -or + ($alertHostText.Trim() -and $alertHostScope -ne "host_99") + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_cross_host_forbidden" } + } + + $awoooiMetadata = Get-AgentField $Alert "awoooi" $null + $dispatchIdentity = Get-AgentField $awoooiMetadata "agent99DispatchIdentity" $null + if ( + -not $awoooiMetadata -or + [string](Get-AgentField $awoooiMetadata "canonicalAssetId" "") -ne "host_99" -or + [string](Get-AgentField $awoooiMetadata "typedDomain" "") -ne "windows_vmware" -or + [string](Get-AgentField $awoooiMetadata "assetResolutionStatus" "") -ne "resolved" -or + -not $dispatchIdentity -or + [string](Get-AgentField $dispatchIdentity "schema_version" "") -ne "agent99_controlled_dispatch_identity_v1" -or + [string](Get-AgentField $dispatchIdentity "route_id" "") -ne "agent99:agent99_control_plane_health:SelfCheck" + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_identity_invalid" } + } + foreach ($field in @( + "project_id", + "incident_id", + "source_fingerprint", + "execution_generation", + "run_id", + "trace_id", + "work_item_id", + "idempotency_key", + "single_flight_key", + "lock_owner", + "canonical_digest" + )) { + if (-not ([string](Get-AgentField $dispatchIdentity $field "")).Trim()) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_identity_invalid" } + } + } + if (-not $dispatchIdentity.PSObject.Properties["approval_id"]) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_identity_invalid" } + } + foreach ($pair in @( + [pscustomobject]@{ identity = "run_id"; projection = "runId" }, + [pscustomobject]@{ identity = "trace_id"; projection = "traceId" }, + [pscustomobject]@{ identity = "work_item_id"; projection = "workItemId" }, + [pscustomobject]@{ identity = "idempotency_key"; projection = "idempotencyKey" }, + [pscustomobject]@{ identity = "execution_generation"; projection = "executionGeneration" } + )) { + if ( + [string](Get-AgentField $routing $pair.projection "") -ne + [string](Get-AgentField $dispatchIdentity $pair.identity "") + ) { + return [pscustomobject]@{ ok = $false; reason = "agent99_control_plane_identity_projection_mismatch" } + } + } + + return [pscustomobject]@{ + ok = $true + reason = "agent99_control_plane_selfcheck_scope_verified" + targetKind = $targetKind + canonicalAssetId = $canonicalAssetId + inventoryHost = "host_99" + } +} + function Test-AgentTypedDispatchEnvelope { param( [object]$Alert, @@ -329,6 +495,10 @@ function Test-AgentTypedDispatchEnvelope { [bool]$ControlledApply ) + $kind = ([string](Get-AgentField $Alert "kind" "")).Trim().ToLowerInvariant() + if ($kind -eq "agent99_control_plane_health") { + return Test-Agent99ControlPlaneHealthEnvelope $Alert $ModeName $ControlledApply + } if ($ModeName -notin $remediationModes) { return [pscustomobject]@{ ok = $true; reason = "no_mutating_dispatch" } } @@ -654,6 +824,25 @@ function Invoke-AgentAlertRoutingSelfTest { message = "AlertChainBroken monitoring pipeline down" } }, + [pscustomobject]@{ + name = "agent99 control plane selfcheck" + expectedActionable = $true + expectedMode = "SelfCheck" + expectedRouteSource = "suggestedMode" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-agent99-control-plane" + source = "awoooi-api-alertmanager" + severity = "critical" + kind = "agent99_control_plane_health" + suggestedMode = "SelfCheck" + controlledApply = $false + force = $false + service = "Agent99" + host = "192.168.0.99" + message = "Agent99 control plane health check failed" + } + }, [pscustomobject]@{ name = "provider freshness" expectedActionable = $true @@ -998,6 +1187,7 @@ foreach ($file in $files) { $routing = Get-AgentField $alert "routing" $null $awoooiMetadata = Get-AgentField $alert "awoooi" $null $dispatchIdentity = Get-AgentField $awoooiMetadata "agent99DispatchIdentity" $null + $typedTargetRoute = Get-AgentField $routing "typedTargetRoute" $null $automationRunId = [string](Get-AgentField $dispatchIdentity "run_id" (Get-AgentField $routing "runId" "")) $traceId = [string](Get-AgentField $dispatchIdentity "trace_id" (Get-AgentField $routing "traceId" "")) $workItemId = [string](Get-AgentField $dispatchIdentity "work_item_id" (Get-AgentField $routing "workItemId" "")) @@ -1011,6 +1201,14 @@ foreach ($file in $files) { $singleFlightKey = [string](Get-AgentField $dispatchIdentity "single_flight_key" "") $lockOwner = [string](Get-AgentField $dispatchIdentity "lock_owner" "") $canonicalDigest = [string](Get-AgentField $dispatchIdentity "canonical_digest" "") + $dispatchScope = [pscustomobject]@{ + schemaVersion = "agent99_dispatch_scope_v1" + canonicalAssetId = [string](Get-AgentField $typedTargetRoute "canonical_asset_id" "") + typedDomain = [string](Get-AgentField $typedTargetRoute "target_kind" "") + executor = [string](Get-AgentField $typedTargetRoute "executor" "") + verifier = [string](Get-AgentField $typedTargetRoute "verifier" "") + routeId = $dispatchRouteId + } $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey) $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $executionGeneration) if (($controlled -and $modeName -in $remediationModes -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) { @@ -1067,6 +1265,7 @@ foreach ($file in $files) { singleFlightKey = $singleFlightKey lockOwner = $lockOwner canonicalDigest = $canonicalDigest + dispatchScope = $dispatchScope createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 @@ -1108,6 +1307,7 @@ foreach ($file in $files) { singleFlightKey = $singleFlightKey lockOwner = $lockOwner canonicalDigest = $canonicalDigest + dispatchScope = $dispatchScope createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 @@ -1159,6 +1359,7 @@ foreach ($file in $files) { executionGeneration = $executionGeneration incidentId = $incidentId approvalId = $approvalId + dispatchScope = $dispatchScope requestPath = $requestPath queuePath = $queuePath alertPath = $processedPath diff --git a/apps/api/src/services/agent99_controlled_dispatch_ledger.py b/apps/api/src/services/agent99_controlled_dispatch_ledger.py index e64ac799a..dae084efb 100644 --- a/apps/api/src/services/agent99_controlled_dispatch_ledger.py +++ b/apps/api/src/services/agent99_controlled_dispatch_ledger.py @@ -303,6 +303,36 @@ def attach_agent99_dispatch_identity( return enriched +def build_agent99_dispatch_scope( + *, + identity: Agent99DispatchIdentity, + kind: Any, + suggested_mode: Any, + target_resource: Any, + controlled_apply_requested: Any, + canonical_asset_id: Any = "", + typed_domain: Any = "", + executor: Any = "", + verifier: Any = "", +) -> dict[str, Any]: + """Build the bounded typed scope persisted with one durable run.""" + + return { + "schema_version": "agent99_dispatch_scope_v1", + "kind": str(kind or "")[:120], + "suggested_mode": str(suggested_mode or "")[:80], + "target_resource": str(target_resource or "")[:240], + "controlled_apply_requested": bool(controlled_apply_requested is True), + "canonical_asset_id": str(canonical_asset_id or "")[:240], + "typed_domain": str(typed_domain or "")[:120], + "executor": str(executor or "")[:160], + "verifier": str(verifier or "")[:160], + # Never trust a transport-supplied route projection. The identity is + # the same-run source of truth for route binding. + "route_id": identity.route_id, + } + + def build_agent99_dispatch_receipt_envelope( *, identity: Agent99DispatchIdentity, @@ -337,6 +367,24 @@ def build_agent99_dispatch_receipt_envelope( and delivery_certainty == "not_delivered" ) public_identity = identity.public_dict() + supplied_scope = ( + dispatch_receipt.get("dispatch_scope") + if isinstance(dispatch_receipt.get("dispatch_scope"), dict) + else {} + ) + dispatch_scope = build_agent99_dispatch_scope( + identity=identity, + kind=dispatch_receipt.get("kind"), + suggested_mode=dispatch_receipt.get("suggested_mode"), + target_resource=dispatch_receipt.get("target_resource"), + controlled_apply_requested=dispatch_receipt.get( + "controlled_apply_requested" + ), + canonical_asset_id=supplied_scope.get("canonical_asset_id"), + typed_domain=supplied_scope.get("typed_domain"), + executor=supplied_scope.get("executor"), + verifier=supplied_scope.get("verifier"), + ) stage_identity = { "run_id": str(identity.run_id), "trace_id": identity.trace_id, @@ -352,15 +400,7 @@ def build_agent99_dispatch_receipt_envelope( else "dispatch_delivery_unknown_reconcile_only" ), "identity": public_identity, - "dispatch_scope": { - "schema_version": "agent99_dispatch_scope_v1", - "kind": str(dispatch_receipt.get("kind") or ""), - "suggested_mode": str(dispatch_receipt.get("suggested_mode") or ""), - "target_resource": str(dispatch_receipt.get("target_resource") or ""), - "controlled_apply_requested": bool( - dispatch_receipt.get("controlled_apply_requested") is True - ), - }, + "dispatch_scope": dispatch_scope, "dispatch_receipt": { "schema_version": str( dispatch_receipt.get("schema_version") @@ -508,13 +548,40 @@ class PostgresAgent99DispatchLedger: payload: dict[str, Any], ) -> dict[str, Any]: identity_payload = identity.public_dict() + routing = ( + payload.get("routing") + if isinstance(payload.get("routing"), dict) + else {} + ) + typed_target_route = ( + routing.get("typedTargetRoute") + if isinstance(routing.get("typedTargetRoute"), dict) + else {} + ) + awoooi = ( + payload.get("awoooi") + if isinstance(payload.get("awoooi"), dict) + else {} + ) + dispatch_scope = build_agent99_dispatch_scope( + identity=identity, + kind=payload.get("kind"), + suggested_mode=payload.get("suggestedMode") or "Status", + target_resource=awoooi.get("targetResource"), + controlled_apply_requested=payload.get("controlledApply"), + canonical_asset_id=typed_target_route.get("canonical_asset_id"), + typed_domain=typed_target_route.get("target_kind"), + executor=typed_target_route.get("executor"), + verifier=typed_target_route.get("verifier"), + ) input_payload = { "schema_version": "agent99_controlled_dispatch_input_v1", "identity": identity_payload, "kind": str(payload.get("kind") or ""), "suggested_mode": str(payload.get("suggestedMode") or "Status"), "controlled_apply": bool(payload.get("controlledApply") is True), - "target_resource": str((payload.get("awoooi") or {}).get("targetResource") or ""), + "target_resource": str(awoooi.get("targetResource") or ""), + "dispatch_scope": dispatch_scope, "stores_secrets": False, } input_json = _stable_json(input_payload) @@ -529,13 +596,7 @@ class PostgresAgent99DispatchLedger: "schema_version": "agent99_controlled_dispatch_receipt_v1", "status": "dispatch_reserved", "identity": identity_payload, - "dispatch_scope": { - "schema_version": "agent99_dispatch_scope_v1", - "kind": input_payload["kind"], - "suggested_mode": input_payload["suggested_mode"], - "target_resource": input_payload["target_resource"], - "controlled_apply_requested": input_payload["controlled_apply"], - }, + "dispatch_scope": dispatch_scope, "dispatch_accepted": False, "controlled_apply_requested": bool(input_payload["controlled_apply"]), "controlled_apply_authorized": False, diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 143b77b97..17712f16b 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -29,7 +29,13 @@ from src.utils.timezone import now_taipei logger = get_logger("awoooi.agent99_sre_bridge") AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600 +AGENT99_CONTROL_PLANE_HEALTH_KIND = "agent99_control_plane_health" +AGENT99_CONTROL_PLANE_HEALTH_MODE = "SelfCheck" +AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID = ( + "agent99:agent99_control_plane_health:SelfCheck" +) AGENT99_DURABLE_ROUTE_KINDS = { + AGENT99_CONTROL_PLANE_HEALTH_KIND, "provider_freshness_signal", "public_route_502", "performance_pressure", @@ -116,6 +122,12 @@ def resolve_agent99_alert_kind( annotations=annotations or {}, ) + if ( + re.sub(r"[^a-z0-9]+", "", alertname.lower()) + == "agent99serviceunhealthy" + ): + return AGENT99_CONTROL_PLANE_HEALTH_KIND + # Explicit backup identities win over generic freshness words. Backup # failure cards intentionally mention freshness, but must stay in the # read-only BackupCheck lane with its stricter no-delete/no-restore policy. @@ -204,6 +216,7 @@ def resolve_agent99_alert_kind( def _suggested_mode_for_kind(kind: str) -> str | None: return { + AGENT99_CONTROL_PLANE_HEALTH_KIND: AGENT99_CONTROL_PLANE_HEALTH_MODE, "provider_freshness_signal": "ProviderFreshness", "public_route_502": "Recover", "performance_pressure": "Perf", @@ -254,6 +267,7 @@ def resolve_agent99_durable_route( def _resolution_policy_for_kind(kind: str) -> str: if kind in { + AGENT99_CONTROL_PLANE_HEALTH_KIND, "provider_freshness_signal", "public_route_502", "performance_pressure", @@ -479,6 +493,13 @@ def build_agent99_sre_alert( "LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot " "the host, change VM power, reset the VM, read secrets or change firewall." ) + elif kind == AGENT99_CONTROL_PLANE_HEALTH_KIND: + instruction = ( + "Run Agent99 SelfCheck only for canonical host_99 in the " + "windows_vmware domain; controlledApply=false; do not reboot, " + "change VM power, execute arbitrary shell, or fall back to another " + "host or domain." + ) payload: dict[str, Any] = { "id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}", @@ -490,7 +511,7 @@ def build_agent99_sre_alert( "title": _safe_text(f"Alertmanager {alertname}", max_length=240), "message": _safe_text(message, max_length=2000), "labels": safe_labels, - "force": True, + "force": kind != AGENT99_CONTROL_PLANE_HEALTH_KIND, "controlledApply": suggested_mode in { "Recover", @@ -552,6 +573,17 @@ def build_agent99_sre_alert( } if suggested_mode: payload["suggestedMode"] = suggested_mode + if kind == AGENT99_CONTROL_PLANE_HEALTH_KIND: + payload["routing"]["actionPolicy"] = { + "schemaVersion": "agent99_control_plane_action_policy_v1", + "allowedModes": [AGENT99_CONTROL_PLANE_HEALTH_MODE], + "controlledApplyAllowed": False, + "rebootAllowed": False, + "vmPowerChangeAllowed": False, + "arbitraryShellAllowed": False, + "crossHostFallbackAllowed": False, + "crossDomainFallbackAllowed": False, + } if instruction: payload["instruction"] = instruction return payload @@ -862,6 +894,55 @@ def dispatch_agent99_sre_alert(payload: dict[str, Any]) -> str: return str(receipt["transport"]) +def _agent99_dispatch_scope_from_payload( + payload: dict[str, Any], +) -> dict[str, Any]: + """Project the public-safe typed scope carried by one Agent99 run.""" + + routing = ( + payload.get("routing") + if isinstance(payload.get("routing"), dict) + else {} + ) + typed_target = ( + routing.get("typedTargetRoute") + if isinstance(routing.get("typedTargetRoute"), dict) + else {} + ) + awoooi = ( + payload.get("awoooi") + if isinstance(payload.get("awoooi"), dict) + else {} + ) + identity = ( + awoooi.get("agent99DispatchIdentity") + if isinstance(awoooi.get("agent99DispatchIdentity"), dict) + else {} + ) + return { + "schema_version": "agent99_dispatch_scope_v1", + "kind": _safe_text(payload.get("kind"), max_length=120), + "suggested_mode": _safe_text( + payload.get("suggestedMode"), max_length=80 + ), + "target_resource": _safe_text( + awoooi.get("targetResource"), max_length=240 + ), + "controlled_apply_requested": bool( + payload.get("controlledApply") is True + ), + "canonical_asset_id": _safe_text( + typed_target.get("canonical_asset_id"), max_length=240 + ), + "typed_domain": _safe_text( + typed_target.get("target_kind"), max_length=120 + ), + "executor": _safe_text(typed_target.get("executor"), max_length=160), + "verifier": _safe_text(typed_target.get("verifier"), max_length=160), + "route_id": _safe_text(identity.get("route_id"), max_length=240), + } + + def dispatch_agent99_sre_alert_with_receipt( payload: dict[str, Any], ) -> dict[str, Any]: @@ -877,6 +958,7 @@ def dispatch_agent99_sre_alert_with_receipt( target_resource = _safe_text(awoooi.get("targetResource"), max_length=240) suggested_mode = _safe_text(payload.get("suggestedMode"), max_length=80) controlled_apply_requested = bool(payload.get("controlledApply") is True) + dispatch_scope = _agent99_dispatch_scope_from_payload(payload) if settings.AGENT99_SRE_ALERT_RELAY_URL: raw_receipt = post_agent99_sre_alert(payload) or {} response_payload: dict[str, Any] = {} @@ -966,6 +1048,7 @@ def dispatch_agent99_sre_alert_with_receipt( "target_resource": target_resource, "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, + "dispatch_scope": dispatch_scope, "http_status": http_status, "transport_accepted": transport_accepted, "accepted": accepted, @@ -1001,6 +1084,7 @@ def dispatch_agent99_sre_alert_with_receipt( "target_resource": target_resource, "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, + "dispatch_scope": dispatch_scope, "transport_accepted": True, "accepted": False, "inbox_triggered": False, @@ -1031,6 +1115,7 @@ def dispatch_agent99_sre_alert_with_receipt( "target_resource": target_resource, "suggested_mode": suggested_mode, "controlled_apply_requested": controlled_apply_requested, + "dispatch_scope": dispatch_scope, "transport_accepted": False, "accepted": False, "inbox_triggered": False, @@ -1130,6 +1215,48 @@ async def bridge_alertmanager_to_agent99( str(route_id or "").strip() or f"agent99:{str(payload.get('kind') or 'monitoring_alert')}:{suggested_mode}" ) + if payload.get("kind") == AGENT99_CONTROL_PLANE_HEALTH_KIND: + typed_target_route = ( + routing.get("typedTargetRoute") + if isinstance(routing.get("typedTargetRoute"), dict) + else {} + ) + exact_control_plane_scope = bool( + suggested_mode == AGENT99_CONTROL_PLANE_HEALTH_MODE + and mutating is False + and payload.get("force") is False + and resolved_route_id == AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID + and typed_target_route.get("resolution_status") == "resolved" + and typed_target_route.get("canonical_asset_id") == "host_99" + and typed_target_route.get("target_kind") == "windows_vmware" + and typed_target_route.get("executor") == "Agent99" + and typed_target_route.get("verifier") + == "agent99_control_plane_selfcheck_independent_verifier" + and typed_target_route.get("cross_domain_fallback_allowed") + is False + ) + if not exact_control_plane_scope: + logger.warning( + "agent99_control_plane_scope_denied_fail_closed", + alert_id=alert_id, + alertname=alertname, + route_id=resolved_route_id, + canonical_asset_id=typed_target_route.get( + "canonical_asset_id" + ), + target_kind=typed_target_route.get("target_kind"), + reason="agent99_control_plane_exact_scope_required", + ) + return { + "status": "failed", + "reason": "agent99_control_plane_exact_scope_required", + "dispatchPerformed": False, + "runtimeClosureVerified": False, + "typedTargetRoute": typed_target_route, + "driftWorkItemId": typed_target_route.get( + "drift_work_item_id" + ), + } durable_route = bool( str(incident_id or "").strip() or str(route_id or "").strip() @@ -1453,7 +1580,19 @@ async def bridge_alertmanager_to_agent99( "status": "transport_exception", "transport": "unknown", "alert_id": str(dispatch_identity.run_id), - "kind": "host_recovery", + "kind": str(payload.get("kind") or "monitoring_alert"), + "suggested_mode": str( + payload.get("suggestedMode") or "Status" + ), + "target_resource": str( + (payload.get("awoooi") or {}).get("targetResource") or "" + ), + "controlled_apply_requested": bool( + payload.get("controlledApply") is True + ), + "dispatch_scope": _agent99_dispatch_scope_from_payload( + payload + ), "accepted": False, "inbox_triggered": False, "delivery_certainty": "unknown", diff --git a/apps/api/src/services/controlled_alert_target_router.py b/apps/api/src/services/controlled_alert_target_router.py index f8a8fb22b..b1e2da1fa 100644 --- a/apps/api/src/services/controlled_alert_target_router.py +++ b/apps/api/src/services/controlled_alert_target_router.py @@ -51,6 +51,7 @@ _WINDOWS_VMWARE_MARKERS = ( "vmware", "vmx", ) +_AGENT99_CONTROL_PLANE_HEALTH_ALERTS = {"agent99serviceunhealthy"} _DISK_ALERTS = { "hostoutofdiskspace", "hostdiskusagehigh", @@ -489,6 +490,53 @@ def resolve_typed_alert_target( execution_role="single_writer_host111_launchagent_executor", ) + if compact_alert in _AGENT99_CONTROL_PLANE_HEALTH_ALERTS: + # Agent99's own health signal is a read-only control-plane route. It + # is never permission to recover a guest, power a VM, reboot a host, + # or fall through to an arbitrary Windows command. An explicit host + # identity that is not host_99 is drift and must fail closed. + explicit_host_scope = _host_scope( + str(labels.get("host") or ""), + str(labels.get("instance") or ""), + str(labels.get("node") or ""), + target_resource, + ) + if ( + explicit_host_scope is not None + and explicit_host_scope != _HOSTS["99"] + ): + return _typed_route( + resolution_status="asset_identity_unresolved", + target_kind="unknown", + target_resource=target_resource, + namespace=namespace, + canonical_asset_id=None, + host=None, + executor=None, + verifier=None, + risk_class="high", + drift_work_item_id=_kubernetes_drift_work_item_id( + alertname, + target_resource, + explicit_host_scope[0], + ), + stateful_level=StatefulLevel.UNRESOLVED.value, + ) + return _typed_route( + resolution_status="resolved", + target_kind="windows_vmware", + target_resource=target_resource or "Agent99", + namespace=namespace, + canonical_asset_id="host_99", + host=_HOSTS["99"][1], + executor="Agent99", + verifier="agent99_control_plane_selfcheck_independent_verifier", + risk_class="high", + allowed_inventory_hosts=["host_99"], + controlled_apply_allowed=False, + execution_role="single_writer_agent99_control_plane_selfcheck_executor", + ) + if any(marker in text for marker in _BACKUP_RESTORE_MARKERS): identity = registry.resolve_identity(target_resource) generic_backup_signal = _compact_token(target_resource) in { diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py index c81da8324..ff815230b 100644 --- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -164,6 +164,57 @@ def test_agent99_mutating_inbox_requires_complete_typed_dispatch_envelope() -> N assert 'reason = "controlled_dispatch_identity_projection_mismatch"' in guard +def test_agent99_control_plane_health_is_exact_selfcheck_no_write() -> None: + source = SRE_INBOX.read_text(encoding="utf-8") + guard = source[ + source.index("function Test-Agent99ControlPlaneHealthEnvelope") : source.index( + "function Test-AgentTypedDispatchEnvelope" + ) + ] + processing = source[source.index("foreach ($file in $files)") :] + + assert '"agent99_control_plane_health" = "SelfCheck"' in source + assert '$ModeName -ne "SelfCheck"' in guard + assert 'reason = "agent99_control_plane_selfcheck_only"' in guard + assert 'reason = "agent99_control_plane_write_forbidden"' in guard + assert 'reason = "agent99_control_plane_force_forbidden"' in guard + assert '"agent99_control_plane_action_policy_v1"' in guard + for policy_field in ( + "controlledApplyAllowed", + "rebootAllowed", + "vmPowerChangeAllowed", + "arbitraryShellAllowed", + "crossHostFallbackAllowed", + "crossDomainFallbackAllowed", + ): + assert f'"{policy_field}"' in guard + assert '$canonicalAssetId -ne "host_99"' in guard + assert '$targetKind -ne "windows_vmware"' in guard + assert '$executor -ne "Agent99"' in guard + assert '"agent99_control_plane_selfcheck_independent_verifier"' in guard + assert '"agent99:agent99_control_plane_health:SelfCheck"' in guard + assert 'reason = "agent99_control_plane_cross_host_forbidden"' in guard + assert 'reason = "agent99_control_plane_cross_domain_forbidden"' in guard + assert 'reason = "agent99_control_plane_identity_projection_mismatch"' in guard + assert ( + '$kind -eq "agent99_control_plane_health"' + in source[ + source.index("function Test-AgentTypedDispatchEnvelope") : source.index( + "function Test-AgentAlertSuppressTelegram" + ) + ] + ) + assert "dispatchScope = $dispatchScope" in processing + for scope_field in ( + "canonicalAssetId", + "typedDomain", + "executor", + "verifier", + "routeId", + ): + assert scope_field in processing + + def test_agent99_typed_guard_rejects_before_dedupe_or_queue_write() -> None: source = SRE_INBOX.read_text(encoding="utf-8") processing = source[source.index("foreach ($file in $files)") :] diff --git a/apps/api/tests/test_agent99_control_plane_health_route.py b/apps/api/tests/test_agent99_control_plane_health_route.py new file mode 100644 index 000000000..42cc7590b --- /dev/null +++ b/apps/api/tests/test_agent99_control_plane_health_route.py @@ -0,0 +1,309 @@ +from __future__ import annotations + +import pytest + +from src.services.agent99_controlled_dispatch_ledger import ( + attach_agent99_dispatch_identity, + build_agent99_dispatch_identity, + build_agent99_dispatch_receipt_envelope, +) +from src.services.agent99_sre_bridge import ( + AGENT99_CONTROL_PLANE_HEALTH_KIND, + AGENT99_CONTROL_PLANE_HEALTH_MODE, + AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID, + bridge_alertmanager_to_agent99, + build_agent99_sre_alert, + dispatch_agent99_sre_alert_with_receipt, + resolve_agent99_alert_kind, + resolve_agent99_durable_route, +) +from src.services.controlled_alert_target_router import resolve_typed_alert_target + + +def _agent99_health_payload(*, host: str = "192.168.0.99") -> dict[str, object]: + return build_agent99_sre_alert( + alert_id="agent99-health-1", + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", + labels={"host": host, "service": "Agent99"}, + fingerprint="fp-agent99-control-plane-health", + ) + + +def test_agent99_service_unhealthy_has_exact_durable_selfcheck_route() -> None: + kind = resolve_agent99_alert_kind( + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="control plane health check failed", + ) + durable_route = resolve_agent99_durable_route( + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="control plane health check failed", + ) + + assert kind == AGENT99_CONTROL_PLANE_HEALTH_KIND + assert durable_route == { + "kind": AGENT99_CONTROL_PLANE_HEALTH_KIND, + "suggested_mode": AGENT99_CONTROL_PLANE_HEALTH_MODE, + "route_id": AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID, + } + assert ( + resolve_agent99_alert_kind( + alertname="Agent99ServiceDegraded", + severity="warning", + namespace="windows-control-plane", + target_resource="Agent99", + message="observation only", + ) + == "monitoring_alert" + ) + + +def test_agent99_control_plane_payload_is_host99_selfcheck_only() -> None: + payload = _agent99_health_payload() + typed_route = payload["routing"]["typedTargetRoute"] # type: ignore[index] + action_policy = payload["routing"]["actionPolicy"] # type: ignore[index] + + assert payload["kind"] == AGENT99_CONTROL_PLANE_HEALTH_KIND + assert payload["suggestedMode"] == AGENT99_CONTROL_PLANE_HEALTH_MODE + assert payload["controlledApply"] is False + assert payload["force"] is False + assert typed_route["resolution_status"] == "resolved" + assert typed_route["canonical_asset_id"] == "host_99" + assert typed_route["target_kind"] == "windows_vmware" + assert typed_route["executor"] == "Agent99" + assert typed_route["verifier"] == ( + "agent99_control_plane_selfcheck_independent_verifier" + ) + assert typed_route["allowed_inventory_hosts"] == ["host_99"] + assert typed_route["controlled_apply_allowed"] is False + assert typed_route["cross_domain_fallback_allowed"] is False + assert action_policy == { + "schemaVersion": "agent99_control_plane_action_policy_v1", + "allowedModes": ["SelfCheck"], + "controlledApplyAllowed": False, + "rebootAllowed": False, + "vmPowerChangeAllowed": False, + "arbitraryShellAllowed": False, + "crossHostFallbackAllowed": False, + "crossDomainFallbackAllowed": False, + } + assert "do not reboot" in payload["instruction"] + assert "change VM power" in payload["instruction"] + assert "arbitrary shell" in payload["instruction"] + + +def test_agent99_control_plane_rejects_cross_host_identity() -> None: + route = resolve_typed_alert_target( + alertname="Agent99ServiceUnhealthy", + namespace="windows-control-plane", + target_resource="Agent99 on host 110", + labels={"host": "192.168.0.110"}, + ) + + assert route["resolution_status"] == "asset_identity_unresolved" + assert route["target_kind"] == "unknown" + assert route["canonical_asset_id"] is None + assert route["agent99_bridge"]["dispatch_allowed"] is False + assert route["cross_domain_fallback_allowed"] is False + + +def test_agent99_dispatch_scope_is_bound_to_same_run(monkeypatch) -> None: + payload = _agent99_health_payload() + identity = build_agent99_dispatch_identity( + project_id="awoooi", + incident_id="INC-AGENT99-HEALTH-1", + source_fingerprint="fp-agent99-control-plane-health", + route_id=AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID, + work_item_id="AIA-SRE-013", + ) + payload = attach_agent99_dispatch_identity(payload, identity) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_RELAY_URL", + "", + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.settings.AGENT99_SRE_ALERT_INBOX_PATH", + "", + ) + + transport_receipt = dispatch_agent99_sre_alert_with_receipt(payload) + envelope = build_agent99_dispatch_receipt_envelope( + identity=identity, + dispatch_receipt=transport_receipt, + ) + + assert envelope["dispatch_scope"] == { + "schema_version": "agent99_dispatch_scope_v1", + "kind": AGENT99_CONTROL_PLANE_HEALTH_KIND, + "suggested_mode": "SelfCheck", + "target_resource": "Agent99", + "controlled_apply_requested": False, + "canonical_asset_id": "host_99", + "typed_domain": "windows_vmware", + "executor": "Agent99", + "verifier": "agent99_control_plane_selfcheck_independent_verifier", + "route_id": AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID, + } + assert envelope["identity"]["run_id"] == str(identity.run_id) + assert envelope["controlled_apply_authorized"] is False + assert envelope["runtime_execution_authorized"] is False + + +@pytest.mark.asyncio +async def test_agent99_control_plane_dispatch_stays_selfcheck_no_write( + monkeypatch, +) -> None: + captured: list[dict[str, object]] = [] + + class Ledger: + async def reserve(self, *, identity, payload): # type: ignore[no-untyped-def] + captured.append(payload) + return { + "status": "reserved", + "claimed": True, + "identity": identity.public_dict(), + "receipt": None, + "claim_token": "agent99-control-plane-claim", + } + + async def mark_delivery_attempt_started( # type: ignore[no-untyped-def] + self, *, identity, claim_token + ) -> dict[str, object]: + return { + "status": "delivery_attempt_fenced", + "identity": identity.public_dict(), + "claim_token": claim_token, + "receipt_persisted": True, + } + + async def complete( # type: ignore[no-untyped-def] + self, *, identity, dispatch_receipt, **_kwargs + ) -> dict[str, object]: + return { + **build_agent99_dispatch_receipt_envelope( + identity=identity, + dispatch_receipt=dispatch_receipt, + ), + "receipt_persisted": True, + } + + async def acquire(*_args, **_kwargs) -> dict[str, object]: + return {"acquired": True, "reason": "acquired"} + + def dispatch(payload: dict[str, object]) -> dict[str, object]: + routing = payload["routing"] + awoooi = payload["awoooi"] + typed = routing["typedTargetRoute"] # type: ignore[index] + identity = awoooi["agent99DispatchIdentity"] # type: ignore[index] + return { + "schema_version": "agent99_sre_dispatch_receipt_v2", + "status": "accepted_queue_persisted", + "transport": "relay", + "alert_id": payload["id"], + "kind": payload["kind"], + "target_resource": awoooi["targetResource"], # type: ignore[index] + "suggested_mode": payload["suggestedMode"], + "controlled_apply_requested": payload["controlledApply"], + "dispatch_scope": { + "schema_version": "agent99_dispatch_scope_v1", + "canonical_asset_id": typed["canonical_asset_id"], + "typed_domain": typed["target_kind"], + "executor": typed["executor"], + "verifier": typed["verifier"], + "route_id": identity["route_id"], + }, + "accepted": True, + "inbox_triggered": True, + "queue_accepted": True, + "dispatch_identity_matched": True, + "delivery_certainty": "delivered", + } + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.get_agent99_dispatch_ledger", + Ledger, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.acquire_agent99_sre_single_flight", + acquire, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", + dispatch, + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="agent99-health-valid", + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", + labels={"host": "192.168.0.99", "service": "Agent99"}, + fingerprint="fp-agent99-control-plane-health-valid", + incident_id="INC-AGENT99-HEALTH-VALID", + route_id=AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID, + work_item_id="AIA-SRE-013", + ) + + assert result["status"] == "dispatched" + assert result["kind"] == AGENT99_CONTROL_PLANE_HEALTH_KIND + assert result["suggestedMode"] == "SelfCheck" + assert len(captured) == 1 + assert captured[0]["controlledApply"] is False + assert captured[0]["force"] is False + assert ( + result["correlatedReceipt"]["dispatch_scope"]["route_id"] # type: ignore[index] + == AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID + ) + assert result["correlatedReceipt"]["controlled_apply_authorized"] is False # type: ignore[index] + assert result["correlatedReceipt"]["runtime_execution_authorized"] is False # type: ignore[index] + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("host", "route_id"), + [ + ("192.168.0.110", AGENT99_CONTROL_PLANE_HEALTH_ROUTE_ID), + ("192.168.0.99", "agent99:host_recovery:Recover"), + ], +) +async def test_agent99_control_plane_scope_mismatch_performs_zero_dispatch( + monkeypatch, + host: str, + route_id: str, +) -> None: + dispatched: list[dict[str, object]] = [] + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert_with_receipt", + lambda payload: dispatched.append(payload), + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="agent99-health-mismatch", + alertname="Agent99ServiceUnhealthy", + severity="critical", + namespace="windows-control-plane", + target_resource="Agent99", + message="Agent99 control plane health check failed", + labels={"host": host, "service": "Agent99"}, + fingerprint="fp-agent99-control-plane-health-mismatch", + incident_id="INC-AGENT99-HEALTH-MISMATCH", + route_id=route_id, + work_item_id="AIA-SRE-013", + ) + + assert result["status"] == "failed" + assert result["reason"] == "agent99_control_plane_exact_scope_required" + assert result["dispatchPerformed"] is False + assert result["runtimeClosureVerified"] is False + assert dispatched == [] diff --git a/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py index e925d0e55..7ffadb731 100644 --- a/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py +++ b/apps/api/tests/test_agent99_controlled_dispatch_ledger_contract.py @@ -58,6 +58,11 @@ def test_accepted_without_inbox_trigger_is_delivery_unknown_not_authorized() -> "suggested_mode": "Recover", "target_resource": "cold-start-gate", "controlled_apply_requested": True, + "canonical_asset_id": "", + "typed_domain": "", + "executor": "", + "verifier": "", + "route_id": "agent99:host_recovery:Recover", }