diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 78fc7cf0e..a913ab191 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -448,6 +448,146 @@ function Get-AgentControlLoopMaintenanceReadback { } } +function Get-AgentPreflightSha256Hex { + param([string]$Text) + + $sha = [Security.Cryptography.SHA256]::Create() + try { + $bytes = $sha.ComputeHash([Text.Encoding]::UTF8.GetBytes([string]$Text)) + return [BitConverter]::ToString($bytes).Replace("-", "").ToLowerInvariant() + } finally { + $sha.Dispose() + } +} + +function Convert-AgentPreflightGuidToNetworkBytes { + param([guid]$Guid) + + [byte[]]$bytes = $Guid.ToByteArray() + return [byte[]]@( + $bytes[3], $bytes[2], $bytes[1], $bytes[0], + $bytes[5], $bytes[4], $bytes[7], $bytes[6], + $bytes[8], $bytes[9], $bytes[10], $bytes[11], + $bytes[12], $bytes[13], $bytes[14], $bytes[15] + ) +} + +function New-AgentPreflightUuidV5 { + param( + [guid]$Namespace, + [string]$Name + ) + + [byte[]]$namespaceBytes = Convert-AgentPreflightGuidToNetworkBytes $Namespace + [byte[]]$nameBytes = [Text.Encoding]::UTF8.GetBytes($Name) + $sha1 = [Security.Cryptography.SHA1]::Create() + try { + [byte[]]$hash = $sha1.ComputeHash($namespaceBytes + $nameBytes) + } finally { + $sha1.Dispose() + } + $hash[6] = [byte](($hash[6] -band 0x0f) -bor 0x50) + $hash[8] = [byte](($hash[8] -band 0x3f) -bor 0x80) + $hex = -join @($hash[0..15] | ForEach-Object { $_.ToString("x2") }) + return [guid]::Parse(( + "{0}-{1}-{2}-{3}-{4}" -f + $hex.Substring(0, 8), + $hex.Substring(8, 4), + $hex.Substring(12, 4), + $hex.Substring(16, 4), + $hex.Substring(20, 12) + )) +} + +function Test-AgentCanonicalPendingCommandIdentity { + param([object]$Payload) + + if (-not $Payload) { return $false } + try { + $projectId = ([string]$Payload.projectId).Trim() + $incidentId = ([string]$Payload.incidentId).Trim() + $sourceFingerprint = ([string]$Payload.sourceFingerprint).Trim() + $routeId = ([string]$Payload.dispatchRouteId).Trim() + $executionGeneration = ([string]$Payload.executionGeneration).Trim() + $approvalId = ([string]$Payload.approvalId).Trim() + $workItemId = ([string]$Payload.workItemId).Trim() + if ( + -not $projectId -or + -not $incidentId -or + -not $sourceFingerprint -or + -not $routeId -or + -not $executionGeneration -or + -not $workItemId + ) { return $false } + + $normalized = [ordered]@{ + approval_id = $approvalId + execution_generation = $executionGeneration + incident_id = $incidentId + project_id = $projectId + route_id = $routeId + source_fingerprint = $sourceFingerprint + work_item_id = $workItemId + } + $canonical = $normalized | ConvertTo-Json -Compress -Depth 4 + $digest = Get-AgentPreflightSha256Hex $canonical + $singleFlightCanonical = [ordered]@{ + incident_id = $incidentId + project_id = $projectId + route_id = $routeId + source_fingerprint = $sourceFingerprint + } | ConvertTo-Json -Compress -Depth 4 + $singleFlightDigest = Get-AgentPreflightSha256Hex $singleFlightCanonical + $runId = (New-AgentPreflightUuidV5 ` + -Namespace ([guid]::Parse("6ba7b811-9dad-11d1-80b4-00c04fd430c8")) ` + -Name "agent99-controlled-dispatch:$canonical").ToString() + $traceId = "00-$($digest.Substring(0, 32))-$($digest.Substring(32, 16))-01" + $idempotencyKey = "agent99-controlled:$digest" + $singleFlightKey = "agent99-controlled-flight:$singleFlightDigest" + $public = [ordered]@{ + approval_id = $approvalId + execution_generation = $executionGeneration + idempotency_key = $idempotencyKey + incident_id = $incidentId + lock_owner = $runId + project_id = $projectId + route_id = $routeId + run_id = $runId + schema_version = "agent99_controlled_dispatch_identity_v1" + single_flight_key = $singleFlightKey + source_fingerprint = $sourceFingerprint + trace_id = $traceId + work_item_id = $workItemId + } + $expected = [ordered]@{ + approvalId = $approvalId + automationRunId = $runId + canonicalDigest = Get-AgentPreflightSha256Hex ( + $public | ConvertTo-Json -Compress -Depth 4 + ) + dispatchIdentitySchemaVersion = "agent99_controlled_dispatch_identity_v1" + dispatchRouteId = $routeId + executionGeneration = $executionGeneration + idempotencyKey = $idempotencyKey + incidentId = $incidentId + lockOwner = $runId + projectId = $projectId + singleFlightKey = $singleFlightKey + sourceFingerprint = $sourceFingerprint + traceId = $traceId + workItemId = $workItemId + } + foreach ($field in $expected.Keys) { + if ([string]$Payload.$field -cne [string]$expected[$field]) { + return $false + } + } + return $true + } catch { + return $false + } +} + function Get-AgentPendingCommandPromotionReadback { param( [object]$MaintenanceReadback, @@ -493,7 +633,10 @@ function Get-AgentPendingCommandPromotionReadback { $result.reason = "maintenance_contract_invalid" return [pscustomobject]$result } - if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) { + if ( + $Scope -in @("PromotionReserve", "PostPromotionMaintenance") -and + -not $MaintenanceReadback.valid + ) { $result.reason = "maintenance_current_state_invalid" return [pscustomobject]$result } @@ -578,6 +721,16 @@ function Get-AgentPendingCommandPromotionReadback { $payload -and @($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0 ) + $allowedReason = [bool]( + $payload -and + [string]$payload.reason -in @( + "host_reboot_detected", + "host_reboot_recovery_pending", + "host_unreachable_detected", + "host112_guest_not_ready", + "host188_edge_services_not_ready" + ) + ) $identityValid = [bool]( -not $parseError -and $propertySetMatches -and @@ -592,8 +745,8 @@ function Get-AgentPendingCommandPromotionReadback { [string]$payload.dispatchRouteId -eq "agent99_local_observer" -and [string]$payload.projectId -eq "awoooi" -and [string]$payload.requestedBy -eq "Agent99" -and - [string]$payload.reason -eq "host_unreachable_detected" -and - [string]$payload.externalId -eq "host_unreachable_detected" -and + $allowedReason -and + [string]$payload.externalId -eq [string]$payload.reason -and [string]$payload.approvalId -eq "" -and [string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and [string]$payload.lockOwner -eq [string]$payload.automationRunId -and @@ -604,8 +757,9 @@ function Get-AgentPendingCommandPromotionReadback { [string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and [string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and [string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and - [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and + [string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):$($payload.reason)" -and [string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and + (Test-AgentCanonicalPendingCommandIdentity $payload) -and -not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and $createdAtValid -and $createdAt -ge $maintenanceTimestamp -and @@ -634,10 +788,14 @@ function Get-AgentPendingCommandPromotionReadback { if ($allValid) { $result.eligibleMaintenanceAutoRecoverCount = $observedCount $result.contractValid = $true - if ($Scope -eq "PromotionReserve") { + if ($Scope -in @("PromotionReserve", "PostPromotionMaintenance")) { $result.deferredCount = $observedCount $result.blockingCount = 0 - $result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion" + $result.reason = if ($Scope -eq "PromotionReserve") { + "bounded_maintenance_auto_recover_deferred_until_post_promotion" + } else { + "bounded_maintenance_auto_recover_deferred_until_control_loop_restore" + } } else { $result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain" } @@ -1013,16 +1171,34 @@ $queues = @( (Get-DirectoryReadback "queue\failed") ) $promotionEvidenceRequired = [bool]($ReadinessScope -ne "PromotionReserve") +$postPromotionControlMaintenanceAccepted = [bool]( + $ReadinessScope -eq "PostPromotionMaintenance" -and + @($tasks | Where-Object { + [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and + [bool]$_.maintenanceFreezeAccepted + }).Count -eq 1 +) +$controlTickEvidenceRequired = [bool]( + $promotionEvidenceRequired -and + -not $postPromotionControlMaintenanceAccepted +) $evidence = @( (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), - (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), + (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $controlTickEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes), (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0) ) +$controlTickEvidence = $evidence | Where-Object { $_.name -eq "control_tick" } | Select-Object -First 1 +if ($controlTickEvidence) { + $controlTickEvidence | Add-Member ` + -NotePropertyName maintenanceFreezeAccepted ` + -NotePropertyValue $postPromotionControlMaintenanceAccepted ` + -Force +} $operatingSystem = Get-CimInstance Win32_OperatingSystem $taskFailures = @($tasks | Where-Object { -not $_.healthy }) diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index 4f0303f18..ca42fbc5e 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -472,11 +472,24 @@ function Invoke-AgentLivePreflight { $ReadinessScope -eq "PromotionReserve" -or ($payload -and $payload.manifest -and [bool]$payload.manifest.runtimeMatched) ) + $controlLoopMaintenanceRestorePending = [bool]( + $ReadinessScope -eq "PostPromotionMaintenance" -and + $payload -and + @($payload.tasks | Where-Object { + [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and + [bool]$_.maintenanceFreezeAccepted + }).Count -eq 1 + ) $pendingCommandsReadyForScope = [bool]( $summary -and [int]$summary.pendingCommandBlockingCount -eq 0 -and ( $ReadinessScope -eq "PromotionReserve" -or + ( + $ReadinessScope -eq "PostPromotionMaintenance" -and + $controlLoopMaintenanceRestorePending -and + [int]$summary.pendingCommandCount -eq [int]$summary.promotionDeferredCommandCount + ) -or [int]$summary.pendingCommandCount -eq 0 ) ) @@ -500,14 +513,6 @@ function Invoke-AgentLivePreflight { $pendingCommandsReadyForScope -and $runtimePlaneBlockingReasons.Count -eq 0 ) - $controlLoopMaintenanceRestorePending = [bool]( - $ReadinessScope -eq "PostPromotionMaintenance" -and - $payload -and - @($payload.tasks | Where-Object { - [string]$_.name -eq "Wooo-Agent99-Control-Loop" -and - [bool]$_.maintenanceFreezeAccepted - }).Count -eq 1 - ) return [pscustomobject]@{ ok = $runtimePlaneReady readinessScope = $ReadinessScope diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py index 251597a32..05b9a0b81 100644 --- a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -1,9 +1,11 @@ from __future__ import annotations import json +import hashlib import os import shutil import subprocess +import uuid from copy import deepcopy from datetime import datetime, timedelta, timezone from pathlib import Path @@ -16,6 +18,72 @@ ROOT = Path(__file__).resolve().parents[3] PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1" +def _canonical_dispatch_identity( + *, incident_id: str, source_fingerprint: str, work_item_id: str +) -> dict[str, str]: + normalized = { + "approval_id": "", + "execution_generation": "1", + "incident_id": incident_id, + "project_id": "awoooi", + "route_id": "agent99_local_observer", + "source_fingerprint": source_fingerprint, + "work_item_id": work_item_id, + } + canonical = json.dumps(normalized, separators=(",", ":")) + digest = hashlib.sha256(canonical.encode()).hexdigest() + single_flight = json.dumps( + { + "incident_id": incident_id, + "project_id": "awoooi", + "route_id": "agent99_local_observer", + "source_fingerprint": source_fingerprint, + }, + separators=(",", ":"), + ) + single_flight_digest = hashlib.sha256(single_flight.encode()).hexdigest() + run_id = str( + uuid.uuid5(uuid.NAMESPACE_URL, f"agent99-controlled-dispatch:{canonical}") + ) + trace_id = f"00-{digest[:32]}-{digest[32:48]}-01" + idempotency_key = f"agent99-controlled:{digest}" + single_flight_key = f"agent99-controlled-flight:{single_flight_digest}" + public = { + "approval_id": "", + "execution_generation": "1", + "idempotency_key": idempotency_key, + "incident_id": incident_id, + "lock_owner": run_id, + "project_id": "awoooi", + "route_id": "agent99_local_observer", + "run_id": run_id, + "schema_version": "agent99_controlled_dispatch_identity_v1", + "single_flight_key": single_flight_key, + "source_fingerprint": source_fingerprint, + "trace_id": trace_id, + "work_item_id": work_item_id, + } + return { + "approvalId": "", + "automationRunId": run_id, + "canonicalDigest": hashlib.sha256( + json.dumps(public, separators=(",", ":")).encode() + ).hexdigest(), + "correlationKey": idempotency_key, + "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1", + "dispatchRouteId": "agent99_local_observer", + "executionGeneration": "1", + "idempotencyKey": idempotency_key, + "incidentId": incident_id, + "lockOwner": run_id, + "projectId": "awoooi", + "singleFlightKey": single_flight_key, + "sourceFingerprint": source_fingerprint, + "traceId": trace_id, + "workItemId": work_item_id, + } + + def _function_source() -> str: source = PREFLIGHT.read_text(encoding="utf-8") start = source.index("function Get-AgentLivePreflightDecision") @@ -195,9 +263,18 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No assert '$controlLoopMaintenance.valid' in source assert '$postPromotionReady = [bool]($runtimeHealthy -or $maintenanceFreezeAccepted)' in source assert '$task.healthy = $postPromotionReady' in source + assert "$controlTickEvidenceRequired" in source + assert "$postPromotionControlMaintenanceAccepted" in source + assert 'NotePropertyName maintenanceFreezeAccepted' in source assert "function Get-AgentPendingCommandPromotionReadback" in source assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source + assert '"bounded_maintenance_auto_recover_deferred_until_control_loop_restore"' in source assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source + assert '"host_reboot_recovery_pending"' in source + assert '[string]$payload.externalId -eq [string]$payload.reason' in source + assert '"agent99-auto:$($payload.incidentId):$($payload.reason)"' in source + assert "function Test-AgentCanonicalPendingCommandIdentity" in source + assert "(Test-AgentCanonicalPendingCommandIdentity $payload) -and" in source assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source assert '$payload.observation -is [pscustomobject]' in source assert 'instructionContentReadback = $false' in source @@ -543,34 +620,27 @@ def test_promotion_reserve_defers_only_exact_maintenance_auto_recover( queue.mkdir() now = datetime.now(timezone.utc) incident_id = "agent99-auto-" + ("a" * 20) + source_fingerprint = "e" * 64 command_id = "auto-recover-20260719-152746-39ca4a72" command_path = queue / f"{command_id}.json" + work_item_id = f"agent99-auto:{incident_id}:host_unreachable_detected" + identity = _canonical_dispatch_identity( + incident_id=incident_id, + source_fingerprint=source_fingerprint, + work_item_id=work_item_id, + ) payload = { - "approvalId": "", - "automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f", - "canonicalDigest": "b" * 64, + **identity, "controlledApply": True, - "correlationKey": "agent99-controlled:" + ("c" * 64), "createdAt": now.isoformat(), - "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1", - "dispatchRouteId": "agent99_local_observer", - "executionGeneration": "1", "externalId": "host_unreachable_detected", "id": command_id, - "idempotencyKey": "agent99-controlled:" + ("c" * 64), - "incidentId": incident_id, "instruction": "recover exact observed unreachable host through controlled playbook", - "lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f", "mode": "Recover", "observation": {}, - "projectId": "awoooi", "reason": "host_unreachable_detected", "requestedBy": "Agent99", - "singleFlightKey": "agent99-controlled-flight:" + ("d" * 64), "source": "agent99-recovery-observer", - "sourceFingerprint": "e" * 64, - "traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01", - "workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected", } function_source = _evidence_function_source() @@ -622,6 +692,38 @@ Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scop "full_runtime_requires_maintenance_auto_recover_to_drain" ) + post_maintenance = run("PostPromotionMaintenance") + assert post_maintenance["deferredCount"] == 1 + assert post_maintenance["blockingCount"] == 0 + assert post_maintenance["contractValid"] is True + assert post_maintenance["reason"] == ( + "bounded_maintenance_auto_recover_deferred_until_control_loop_restore" + ) + + payload["externalId"] = "host_reboot_recovery_pending" + payload["reason"] = "host_reboot_recovery_pending" + payload["workItemId"] = ( + f"agent99-auto:{incident_id}:host_reboot_recovery_pending" + ) + command_path.write_text(json.dumps(payload), encoding="utf-8") + tampered_durable_boot_pending = run("PostPromotionMaintenance") + assert tampered_durable_boot_pending["deferredCount"] == 0 + assert tampered_durable_boot_pending["blockingCount"] == 1 + assert tampered_durable_boot_pending["contractValid"] is False + + payload.update( + _canonical_dispatch_identity( + incident_id=incident_id, + source_fingerprint=source_fingerprint, + work_item_id=payload["workItemId"], + ) + ) + command_path.write_text(json.dumps(payload), encoding="utf-8") + durable_boot_pending = run("PostPromotionMaintenance") + assert durable_boot_pending["deferredCount"] == 1 + assert durable_boot_pending["blockingCount"] == 0 + assert durable_boot_pending["contractValid"] is True + payload["mode"] = "Status" command_path.write_text(json.dumps(payload), encoding="utf-8") invalid = run("PromotionReserve") diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 0c4005e07..357bc871d 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -429,6 +429,7 @@ def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> assert "postVerifyAttempts = $postVerifyAttempts" in source assert "postVerifyRetryCount = $postVerifyRetryCount" in source assert "controlLoopMaintenanceRestorePending = $controlLoopMaintenanceRestorePending" in source + assert '[int]$summary.pendingCommandCount -eq [int]$summary.promotionDeferredCommandCount' in source assert '"deployed_verified_maintenance_restore_pending"' in source assert "fullRuntimeVerifierPending = $maintenanceRestorePending" in source assert "completionClaim = [bool](-not $maintenanceRestorePending)" in source