From 1de58df5099960639537234b2799be507b86e18b Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 22:38:43 +0800 Subject: [PATCH] fix(agent99): reconcile accepted ignored dispatch safely --- agent99-sre-alert-relay.ps1 | 31 +++++++++++++++++-- .../tests/test_agent99_same_run_reconcile.py | 11 +++++++ .../deploy-agent99-via-windows99-ssh.sh | 6 +++- ...remote_atomic_deploy_transport_contract.py | 3 ++ 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/agent99-sre-alert-relay.ps1 b/agent99-sre-alert-relay.ps1 index 283860c36..f158ea7f2 100644 --- a/agent99-sre-alert-relay.ps1 +++ b/agent99-sre-alert-relay.ps1 @@ -17,6 +17,7 @@ $BinDir = Join-Path $AgentRoot "bin" $QueueDir = Join-Path $AgentRoot "queue" $ProcessedDir = Join-Path $QueueDir "processed" $SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile" +$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored" $SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1" New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir | Out-Null @@ -381,6 +382,9 @@ function Convert-AgentStoredDispatchIdentity { $identity = Get-AgentField $Stored "identity" $null if ($identity) { return $identity } + $awoooi = Get-AgentField $Stored "awoooi" $null + $acceptedAlertIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null + if ($acceptedAlertIdentity) { return $acceptedAlertIdentity } return [pscustomobject]@{ schema_version = "agent99_controlled_dispatch_identity_v1" project_id = [string](Get-AgentField $Stored "projectId" "") @@ -407,13 +411,32 @@ function Get-AgentExistingRecoverIdentity { $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "running-*.json" -File -ErrorAction SilentlyContinue) $files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Where-Object { $_.Name -notmatch "^(processed|failed|running|same-run-status)-" }) - foreach ($file in @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) { + # The API's durable receipt predates the inbox's final classification. The + # exact legacy Recover envelope was accepted by the relay and then moved to + # ignored by an older inbox build. Admit only that immutable source envelope + # as identity evidence so the externally-resolved incident can run Status; + # never replay Recover or accept a generic ignored/reconcile payload. + $acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json" + $files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256) + $files += @(Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern -File -ErrorAction SilentlyContinue) + foreach ($file in $files) { try { $stored = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json if ([bool](Get-AgentField $stored "reconcileOnly" $false)) { continue } - if ([string](Get-AgentField $stored "mode" "") -ne "Recover") { continue } - if ([string](Get-AgentField $stored "automationRunId" "") -ne $RunId) { continue } + $storedMode = [string](Get-AgentField $stored "mode" (Get-AgentField $stored "suggestedMode" "")) + if ($storedMode -ne "Recover") { continue } $identity = Convert-AgentStoredDispatchIdentity $stored + if ([string](Get-AgentField $identity "run_id" "") -ne $RunId) { continue } + if ($file.DirectoryName -eq $AcceptedIgnoredDir) { + if ( + [string](Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId" -or + [string](Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager" -or + [string](Get-AgentField $stored "kind" "") -ne "host_recovery" -or + [string](Get-AgentField $stored "service" "") -ne "cold-start-gate" -or + (Get-AgentField $stored "controlledApply" $null) -isnot [bool] -or + -not [bool](Get-AgentField $stored "controlledApply" $false) + ) { continue } + } $check = Test-AgentSameRunDispatchIdentity $identity if ($check.ok) { return $identity } } catch { @@ -782,6 +805,8 @@ try { event = "same_run_status_reconcile" ok = [bool]$reconcile.ok status = [string]$reconcile.status + reason = [string](Get-AgentField $reconcile "reason" "") + httpStatus = [int]$reconcile.httpStatus remote = $remote runId = if ($reconcile.ok) { [string]$reconcile.automationRunId } else { "rejected" } reconcileOnly = $true diff --git a/apps/api/tests/test_agent99_same_run_reconcile.py b/apps/api/tests/test_agent99_same_run_reconcile.py index da80a582f..05063f0d3 100644 --- a/apps/api/tests/test_agent99_same_run_reconcile.py +++ b/apps/api/tests/test_agent99_same_run_reconcile.py @@ -734,6 +734,15 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert 'work_item_id" "") -ne "agent99-dispatch:awoooi:' in relay assert 'trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$"' in relay assert "function Get-AgentExistingRecoverIdentity" in relay + assert '$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"' in relay + assert '$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"' in relay + assert '$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)' in relay + assert 'Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern' in relay + assert 'Get-AgentField $awoooi "agent99DispatchIdentity"' in relay + assert 'Get-AgentField $stored "id" "") -ne "awoooi-agent99-$RunId"' in relay + assert 'Get-AgentField $stored "source" "") -ne "awoooi-api-alertmanager"' in relay + assert 'Get-AgentField $stored "kind" "") -ne "host_recovery"' in relay + assert 'Get-AgentField $stored "service" "") -ne "cold-start-gate"' in relay assert "existing_dispatch_identity_not_found" in relay assert "existing_dispatch_identity_mismatch" in relay assert "function Test-AgentSameRunOutcomeEligible" in relay @@ -748,6 +757,8 @@ def test_windows_relay_and_control_plane_enforce_same_run_no_write_contract() -> assert 'reconcileOnly = $true' in relay assert 'suppressTelegram = $true' in relay assert 'sourceEventResolutionPolicy = "external_source_receipt_required"' in relay + assert 'reason = [string](Get-AgentField $reconcile "reason" "")' in relay + assert 'httpStatus = [int]$reconcile.httpStatus' in relay assert '"-ReconcileQueueOnly", "-ReconcileQueueId", $queueId' in relay assert '"-SuppressAlerts", "-SuppressReason"' in relay diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index 76b773ff0..12e557f03 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -426,7 +426,11 @@ if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p $c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''} try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name} $z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false} -[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status='check_ready';mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction='rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}|ConvertTo-Json -Compress -Depth 8''' +$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0) +$status=if($ready){'check_ready'}else{'check_failed_no_apply'} +$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'} +[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=14;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8 +if(-not$ready){exit 65}''' script = ( script.replace("__SOURCE_REVISION__", source_revision) .replace("__MANIFEST_SHA256__", manifest_sha256) diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 522ec2a0b..15aa9709d 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -671,6 +671,9 @@ def test_check_mode_uses_bounded_control_command_without_ssh_stdin( assert "agent99-contract-check.ps1" in check_script assert "Get-FileHash" in check_script assert "runtimeContract" in check_script + assert "$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)" in check_script + assert "'check_failed_no_apply'" in check_script + assert "if(-not$ready){exit 65}" in check_script assert check_script.count("ProgressPreference") == 1 assert "remoteWritePerformed" in check_script assert "$z[$n]=$false" in check_script