diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 60205aeba..fe2163d56 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2497,6 +2497,7 @@ function Invoke-SshText { $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + $null = $process.Handle if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { $stopwatch.Stop() @@ -6406,9 +6407,9 @@ function Invoke-AgentHost112GuestRecovery { # The guest executor uses exit 1 for a completed-but-degraded readback. Keep # that semantic result in executor_exit_code while reserving transport.ok for # SSH/sudo/executor delivery failures. - $checkCommand = ('/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) - $dryRunCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) - $applyCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf "executor_exit_code=%s\n" "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + $checkCommand = ('/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf ''executor_exit_code=%s\n'' "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + $dryRunCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf ''executor_exit_code=%s\n'' "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) + $applyCommand = ('sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}; executor_exit=$?; printf ''executor_exit_code=%s\n'' "$executor_exit"; case "$executor_exit" in 0|1) exit 0 ;; *) exit "$executor_exit" ;; esac' -f $effectiveTraceId, $effectiveRunId, $effectiveWorkItemId) $beforeTransport = Invoke-AgentHost112SshText $checkCommand $script:Agent99Host112CheckTimeoutSeconds 1 $before = Convert-AgentHost112GuestReadback $beforeTransport $effectiveTraceId $effectiveRunId $effectiveWorkItemId @@ -7046,22 +7047,78 @@ function Get-HarborState { $state } +function Convert-AgentAwoooDeploymentReadback { + param( + [string]$DeploymentText, + [string]$PodText, + [int]$DeploymentExitCode, + [int]$PodExitCode + ) + + $requiredNames = @("awoooi-api", "awoooi-web", "awoooi-worker") + $deployments = @() + foreach ($line in @($DeploymentText -split "`r?`n")) { + $match = [regex]::Match($line.Trim(), '^(awoooi-(?:api|web|worker))\s+([0-9]+|)\s+([0-9]+|)\s+([0-9]+|)\s+([0-9]+|)$') + if (-not $match.Success) { continue } + $numbers = @(1..4 | ForEach-Object { + $value = $match.Groups[$_ + 1].Value + if ($value -match '^[0-9]+$') { [int]$value } else { 0 } + }) + $deployments += [pscustomobject]@{ + name = $match.Groups[1].Value + desired = $numbers[0] + updated = $numbers[1] + ready = $numbers[2] + available = $numbers[3] + ok = [bool]($numbers[0] -gt 0 -and $numbers[1] -eq $numbers[0] -and $numbers[2] -eq $numbers[0] -and $numbers[3] -eq $numbers[0]) + } + } + + $pods = @() + foreach ($line in @($PodText -split "`r?`n")) { + $match = [regex]::Match($line.Trim(), '^(awoooi-(?:api|web|worker)-[^\s]+)\s+([^\s]+)\s+([^\s]+)$') + if (-not $match.Success) { continue } + $pods += [pscustomobject]@{ + name = $match.Groups[1].Value + phase = $match.Groups[2].Value + waitingReason = if ($match.Groups[3].Value -eq "") { "none" } else { $match.Groups[3].Value } + } + } + + $missing = @($requiredNames | Where-Object { $_ -notin @($deployments.name) }) + $notReady = @($deployments | Where-Object { -not $_.ok }) + $imagePullFailure = [bool](@($pods | Where-Object { $_.waitingReason -in @("ImagePullBackOff", "ErrImagePull") }).Count -gt 0) + $crashFailure = [bool](@($pods | Where-Object { $_.phase -eq "Pending" -or $_.waitingReason -eq "CrashLoopBackOff" }).Count -gt 0) + $deploymentSummary = @($deployments | ForEach-Object { "$($_.name):$($_.ready)/$($_.desired):updated=$($_.updated):available=$($_.available)" }) -join "," + $podSummary = @($pods | ForEach-Object { "$($_.name):$($_.phase):$($_.waitingReason)" }) -join "," + + [pscustomobject]@{ + deployReady = [bool]($DeploymentExitCode -eq 0 -and $PodExitCode -eq 0 -and $missing.Count -eq 0 -and $notReady.Count -eq 0) + imagePullFailure = $imagePullFailure + crashFailure = $crashFailure + deployments = $deployments + pods = $pods + missingDeployments = $missing + notReadyDeployments = @($notReady.name) + deployOutput = $deploymentSummary + podOutput = $podSummary + readbackExitCode = if ($DeploymentExitCode -ne 0) { $DeploymentExitCode } else { $PodExitCode } + rawDeploymentSpecRead = $false + secretValueRead = $false + } +} + function Get-AwoooState { $nestedSsh = "ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o NumberOfPasswordPrompts=0 -o ConnectTimeout=8 -l $SshUser 192.168.0.120" - $readback = Invoke-SshText "192.168.0.110" "$nestedSsh sudo -n kubectl get deploy,pods -n awoooi-prod" 60 1 - - $state = [pscustomobject]@{ - deployReady = ( - $readback.output -match "deployment.apps/awoooi-api\s+1/1" -and - $readback.output -match "deployment.apps/awoooi-web\s+2/2" -and - $readback.output -match "deployment.apps/awoooi-worker\s+1/1" - ) - imagePullFailure = ($readback.output -match "ImagePullBackOff|ErrImagePull") - crashFailure = ($readback.output -match "CrashLoopBackOff|Pending") - deployOutput = $readback.output - podOutput = $readback.output - readbackExitCode = $readback.exitCode - } + $deploymentCommand = "$nestedSsh sudo -n kubectl get deployment awoooi-api awoooi-web awoooi-worker -n awoooi-prod -o custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas,UPDATED:.status.updatedReplicas,READY:.status.readyReplicas,AVAILABLE:.status.availableReplicas --no-headers" + $podCommand = "$nestedSsh sudo -n kubectl get pods -n awoooi-prod -o custom-columns=NAME:.metadata.name,PHASE:.status.phase,WAITING:.status.containerStatuses[0].state.waiting.reason --no-headers" + $deploymentReadback = Invoke-SshText "192.168.0.110" $deploymentCommand 60 1 + $podReadback = Invoke-SshText "192.168.0.110" $podCommand 60 1 + $state = Convert-AgentAwoooDeploymentReadback ` + -DeploymentText ([string]$deploymentReadback.output) ` + -PodText ([string]$podReadback.output) ` + -DeploymentExitCode ([int]$deploymentReadback.exitCode) ` + -PodExitCode ([int]$podReadback.exitCode) Write-AgentLog "awoooi deployReady=$($state.deployReady) imagePullFailure=$($state.imagePullFailure) crashFailure=$($state.crashFailure)" $state } diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index 910a747a8..a9c243a21 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -48,7 +48,8 @@ foreach ($functionName in @( "Get-AgentHost112TransportContract", "Get-AgentHost112TimeoutContract", "Convert-AgentHost112GuestReadback", - "Invoke-AgentHost112GuestRecovery" + "Invoke-AgentHost112GuestRecovery", + "Convert-AgentAwoooDeploymentReadback" )) { $definition = $ast.Find({ param($node) @@ -145,6 +146,35 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" } +if ($formatFunctions.ContainsKey("Convert-AgentAwoooDeploymentReadback")) { + $greenDeployments = @( + "awoooi-api 2 2 2 2", + "awoooi-web 2 2 2 2", + "awoooi-worker 1 1 1 1" + ) -join "`n" + $greenPods = @( + "awoooi-api-a Running ", + "awoooi-api-b Running ", + "awoooi-web-a Running ", + "awoooi-worker-a Running " + ) -join "`n" + $greenWorkloads = Convert-AgentAwoooDeploymentReadback $greenDeployments $greenPods 0 0 + Add-SyntheticCheck "recovery:awoooi_dynamic_replica_readback" ( + $greenWorkloads.deployReady -and + @($greenWorkloads.deployments | Where-Object { $_.name -eq "awoooi-api" -and $_.desired -eq 2 -and $_.ready -eq 2 }).Count -eq 1 -and + -not $greenWorkloads.rawDeploymentSpecRead -and + -not $greenWorkloads.secretValueRead + ) "dynamic desired/updated/ready/available counts close without raw deployment specs" + + $partialWorkloads = Convert-AgentAwoooDeploymentReadback ($greenDeployments.Replace("awoooi-api 2 2 2 2", "awoooi-api 2 2 1 1")) $greenPods 0 0 + Add-SyntheticCheck "recovery:awoooi_partial_rollout_fails_closed" ( + -not $partialWorkloads.deployReady -and + @($partialWorkloads.notReadyDeployments) -contains "awoooi-api" + ) "one unavailable replica remains a named blocker" +} else { + Add-SyntheticCheck "recovery:awoooi_deployment_parser" $false "Convert-AgentAwoooDeploymentReadback not found" +} + if ( $formatFunctions.ContainsKey("Get-HostSshUser") -and $formatFunctions.ContainsKey("Get-AgentSshTransportConfig") -and diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index d2c99cc12..9ec00e760 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -132,6 +132,7 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> assert "/usr/local/sbin/awoooi-host112-guest-readiness --check --trace-id {0} --run-id {1} --work-item-id {2}" in function assert "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --dry-run --trace-id {0} --run-id {1} --work-item-id {2}" in function assert "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply --trace-id {0} --run-id {1} --work-item-id {2}" in function + assert function.count("printf ''executor_exit_code=%s\\n''") == 3 assert function.count("Invoke-AgentHost112SshText $checkCommand") == 2 assert function.count("Invoke-AgentHost112SshText $dryRunCommand") == 2 assert 'schemaVersion = "agent99_host112_guest_recovery_v3"' in function @@ -162,6 +163,20 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> assert f'"{forbidden}"' in function +def test_agent99_awoooi_readback_uses_dynamic_replica_counts_without_raw_specs() -> None: + source = CONTROL.read_text(encoding="utf-8") + parser = source[source.index("function Convert-AgentAwoooDeploymentReadback") :] + parser = parser[: parser.index("function Find-Vmrun")] + + assert "custom-columns=NAME:.metadata.name,DESIRED:.spec.replicas" in parser + assert 'requiredNames = @("awoooi-api", "awoooi-web", "awoooi-worker")' in parser + assert "$numbers[2] -eq $numbers[0]" in parser + assert "$numbers[3] -eq $numbers[0]" in parser + assert "rawDeploymentSpecRead = $false" in parser + assert "secretValueRead = $false" in parser + assert "deployment.apps/awoooi-api\\s+1/1" not in parser + + def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None: config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index 97b8f61ed..d15f0bb86 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -590,6 +590,8 @@ def test_unhealthy_executor_exit_one_is_delivered_readback_not_transport_failure synthetic = read(ROOT / "agent99-synthetic-tests.ps1") assert function.count('case "$executor_exit" in 0|1) exit 0') == 3 + assert function.count("printf ''executor_exit_code=%s\\n''") == 3 + assert 'printf "executor_exit_code=%s\\n"' not in function assert "$before.executorExitCode -in @(\"0\", \"1\")" in function assert "$before.readbackPresent -and" in function assert "$before.identityMatches -and" in function @@ -598,6 +600,17 @@ def test_unhealthy_executor_exit_one_is_delivered_readback_not_transport_failure assert "host112_general_candidate_not_blocked_by_healthy_manager" in synthetic +def test_ssh_transport_materializes_process_handle_before_waiting() -> None: + control = read(CONTROL) + function = control[control.index("function Invoke-SshText") :] + function = function[: function.index("function Quote-ShSingle")] + + assert "$null = $process.Handle" in function + assert function.index("$null = $process.Handle") < function.index( + "$process.WaitForExit($TimeoutSeconds * 1000)" + ) + + def test_general_guest_candidate_is_not_blocked_by_manager_candidate() -> None: control = read(CONTROL) function = control[control.index("function Invoke-AgentHost112GuestRecovery") :]