diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md index ac4b69da7..089e2f233 100644 --- a/AGENT99-AI-WORKSTATION-SOP.md +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -733,6 +733,8 @@ Agent99 now uses a source-trusted deployment path, a dedicated bounded SSH trans - `policyVersion` migration updates stale live policies and prevents an old command or marker from surviving a new source deployment. - Bounded SSH process lifecycle: - Every SSH call has a timeout; timeout cleanup terminates the SSH process tree and verifies that the client exited. + - Every Agent99 process acquires the shared `C:\Wooo\Agent99\state\ssh-transport.lock` with an exclusive file handle before starting SSH. This serializes scheduled tasks and operator runs across Windows sessions; the OS releases the handle if a process exits. + - Lock acquisition waits at most 90 seconds. Timeout evidence is `ssh_transport_lock_timeout`; it must not start another SSH client or be reported as a service outage. - Before controlled `Perf` or `LoadShed`, the stale-process guard only considers Agent99 allowlisted targets using `BatchMode=yes` and explicitly excludes `-N`, `-L`, `-R`, and `-D` tunnels. - A process is eligible after 15 minutes with a missing parent, or after the 60-minute hard limit. Evidence records selected process metadata but never the command line. - Live receipts: diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index f09f39759..fe7c4e922 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -67,6 +67,7 @@ $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced" Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present" Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded" +Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index ebdd72632..c14b7a7a0 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -66,6 +66,81 @@ function Get-AgentSshIdentityArguments { @("-i", $SshIdentityFile, "-o", "IdentitiesOnly=yes") } +function Get-AgentSshTransportConfig { + $configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null } + [pscustomobject]@{ + serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized)) + lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 } + lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 } + lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock" + } +} + +function Enter-AgentSshTransportLock { + $transport = Get-AgentSshTransportConfig + if (-not $transport.serialized) { + return [pscustomobject]@{ acquired = $true; disabled = $true; reason = "serialization_disabled"; waitMs = 0; stream = $null } + } + + $lockDir = Split-Path -Parent $transport.lockPath + New-Item -ItemType Directory -Force -Path $lockDir | Out-Null + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + while ($stopwatch.Elapsed.TotalSeconds -lt $transport.lockTimeoutSeconds) { + try { + $stream = [IO.File]::Open( + $transport.lockPath, + [IO.FileMode]::OpenOrCreate, + [IO.FileAccess]::ReadWrite, + [IO.FileShare]::None + ) + $stopwatch.Stop() + return [pscustomobject]@{ + acquired = $true + disabled = $false + reason = "acquired" + waitMs = $stopwatch.ElapsedMilliseconds + stream = $stream + } + } catch [System.IO.IOException] { + Start-Sleep -Milliseconds $transport.lockPollMilliseconds + } catch [System.UnauthorizedAccessException] { + $stopwatch.Stop() + return [pscustomobject]@{ + acquired = $false + disabled = $false + reason = "access_denied" + waitMs = $stopwatch.ElapsedMilliseconds + stream = $null + } + } + } + + $stopwatch.Stop() + [pscustomobject]@{ + acquired = $false + disabled = $false + reason = "timeout" + waitMs = $stopwatch.ElapsedMilliseconds + stream = $null + } +} + +function Exit-AgentSshTransportLock { + param([object]$TransportLock) + if ($TransportLock -and $TransportLock.stream) { + try { $TransportLock.stream.Dispose() } catch {} + } +} + +function Add-AgentSshTransportMetadata { + param([object]$Result, [object]$TransportLock) + if ($Result) { + $Result | Add-Member -NotePropertyName transportSerialized -NotePropertyValue ([bool](-not $TransportLock.disabled)) -Force + $Result | Add-Member -NotePropertyName transportLockWaitMs -NotePropertyValue ([long]$TransportLock.waitMs) -Force + } + $Result +} + function Get-AgentSshProcessGuardConfig { $configured = if ($Config.PSObject.Properties["sshProcessGuard"]) { $Config.sshProcessGuard } else { $null } [pscustomobject]@{ @@ -1275,86 +1350,104 @@ function Invoke-SshText { [string]$User = $null ) - $last = $null - $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost } - for ($attempt = 1; $attempt -le $Retries; $attempt++) { - $args = @(Get-AgentSshIdentityArguments) + @( - "-o", "BatchMode=yes", - "-o", "StrictHostKeyChecking=accept-new", - "-o", "NumberOfPasswordPrompts=0", - "-o", "ConnectTimeout=$TimeoutSeconds", - "-o", "ServerAliveInterval=20", - "-o", "ServerAliveCountMax=3", - "-l", $userToUse, - $TargetHost, - $Command - ) - - try { - $stopwatch = [Diagnostics.Stopwatch]::StartNew() - $id = [guid]::NewGuid().ToString("N") - $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" - $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" - $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru - - if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { - $stopwatch.Stop() - try { - & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null - } catch {} - if (-not $process.WaitForExit(2000) -and -not $process.HasExited) { - $process.Kill() - $process.WaitForExit(2000) | Out-Null - } - Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null - $last = [pscustomobject]@{ - ok = $false - exitCode = -2 - output = "timeout_after_${TimeoutSeconds}s" - elapsedMs = $stopwatch.ElapsedMilliseconds - } - } else { - $process.WaitForExit() | Out-Null - $stopwatch.Stop() - $process.Refresh() - $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } - $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } - Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null - $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" - $exitCode = $process.ExitCode - if ($null -eq $exitCode) { - $exitCode = if ($stderr) { -3 } else { 0 } - } - $last = [pscustomobject]@{ - ok = ($exitCode -eq 0) - exitCode = $exitCode - output = $text - elapsedMs = $stopwatch.ElapsedMilliseconds - } - } - } catch { - $last = [pscustomobject]@{ - ok = $false - exitCode = -1 - output = $_.Exception.Message - elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null } - } - } - - if ($last.ok) { - if ($attempt -gt 1) { - Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt" - } - return $last - } - - if ($attempt -lt $Retries) { - Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)" - Start-Sleep -Seconds 2 + $transportLock = Enter-AgentSshTransportLock + if (-not $transportLock.acquired) { + Write-AgentLog "ssh_transport_lock_failed host=$TargetHost reason=$($transportLock.reason) waitMs=$($transportLock.waitMs)" + return [pscustomobject]@{ + ok = $false + exitCode = -4 + output = "ssh_transport_lock_$($transportLock.reason)" + elapsedMs = $transportLock.waitMs + transportSerialized = $true + transportLockWaitMs = $transportLock.waitMs } } - $last + try { + $last = $null + $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost } + for ($attempt = 1; $attempt -le $Retries; $attempt++) { + $args = @(Get-AgentSshIdentityArguments) + @( + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=$TimeoutSeconds", + "-o", "ServerAliveInterval=20", + "-o", "ServerAliveCountMax=3", + "-l", $userToUse, + $TargetHost, + $Command + ) + + $stopwatch = $null + try { + $stopwatch = [Diagnostics.Stopwatch]::StartNew() + $id = [guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" + $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" + $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru + + if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $stopwatch.Stop() + try { + & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null + } catch {} + if (-not $process.WaitForExit(2000) -and -not $process.HasExited) { + $process.Kill() + $process.WaitForExit(2000) | Out-Null + } + Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + $last = [pscustomobject]@{ + ok = $false + exitCode = -2 + output = "timeout_after_${TimeoutSeconds}s" + elapsedMs = $stopwatch.ElapsedMilliseconds + } + } else { + $process.WaitForExit() | Out-Null + $stopwatch.Stop() + $process.Refresh() + $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } + $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } + Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null + $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n" + $exitCode = $process.ExitCode + if ($null -eq $exitCode) { + $exitCode = if ($stderr) { -3 } else { 0 } + } + $last = [pscustomobject]@{ + ok = ($exitCode -eq 0) + exitCode = $exitCode + output = $text + elapsedMs = $stopwatch.ElapsedMilliseconds + } + } + } catch { + $last = [pscustomobject]@{ + ok = $false + exitCode = -1 + output = $_.Exception.Message + elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null } + } + } + + if ($last.ok) { + if ($attempt -gt 1) { + Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt" + } + return (Add-AgentSshTransportMetadata $last $transportLock) + } + + if ($attempt -lt $Retries) { + Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)" + Start-Sleep -Seconds 2 + } + } + + Add-AgentSshTransportMetadata $last $transportLock + } finally { + Exit-AgentSshTransportLock $transportLock + } } function Quote-ShSingle { diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 01c9ed535..9685c0b24 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -169,6 +169,10 @@ try { $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") + Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{}) + Add-DefaultProperty $config.sshTransport "serialized" $true + Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90 + Add-DefaultProperty $config.sshTransport "lockPollMilliseconds" 200 Add-DefaultProperty $config "sshProcessGuard" ([pscustomobject]@{}) Add-DefaultProperty $config.sshProcessGuard "enabled" $true Add-DefaultProperty $config.sshProcessGuard "staleMinutes" 15 diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 407b64e8b..4d90f5567 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -3,6 +3,11 @@ "agentRoot": "C:\\Wooo\\Agent99", "sshUser": "wooo", "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519", + "sshTransport": { + "serialized": true, + "lockTimeoutSeconds": 90, + "lockPollMilliseconds": 200 + }, "sshProcessGuard": { "enabled": true, "staleMinutes": 15, diff --git a/agent99.config.example.json b/agent99.config.example.json index 9815659e8..411cc6ca1 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -3,6 +3,11 @@ "agentRoot": "C:\\Wooo\\Agent99", "sshUser": "wooo", "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519", + "sshTransport": { + "serialized": true, + "lockTimeoutSeconds": 90, + "lockPollMilliseconds": 200 + }, "sshProcessGuard": { "enabled": true, "staleMinutes": 15, diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 814f37af4..bdc47a1df 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -118,3 +118,21 @@ def test_agent99_bounds_stale_ssh_processes_without_touching_tunnels() -> None: "staleMinutes": 15, "hardStaleMinutes": 60, } + + +def test_agent99_serializes_ssh_across_scheduled_task_processes() -> None: + source = CONTROL.read_text(encoding="utf-8") + config = json.loads((ROOT / "agent99.config.99.example.json").read_text()) + + assert "function Enter-AgentSshTransportLock" in source + assert "[IO.FileShare]::None" in source + assert '"ssh-transport.lock"' in source + invoke_ssh = source[source.index("function Invoke-SshText") :] + assert "Enter-AgentSshTransportLock" in invoke_ssh + assert "Exit-AgentSshTransportLock" in invoke_ssh + assert "ssh_transport_lock_failed" in source + assert config["sshTransport"] == { + "serialized": True, + "lockTimeoutSeconds": 90, + "lockPollMilliseconds": 200, + } diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index a14e8191b..c15ccbe08 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.24 +> **版本**: v5.25 > **最後更新**: 2026-07-10 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-10 全主機重啟後 Agent99 transport、效能修復與證據鏈收斂 +> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention 與 false-down readback 收斂 --- @@ -29,6 +29,8 @@ Agent99 SSH transport 使用 99 本機專用 Ed25519 identity `C:\Wooo\Agent99\k 所有 SSH command 都必須 bounded timeout。Timeout 時先用 `taskkill /T` 終止 process tree，再確認 process 是否仍存活；`Perf` / `LoadShed` controlled apply 執行前會清理只符合以下全部條件的 stale client：Agent99 allowlisted target、`BatchMode=yes`、沒有 `-N/-L/-R/-D` tunnel、超過 15 分鐘且 parent 已消失，或超過 60 分鐘 hard limit。guard 不記錄 command line，不碰互動式 SSH 或 tunnel，結果寫入 `sshProcessGuard` evidence。 +Agent99 的 Heartbeat、Performance、手動 Status 或告警修復可能由不同 Windows process 同時啟動。所有 `Invoke-SshText` 必須先取得 `C:\Wooo\Agent99\state\ssh-transport.lock` 的 exclusive file handle，跨 SYSTEM scheduled task 與 RDP operator session 一次只允許一條 Agent99 SSH command；程序結束時由 OS 自動釋放 handle。預設最多等待 90 秒，逾時回 `ssh_transport_lock_timeout`，不得再開新 client。若同輪五主機 ping 全通，但 Harbor、AWOOOI 與所有 performance readback 同時空白，優先判定為 transport contention，必須等序列化後重驗，不得宣稱所有服務同時故障。 + 效能修復不得以 command exit code 取代 post-verifier。只要 action 配置 verifier，成功判定完全由 verifier 決定；verifier 失敗必須記為 failed，僅建立 10 分鐘 failure backoff，不得建立 success cooldown。policy 變更必須提升 `policyVersion`，由 deployer migration 更新 99 live config，避免舊 command 或 marker 繼續造成 false green。 本輪 live receipt：`agent99-load-shed-marker-correction.json` 已封存 188 舊 false-success marker，`agent99-stale-ssh-cleanup-v2.json` 已受控終止 7 個逾時 orphan SSH client；`agent99-Perf-20260710-224454.json` 顯示 110、112、120、121、188 全部 readback OK，route 分別為 direct、direct、via110、via110、via110。110 load/core `0.60`、disk `84%`；188 load/core `0.18`、disk 由 `90%` 降至 `88%`。這證明 transport、效能讀回與 verifier 修復 lane 已恢復，不等於已完成下一次實際全主機 cold-start 的 10 分鐘 SLA 驗收。