fix(agent99): serialize cross-process SSH transport

AGENT99-AI-WORKSTATION-SOP.md

@@ -733,6 +733,8 @@ Agent99 now uses a source-trusted deployment path, a dedicated bounded SSH trans
   - `policyVersion` migration updates stale live policies and prevents an old command or marker from surviving a new source deployment.
 - Bounded SSH process lifecycle:
   - Every SSH call has a timeout; timeout cleanup terminates the SSH process tree and verifies that the client exited.
+  - Every Agent99 process acquires the shared `C:\Wooo\Agent99\state\ssh-transport.lock` with an exclusive file handle before starting SSH. This serializes scheduled tasks and operator runs across Windows sessions; the OS releases the handle if a process exits.
+  - Lock acquisition waits at most 90 seconds. Timeout evidence is `ssh_transport_lock_timeout`; it must not start another SSH client or be reported as a service outage.
   - Before controlled `Perf` or `LoadShed`, the stale-process guard only considers Agent99 allowlisted targets using `BatchMode=yes` and explicitly excludes `-N`, `-L`, `-R`, and `-D` tunnels.
   - A process is eligible after 15 minutes with a missing parent, or after the 60-minute hard limit. Evidence records selected process metadata but never the command line.
 - Live receipts:

agent99-contract-check.ps1

@@ -67,6 +67,7 @@ $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps
 Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced"
 Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present"
 Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded"
+Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized"
 Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue"
 Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present"
 Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present"

agent99-control-plane.ps1

@@ -66,6 +66,81 @@ function Get-AgentSshIdentityArguments {
   @("-i", $SshIdentityFile, "-o", "IdentitiesOnly=yes")
 }
 
+function Get-AgentSshTransportConfig {
+  $configured = if ($Config.PSObject.Properties["sshTransport"]) { $Config.sshTransport } else { $null }
+  [pscustomobject]@{
+    serialized = [bool](-not $configured -or -not $configured.PSObject.Properties["serialized"] -or (Convert-AgentBool $configured.serialized))
+    lockTimeoutSeconds = if ($configured -and $configured.PSObject.Properties["lockTimeoutSeconds"]) { [math]::Max(10, [int]$configured.lockTimeoutSeconds) } else { 90 }
+    lockPollMilliseconds = if ($configured -and $configured.PSObject.Properties["lockPollMilliseconds"]) { [math]::Min(2000, [math]::Max(50, [int]$configured.lockPollMilliseconds)) } else { 200 }
+    lockPath = Join-Path (Join-Path $Config.agentRoot "state") "ssh-transport.lock"
+  }
+}
+
+function Enter-AgentSshTransportLock {
+  $transport = Get-AgentSshTransportConfig
+  if (-not $transport.serialized) {
+    return [pscustomobject]@{ acquired = $true; disabled = $true; reason = "serialization_disabled"; waitMs = 0; stream = $null }
+  }
+
+  $lockDir = Split-Path -Parent $transport.lockPath
+  New-Item -ItemType Directory -Force -Path $lockDir | Out-Null
+  $stopwatch = [Diagnostics.Stopwatch]::StartNew()
+  while ($stopwatch.Elapsed.TotalSeconds -lt $transport.lockTimeoutSeconds) {
+    try {
+      $stream = [IO.File]::Open(
+        $transport.lockPath,
+        [IO.FileMode]::OpenOrCreate,
+        [IO.FileAccess]::ReadWrite,
+        [IO.FileShare]::None
+      )
+      $stopwatch.Stop()
+      return [pscustomobject]@{
+        acquired = $true
+        disabled = $false
+        reason = "acquired"
+        waitMs = $stopwatch.ElapsedMilliseconds
+        stream = $stream
+      }
+    } catch [System.IO.IOException] {
+      Start-Sleep -Milliseconds $transport.lockPollMilliseconds
+    } catch [System.UnauthorizedAccessException] {
+      $stopwatch.Stop()
+      return [pscustomobject]@{
+        acquired = $false
+        disabled = $false
+        reason = "access_denied"
+        waitMs = $stopwatch.ElapsedMilliseconds
+        stream = $null
+      }
+    }
+  }
+
+  $stopwatch.Stop()
+  [pscustomobject]@{
+    acquired = $false
+    disabled = $false
+    reason = "timeout"
+    waitMs = $stopwatch.ElapsedMilliseconds
+    stream = $null
+  }
+}
+
+function Exit-AgentSshTransportLock {
+  param([object]$TransportLock)
+  if ($TransportLock -and $TransportLock.stream) {
+    try { $TransportLock.stream.Dispose() } catch {}
+  }
+}
+
+function Add-AgentSshTransportMetadata {
+  param([object]$Result, [object]$TransportLock)
+  if ($Result) {
+    $Result | Add-Member -NotePropertyName transportSerialized -NotePropertyValue ([bool](-not $TransportLock.disabled)) -Force
+    $Result | Add-Member -NotePropertyName transportLockWaitMs -NotePropertyValue ([long]$TransportLock.waitMs) -Force
+  }
+  $Result
+}
+
 function Get-AgentSshProcessGuardConfig {
   $configured = if ($Config.PSObject.Properties["sshProcessGuard"]) { $Config.sshProcessGuard } else { $null }
   [pscustomobject]@{
@@ -1275,86 +1350,104 @@ function Invoke-SshText {
     [string]$User = $null
   )
 
-  $last = $null
-  $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost }
-  for ($attempt = 1; $attempt -le $Retries; $attempt++) {
-    $args = @(Get-AgentSshIdentityArguments) + @(
-      "-o", "BatchMode=yes",
-      "-o", "StrictHostKeyChecking=accept-new",
-      "-o", "NumberOfPasswordPrompts=0",
-      "-o", "ConnectTimeout=$TimeoutSeconds",
-      "-o", "ServerAliveInterval=20",
-      "-o", "ServerAliveCountMax=3",
-      "-l", $userToUse,
-      $TargetHost,
-      $Command
-    )
-
-    try {
-      $stopwatch = [Diagnostics.Stopwatch]::StartNew()
-      $id = [guid]::NewGuid().ToString("N")
-      $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out"
-      $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err"
-      $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
-
-      if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
-        $stopwatch.Stop()
-        try {
-          & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null
-        } catch {}
-        if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
-          $process.Kill()
-          $process.WaitForExit(2000) | Out-Null
-        }
-        Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
-        $last = [pscustomobject]@{
-          ok = $false
-          exitCode = -2
-          output = "timeout_after_${TimeoutSeconds}s"
-          elapsedMs = $stopwatch.ElapsedMilliseconds
-        }
-      } else {
-        $process.WaitForExit() | Out-Null
-        $stopwatch.Stop()
-        $process.Refresh()
-        $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" }
-        $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" }
-        Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
-        $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n"
-        $exitCode = $process.ExitCode
-        if ($null -eq $exitCode) {
-          $exitCode = if ($stderr) { -3 } else { 0 }
-        }
-        $last = [pscustomobject]@{
-          ok = ($exitCode -eq 0)
-          exitCode = $exitCode
-          output = $text
-          elapsedMs = $stopwatch.ElapsedMilliseconds
-        }
-      }
-    } catch {
-      $last = [pscustomobject]@{
-        ok = $false
-        exitCode = -1
-        output = $_.Exception.Message
-        elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null }
-      }
-    }
-
-    if ($last.ok) {
-      if ($attempt -gt 1) {
-        Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt"
-      }
-      return $last
-    }
-
-    if ($attempt -lt $Retries) {
-      Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)"
-      Start-Sleep -Seconds 2
+  $transportLock = Enter-AgentSshTransportLock
+  if (-not $transportLock.acquired) {
+    Write-AgentLog "ssh_transport_lock_failed host=$TargetHost reason=$($transportLock.reason) waitMs=$($transportLock.waitMs)"
+    return [pscustomobject]@{
+      ok = $false
+      exitCode = -4
+      output = "ssh_transport_lock_$($transportLock.reason)"
+      elapsedMs = $transportLock.waitMs
+      transportSerialized = $true
+      transportLockWaitMs = $transportLock.waitMs
     }
   }
 
-  $last
+  try {
+    $last = $null
+    $userToUse = if ($User) { $User } else { Get-HostSshUser $TargetHost }
+    for ($attempt = 1; $attempt -le $Retries; $attempt++) {
+      $args = @(Get-AgentSshIdentityArguments) + @(
+        "-o", "BatchMode=yes",
+        "-o", "StrictHostKeyChecking=accept-new",
+        "-o", "NumberOfPasswordPrompts=0",
+        "-o", "ConnectTimeout=$TimeoutSeconds",
+        "-o", "ServerAliveInterval=20",
+        "-o", "ServerAliveCountMax=3",
+        "-l", $userToUse,
+        $TargetHost,
+        $Command
+      )
+
+      $stopwatch = $null
+      try {
+        $stopwatch = [Diagnostics.Stopwatch]::StartNew()
+        $id = [guid]::NewGuid().ToString("N")
+        $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out"
+        $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err"
+        $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
+
+        if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
+          $stopwatch.Stop()
+          try {
+            & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null
+          } catch {}
+          if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
+            $process.Kill()
+            $process.WaitForExit(2000) | Out-Null
+          }
+          Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
+          $last = [pscustomobject]@{
+            ok = $false
+            exitCode = -2
+            output = "timeout_after_${TimeoutSeconds}s"
+            elapsedMs = $stopwatch.ElapsedMilliseconds
+          }
+        } else {
+          $process.WaitForExit() | Out-Null
+          $stopwatch.Stop()
+          $process.Refresh()
+          $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" }
+          $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" }
+          Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
+          $text = (@($stdout, $stderr) | Where-Object { $_ }) -join "`n"
+          $exitCode = $process.ExitCode
+          if ($null -eq $exitCode) {
+            $exitCode = if ($stderr) { -3 } else { 0 }
+          }
+          $last = [pscustomobject]@{
+            ok = ($exitCode -eq 0)
+            exitCode = $exitCode
+            output = $text
+            elapsedMs = $stopwatch.ElapsedMilliseconds
+          }
+        }
+      } catch {
+        $last = [pscustomobject]@{
+          ok = $false
+          exitCode = -1
+          output = $_.Exception.Message
+          elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null }
+        }
+      }
+
+      if ($last.ok) {
+        if ($attempt -gt 1) {
+          Write-AgentLog "ssh_retry_recovered host=$TargetHost attempt=$attempt"
+        }
+        return (Add-AgentSshTransportMetadata $last $transportLock)
+      }
+
+      if ($attempt -lt $Retries) {
+        Write-AgentLog "ssh_retry host=$TargetHost attempt=$attempt exit=$($last.exitCode)"
+        Start-Sleep -Seconds 2
+      }
+    }
+
+    Add-AgentSshTransportMetadata $last $transportLock
+  } finally {
+    Exit-AgentSshTransportLock $transportLock
+  }
 }
 
 function Quote-ShSingle {

agent99-deploy.ps1

@@ -169,6 +169,10 @@ try {
 
   $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json
   Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519")
+  Add-DefaultProperty $config "sshTransport" ([pscustomobject]@{})
+  Add-DefaultProperty $config.sshTransport "serialized" $true
+  Add-DefaultProperty $config.sshTransport "lockTimeoutSeconds" 90
+  Add-DefaultProperty $config.sshTransport "lockPollMilliseconds" 200
   Add-DefaultProperty $config "sshProcessGuard" ([pscustomobject]@{})
   Add-DefaultProperty $config.sshProcessGuard "enabled" $true
   Add-DefaultProperty $config.sshProcessGuard "staleMinutes" 15

agent99.config.99.example.json

@@ -3,6 +3,11 @@
   "agentRoot": "C:\\Wooo\\Agent99",
   "sshUser": "wooo",
   "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519",
+  "sshTransport": {
+    "serialized": true,
+    "lockTimeoutSeconds": 90,
+    "lockPollMilliseconds": 200
+  },
   "sshProcessGuard": {
     "enabled": true,
     "staleMinutes": 15,

agent99.config.example.json

@@ -3,6 +3,11 @@
   "agentRoot": "C:\\Wooo\\Agent99",
   "sshUser": "wooo",
   "sshIdentityFile": "C:\\Wooo\\Agent99\\keys\\agent99_ed25519",
+  "sshTransport": {
+    "serialized": true,
+    "lockTimeoutSeconds": 90,
+    "lockPollMilliseconds": 200
+  },
   "sshProcessGuard": {
     "enabled": true,
     "staleMinutes": 15,

apps/api/tests/test_agent99_transport_recovery_deploy_contract.py

@@ -118,3 +118,21 @@ def test_agent99_bounds_stale_ssh_processes_without_touching_tunnels() -> None:
         "staleMinutes": 15,
         "hardStaleMinutes": 60,
     }
+
+
+def test_agent99_serializes_ssh_across_scheduled_task_processes() -> None:
+    source = CONTROL.read_text(encoding="utf-8")
+    config = json.loads((ROOT / "agent99.config.99.example.json").read_text())
+
+    assert "function Enter-AgentSshTransportLock" in source
+    assert "[IO.FileShare]::None" in source
+    assert '"ssh-transport.lock"' in source
+    invoke_ssh = source[source.index("function Invoke-SshText") :]
+    assert "Enter-AgentSshTransportLock" in invoke_ssh
+    assert "Exit-AgentSshTransportLock" in invoke_ssh
+    assert "ssh_transport_lock_failed" in source
+    assert config["sshTransport"] == {
+        "serialized": True,
+        "lockTimeoutSeconds": 90,
+        "lockPollMilliseconds": 200,
+    }

docs/runbooks/REBOOT-RECOVERY-SOP.md

@@ -1,9 +1,9 @@
 # AWOOOI 重開機恢復 SOP
 
-> **版本**: v5.24
+> **版本**: v5.25
 > **最後更新**: 2026-07-10 (台北時間)
 > **更新者**: Codex
-> **觸發事件**: 2026-07-10 全主機重啟後 Agent99 transport、效能修復與證據鏈收斂
+> **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention 與 false-down readback 收斂
 
 ---
 
@@ -29,6 +29,8 @@ Agent99 SSH transport 使用 99 本機專用 Ed25519 identity `C:\Wooo\Agent99\k
 
 所有 SSH command 都必須 bounded timeout。Timeout 時先用 `taskkill /T` 終止 process tree，再確認 process 是否仍存活；`Perf` / `LoadShed` controlled apply 執行前會清理只符合以下全部條件的 stale client：Agent99 allowlisted target、`BatchMode=yes`、沒有 `-N/-L/-R/-D` tunnel、超過 15 分鐘且 parent 已消失，或超過 60 分鐘 hard limit。guard 不記錄 command line，不碰互動式 SSH 或 tunnel，結果寫入 `sshProcessGuard` evidence。
 
+Agent99 的 Heartbeat、Performance、手動 Status 或告警修復可能由不同 Windows process 同時啟動。所有 `Invoke-SshText` 必須先取得 `C:\Wooo\Agent99\state\ssh-transport.lock` 的 exclusive file handle，跨 SYSTEM scheduled task 與 RDP operator session 一次只允許一條 Agent99 SSH command；程序結束時由 OS 自動釋放 handle。預設最多等待 90 秒，逾時回 `ssh_transport_lock_timeout`，不得再開新 client。若同輪五主機 ping 全通，但 Harbor、AWOOOI 與所有 performance readback 同時空白，優先判定為 transport contention，必須等序列化後重驗，不得宣稱所有服務同時故障。
+
 效能修復不得以 command exit code 取代 post-verifier。只要 action 配置 verifier，成功判定完全由 verifier 決定；verifier 失敗必須記為 failed，僅建立 10 分鐘 failure backoff，不得建立 success cooldown。policy 變更必須提升 `policyVersion`，由 deployer migration 更新 99 live config，避免舊 command 或 marker 繼續造成 false green。
 
 本輪 live receipt：`agent99-load-shed-marker-correction.json` 已封存 188 舊 false-success marker，`agent99-stale-ssh-cleanup-v2.json` 已受控終止 7 個逾時 orphan SSH client；`agent99-Perf-20260710-224454.json` 顯示 110、112、120、121、188 全部 readback OK，route 分別為 direct、direct、via110、via110、via110。110 load/core `0.60`、disk `84%`；188 load/core `0.18`、disk 由 `90%` 降至 `88%`。這證明 transport、效能讀回與 verifier 修復 lane 已恢復，不等於已完成下一次實際全主機 cold-start 的 10 分鐘 SLA 驗收。