fix(agent99): prioritize recovery transport lifecycle

2026-07-14 14:49:46 +08:00
parent a661824f33
commit d1b28a8375
3 changed files with 292 additions and 8 deletions
--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -124,6 +124,57 @@ function Convert-AgentPublicReceiptLeaf {
  return $safe
 }

+function Get-AgentSafeHttpFailureReceipt {
+  param([object]$ErrorRecord)
+
+  $httpStatus = $null
+  $detailCode = "http_error"
+  $errorType = "unknown"
+  $response = $null
+  try { $errorType = [string]$ErrorRecord.Exception.GetType().Name } catch {}
+  try { $response = $ErrorRecord.Exception.Response } catch {}
+  if ($response) {
+    try { $httpStatus = [int]$response.StatusCode } catch {}
+    $stream = $null
+    $reader = $null
+    try {
+      $stream = $response.GetResponseStream()
+      if ($stream) {
+        $reader = New-Object IO.StreamReader($stream)
+        $body = [string]$reader.ReadToEnd()
+        if ($body -and $body.Length -le 4096) {
+          $parsed = $body | ConvertFrom-Json
+          $detail = if ($parsed -and $parsed.PSObject.Properties["detail"]) { $parsed.detail } else { $null }
+          if ($detail -is [string] -and $detail -match "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$") {
+            $detailCode = [string]$detail
+          } elseif ($detail -is [System.Collections.IEnumerable]) {
+            $detailCode = "request_validation_failed"
+          }
+        }
+      }
+    } catch {
+      $detailCode = "http_error_detail_unavailable"
+    } finally {
+      if ($reader) { try { $reader.Dispose() } catch {} }
+      elseif ($stream) { try { $stream.Dispose() } catch {} }
+    }
+  }
+  $retryable = [bool](
+    $null -eq $httpStatus -or
+    $httpStatus -eq 408 -or
+    $httpStatus -eq 429 -or
+    $httpStatus -ge 500
+  )
+  [pscustomobject]@{
+    httpStatus = $httpStatus
+    errorCode = $detailCode
+    errorType = $errorType
+    retryable = $retryable
+    rawResponseStored = $false
+    secretValueLogged = $false
+  }
+}
+
 function Send-AgentOutcomeWriteback {
  param(
    [object]$Result,
@@ -161,7 +212,10 @@ function Send-AgentOutcomeWriteback {
    learning_receipt_refs = [pscustomobject]@{}
  } | ConvertTo-Json -Depth 12 -Compress
  $lastStatus = "outcome_writeback_failed"
+  $lastFailure = $null
+  $attemptsMade = 0
  for ($attempt = 1; $attempt -le 3; $attempt++) {
+    $attemptsMade = $attempt
    try {
      $response = Invoke-RestMethod -Method Post -Uri $url -Headers @{ "X-Agent99-Relay-Token" = $token } -ContentType "application/json" -Body $body -TimeoutSec 20
      return [pscustomobject]@{
@@ -169,13 +223,34 @@ function Send-AgentOutcomeWriteback {
        status = if ($response -and $response.status) { [string]$response.status } else { "outcome_writeback_acknowledged" }
        attempts = $attempt
        runtimeClosureVerified = [bool]($response -and $response.runtime_closure_verified -eq $true)
+        httpStatus = 200
+        errorCode = $null
+        rawResponseStored = $false
+        secretValueLogged = $false
      }
    } catch {
      $lastStatus = "outcome_writeback_http_error"
-      if ($attempt -lt 3) { Start-Sleep -Seconds 2 }
+      $lastFailure = Get-AgentSafeHttpFailureReceipt $_
+      Write-AgentLog "outcome_writeback_failed attempt=$attempt httpStatus=$($lastFailure.httpStatus) errorCode=$($lastFailure.errorCode) retryable=$($lastFailure.retryable)"
+      if ($attempt -lt 3 -and $lastFailure.retryable) {
+        Start-Sleep -Seconds 2
+      } else {
+        break
+      }
    }
  }
-  [pscustomobject]@{ ok = $false; status = $lastStatus; attempts = 3; runtimeClosureVerified = $false }
+  [pscustomobject]@{
+    ok = $false
+    status = $lastStatus
+    attempts = $attemptsMade
+    runtimeClosureVerified = $false
+    httpStatus = if ($lastFailure) { $lastFailure.httpStatus } else { $null }
+    errorCode = if ($lastFailure) { $lastFailure.errorCode } else { "http_error" }
+    errorType = if ($lastFailure) { $lastFailure.errorType } else { "unknown" }
+    retryable = [bool]($lastFailure -and $lastFailure.retryable)
+    rawResponseStored = $false
+    secretValueLogged = $false
+  }
 }

 function Get-HostSshUser {
@@ -201,6 +276,124 @@ function Get-AgentSshTransportConfig {
  }
 }

+function New-AgentRecoveryTransportMutex {
+  [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
+}
+
+function Test-AgentRecoveryTransportLease {
+  $mutex = $null
+  $acquired = $false
+  $abandoned = $false
+  try {
+    $mutex = New-AgentRecoveryTransportMutex
+    try {
+      $acquired = [bool]$mutex.WaitOne(0)
+    } catch [Threading.AbandonedMutexException] {
+      $acquired = $true
+      $abandoned = $true
+    }
+    [pscustomobject]@{
+      active = [bool](-not $acquired)
+      available = $true
+      abandonedRecovered = $abandoned
+      reason = if ($acquired) { "available" } else { "recovery_active" }
+      mutexName = "Global\WoooAgent99RecoveryTransportV1"
+    }
+  } catch {
+    [pscustomobject]@{
+      active = $true
+      available = $false
+      abandonedRecovered = $false
+      reason = "recovery_lease_probe_failed"
+      mutexName = "Global\WoooAgent99RecoveryTransportV1"
+    }
+  } finally {
+    if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
+    if ($mutex) { try { $mutex.Dispose() } catch {} }
+  }
+}
+
+function Enter-AgentRecoveryTransportLease {
+  param([string]$RunId)
+
+  $mutex = $null
+  $acquired = $false
+  $abandoned = $false
+  try {
+    $mutex = New-AgentRecoveryTransportMutex
+    try {
+      $acquired = [bool]$mutex.WaitOne(0)
+    } catch [Threading.AbandonedMutexException] {
+      $acquired = $true
+      $abandoned = $true
+    }
+    if (-not $acquired) {
+      $mutex.Dispose()
+      return [pscustomobject]@{
+        acquired = $false
+        reason = "recovery_already_active"
+        runId = $RunId
+        mutex = $null
+        metadataPath = $null
+        metadataWritten = $false
+        abandonedRecovered = $false
+      }
+    }
+
+    $stateDir = Join-Path $Config.agentRoot "state"
+    $metadataPath = Join-Path $stateDir "recovery-transport-lease.json"
+    New-Item -ItemType Directory -Force -Path $stateDir | Out-Null
+    $metadataWritten = $false
+    try {
+      [pscustomobject]@{
+        schemaVersion = "agent99_recovery_transport_lease_v1"
+        runId = $RunId
+        traceId = $TraceId
+        workItemId = $WorkItemId
+        processId = $PID
+        acquiredAt = (Get-Date -Format o)
+        purpose = "exclusive_recover_ssh_transport"
+        storesSecret = $false
+      } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $metadataPath -Encoding UTF8
+      $metadataWritten = $true
+    } catch {
+      Write-AgentLog "recovery_transport_lease_metadata_failed runId=$RunId errorType=$($_.Exception.GetType().Name)"
+    }
+    [pscustomobject]@{
+      acquired = $true
+      reason = "acquired"
+      runId = $RunId
+      mutex = $mutex
+      metadataPath = $metadataPath
+      metadataWritten = $metadataWritten
+      abandonedRecovered = $abandoned
+    }
+  } catch {
+    if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
+    if ($mutex) { try { $mutex.Dispose() } catch {} }
+    [pscustomobject]@{
+      acquired = $false
+      reason = "recovery_lease_acquire_failed"
+      runId = $RunId
+      mutex = $null
+      metadataPath = $null
+      metadataWritten = $false
+      abandonedRecovered = $false
+    }
+  }
+}
+
+function Exit-AgentRecoveryTransportLease {
+  param([object]$Lease)
+  if (-not $Lease -or -not $Lease.acquired -or -not $Lease.mutex) { return $false }
+  if ($Lease.metadataPath) {
+    Remove-Item -LiteralPath $Lease.metadataPath -Force -ErrorAction SilentlyContinue | Out-Null
+  }
+  try { $Lease.mutex.ReleaseMutex() } catch {}
+  try { $Lease.mutex.Dispose() } catch {}
+  return $true
+}
+
 function Enter-AgentSshTransportLock {
  $transport = Get-AgentSshTransportConfig
  if (-not $transport.serialized) {
@@ -830,6 +1023,7 @@ function Get-AgentIncidentCardModel {
  $targetName = [string](Get-AgentObjectValue $Data "target" (Get-AgentObjectValue $Data "alertService" ""))
  $alertId = [string](Get-AgentObjectValue $Data "alertId" "")
  $correlationKey = [string](Get-AgentObjectValue $Data "correlationKey" "")
+  $runKey = [string](Get-AgentObjectValue $Data "runId" (Get-AgentObjectValue $Data "automationRunId" (Get-AgentObjectValue $Data "id" "")))
  $problem = if ($Data -and $Data.PSObject.Properties["problem"]) { $Data.problem } else { $null }
  $outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null }
  $problemKey = [string](Get-AgentObjectValue $problem "key" "")
@@ -1092,6 +1286,7 @@ function Get-AgentIncidentCardModel {
    lifecycle = $lifecycle
    lifecycleLabel = $lifecycleLabel
    stateKey = "$lifecycle|$Severity"
+    runKey = $runKey
    title = Limit-AgentTextLine $title 96
    target = Limit-AgentTextLine $target 96
    impact = Limit-AgentTextLine $impact 240
@@ -1193,6 +1388,7 @@ function Save-AgentTelegramIncidentState {
    incidentId = $Card.incidentId
    fingerprint = $Card.fingerprint
    lastStateKey = $Card.stateKey
+    lastRunKey = $Card.runKey
    lastLifecycle = $Card.lifecycle
    lastSeverity = $Card.severity
    rootMessageId = $rootMessageId
@@ -1971,7 +2167,13 @@ function Record-AgentEvent {
    $incidentCard = Get-AgentIncidentCardModel $Severity $EventType $Message $Data
    $incidentState = Get-AgentTelegramIncidentState $incidentCard
    $forceLifecycleNotification = [bool]($Data -and $Data.PSObject.Properties["forceLifecycleNotification"] -and (Convert-AgentBool $Data.forceLifecycleNotification))
-    if (-not $forceLifecycleNotification -and $incidentState.value -and [string]$incidentState.value.lastStateKey -eq $incidentCard.stateKey) {
+    $previousRunKey = if ($incidentState.value -and $incidentState.value.PSObject.Properties["lastRunKey"]) { [string]$incidentState.value.lastRunKey } else { "" }
+    $sameLifecycleRun = [bool](
+      $incidentState.value -and
+      [string]$incidentState.value.lastStateKey -eq $incidentCard.stateKey -and
+      (-not $incidentCard.runKey -or $previousRunKey -eq [string]$incidentCard.runKey)
+    )
+    if (-not $forceLifecycleNotification -and $sameLifecycleRun) {
      Write-AgentLog "telegram_lifecycle_suppressed incident=$($incidentCard.incidentId) state=$($incidentCard.stateKey)"
      $script:TelegramAttempts += [pscustomobject]@{
        timestamp = (Get-Date -Format o)
@@ -1994,6 +2196,7 @@ function Record-AgentEvent {
    }
    $dedupeParts = @($EventType, $Severity)
    $dedupeParts += "lifecycle=$($incidentCard.stateKey)"
+    if ($incidentCard.runKey) { $dedupeParts += "run=$($incidentCard.runKey)" }
    if ($Data) {
      foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source", "scope", "completed", "withinSlo", "rebootSloClaimed")) {
        if ($Data.PSObject.Properties[$propName]) {
@@ -6990,6 +7193,62 @@ if ($SelfTestBackupReadbackContract) {
  exit 0
 }

+$recoveryRunId = if ($Mode -eq "Recover") {
+  if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) }
+} else {
+  ""
+}
+$recoveryTransportLease = $null
+$recoveryTransportLeaseReceipt = $null
+if ($Mode -eq "Recover") {
+  $recoveryTransportLease = Enter-AgentRecoveryTransportLease $recoveryRunId
+  $recoveryTransportLeaseReceipt = [pscustomobject]@{
+    schemaVersion = "agent99_recovery_transport_lease_receipt_v1"
+    acquired = [bool]$recoveryTransportLease.acquired
+    reason = [string]$recoveryTransportLease.reason
+    runId = $recoveryRunId
+    metadataWritten = [bool]$recoveryTransportLease.metadataWritten
+    metadataRef = if ($recoveryTransportLease.metadataPath) { Convert-AgentPublicReceiptLeaf $recoveryTransportLease.metadataPath } else { $null }
+    abandonedRecovered = [bool]$recoveryTransportLease.abandonedRecovered
+    released = $false
+  }
+  if (-not $recoveryTransportLease.acquired) {
+    $deferred = [pscustomobject]@{
+      schemaVersion = "agent99_recovery_transport_deferred_v1"
+      timestamp = (Get-Date -Format o)
+      mode = $Mode
+      controlledApply = [bool]$ControlledApply
+      deferred = $true
+      reason = [string]$recoveryTransportLease.reason
+      recoveryTransportLease = $recoveryTransportLeaseReceipt
+      evidenceLog = $logPath
+    }
+    Write-AgentLog "recovery_transport_deferred runId=$recoveryRunId reason=$($recoveryTransportLease.reason)"
+    $deferred | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath
+    $deferred | ConvertTo-Json -Depth 6
+    exit 0
+  }
+} elseif ($Mode -in @("Status", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck")) {
+  $recoveryLeaseObservation = Test-AgentRecoveryTransportLease
+  if ($recoveryLeaseObservation.active) {
+    $deferred = [pscustomobject]@{
+      schemaVersion = "agent99_background_mode_deferred_v1"
+      timestamp = (Get-Date -Format o)
+      mode = $Mode
+      controlledApply = [bool]$ControlledApply
+      deferred = $true
+      reason = [string]$recoveryLeaseObservation.reason
+      recoveryTransportLease = $recoveryLeaseObservation
+      runtimeWritePerformed = $false
+      evidenceLog = $logPath
+    }
+    Write-AgentLog "background_mode_deferred mode=$Mode reason=$($recoveryLeaseObservation.reason)"
+    $deferred | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath
+    $deferred | ConvertTo-Json -Depth 6
+    exit 0
+  }
+}
+
 Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath"
 if ($ReconcileOnly -or $ReconcileQueueOnly) {
  Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -NoAlert
@@ -7020,11 +7279,6 @@ $recoverySlo = $null
 $recoveryCoordinator = $null
 $recoveryPhases = @()
 $recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null }
-$recoveryRunId = if ($Mode -eq "Recover") {
-  if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) }
-} else {
-  ""
-}
 $recoveryConfig = Get-AgentRecoverySloConfig

 if ($Mode -in @("Status", "Recover")) {
@@ -7269,12 +7523,19 @@ $result = [pscustomobject]@{
  providerFreshness = $providerFreshness
  securityTriage = $securityTriage
  controlTick = $controlTick
+  recoveryTransportLease = $recoveryTransportLeaseReceipt
  events = $script:Events
  telegram = $script:TelegramAttempts
  evidenceLog = $logPath
 }

 $result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath
+if ($recoveryTransportLease -and $recoveryTransportLease.acquired) {
+  $released = Exit-AgentRecoveryTransportLease $recoveryTransportLease
+  $recoveryTransportLeaseReceipt | Add-Member -NotePropertyName released -NotePropertyValue ([bool]$released) -Force
+  $result | Add-Member -NotePropertyName recoveryTransportLease -NotePropertyValue $recoveryTransportLeaseReceipt -Force
+  $result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath
+}
 Write-AgentLog "agent99_complete json=$jsonPath"

 exit 0
--- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py
+++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py
@@ -78,6 +78,12 @@ def test_agent99_posts_fenced_outcome_to_production_ingestion() -> None:
    assert "agent99_outcome_receipt_id" in source
    assert "post_verifier_evidence_ref" in source
    assert "source_event_evidence_ref" in source
+    assert "function Get-AgentSafeHttpFailureReceipt" in source
+    assert "httpStatus = $httpStatus" in source
+    assert "errorCode = $detailCode" in source
+    assert "rawResponseStored = $false" in source
+    assert "secretValueLogged = $false" in source
+    assert "$lastFailure.retryable" in source
    for field in (
        "projectId",
        "sourceFingerprint",
--- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py
+++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py
@@ -107,6 +107,9 @@ def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None:
    assert 'rootMessageId = $rootMessageId' in source
    assert 'reason = "same_lifecycle_state"' in source
    assert 'reason = "dedupe_window"' in source
+    assert "lastRunKey = $Card.runKey" in source
+    assert "$previousRunKey -eq [string]$incidentCard.runKey" in source
+    assert '$dedupeParts += "run=$($incidentCard.runKey)"' in source


 def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None:
@@ -410,6 +413,20 @@ def test_agent99_serializes_ssh_across_scheduled_task_processes() -> None:
    }


+def test_agent99_recover_owns_transport_and_background_modes_defer() -> None:
+    source = CONTROL.read_text(encoding="utf-8")
+
+    assert "function Enter-AgentRecoveryTransportLease" in source
+    assert "function Test-AgentRecoveryTransportLease" in source
+    assert "function Exit-AgentRecoveryTransportLease" in source
+    assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
+    assert 'schemaVersion = "agent99_recovery_transport_lease_v1"' in source
+    assert 'schemaVersion = "agent99_background_mode_deferred_v1"' in source
+    assert 'reason = "recovery_already_active"' in source
+    assert 'purpose = "exclusive_recover_ssh_transport"' in source
+    assert "Exit-AgentRecoveryTransportLease $recoveryTransportLease" in source
+
+
 def test_agent99_emits_transport_wait_and_keeps_running_relay_healthy() -> None:
    source = CONTROL.read_text(encoding="utf-8")
    config = json.loads((ROOT / "agent99.config.99.example.json").read_text())