diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index b1015e916..60205aeba 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -124,6 +124,57 @@ function Convert-AgentPublicReceiptLeaf { return $safe } +function Get-AgentSafeHttpFailureReceipt { + param([object]$ErrorRecord) + + $httpStatus = $null + $detailCode = "http_error" + $errorType = "unknown" + $response = $null + try { $errorType = [string]$ErrorRecord.Exception.GetType().Name } catch {} + try { $response = $ErrorRecord.Exception.Response } catch {} + if ($response) { + try { $httpStatus = [int]$response.StatusCode } catch {} + $stream = $null + $reader = $null + try { + $stream = $response.GetResponseStream() + if ($stream) { + $reader = New-Object IO.StreamReader($stream) + $body = [string]$reader.ReadToEnd() + if ($body -and $body.Length -le 4096) { + $parsed = $body | ConvertFrom-Json + $detail = if ($parsed -and $parsed.PSObject.Properties["detail"]) { $parsed.detail } else { $null } + if ($detail -is [string] -and $detail -match "^[A-Za-z0-9][A-Za-z0-9_.:-]{0,159}$") { + $detailCode = [string]$detail + } elseif ($detail -is [System.Collections.IEnumerable]) { + $detailCode = "request_validation_failed" + } + } + } + } catch { + $detailCode = "http_error_detail_unavailable" + } finally { + if ($reader) { try { $reader.Dispose() } catch {} } + elseif ($stream) { try { $stream.Dispose() } catch {} } + } + } + $retryable = [bool]( + $null -eq $httpStatus -or + $httpStatus -eq 408 -or + $httpStatus -eq 429 -or + $httpStatus -ge 500 + ) + [pscustomobject]@{ + httpStatus = $httpStatus + errorCode = $detailCode + errorType = $errorType + retryable = $retryable + rawResponseStored = $false + secretValueLogged = $false + } +} + function Send-AgentOutcomeWriteback { param( [object]$Result, @@ -161,7 +212,10 @@ function Send-AgentOutcomeWriteback { learning_receipt_refs = [pscustomobject]@{} } | ConvertTo-Json -Depth 12 -Compress $lastStatus = "outcome_writeback_failed" + $lastFailure = $null + $attemptsMade = 0 for ($attempt = 1; $attempt -le 3; $attempt++) { + $attemptsMade = $attempt try { $response = Invoke-RestMethod -Method Post -Uri $url -Headers @{ "X-Agent99-Relay-Token" = $token } -ContentType "application/json" -Body $body -TimeoutSec 20 return [pscustomobject]@{ @@ -169,13 +223,34 @@ function Send-AgentOutcomeWriteback { status = if ($response -and $response.status) { [string]$response.status } else { "outcome_writeback_acknowledged" } attempts = $attempt runtimeClosureVerified = [bool]($response -and $response.runtime_closure_verified -eq $true) + httpStatus = 200 + errorCode = $null + rawResponseStored = $false + secretValueLogged = $false } } catch { $lastStatus = "outcome_writeback_http_error" - if ($attempt -lt 3) { Start-Sleep -Seconds 2 } + $lastFailure = Get-AgentSafeHttpFailureReceipt $_ + Write-AgentLog "outcome_writeback_failed attempt=$attempt httpStatus=$($lastFailure.httpStatus) errorCode=$($lastFailure.errorCode) retryable=$($lastFailure.retryable)" + if ($attempt -lt 3 -and $lastFailure.retryable) { + Start-Sleep -Seconds 2 + } else { + break + } } } - [pscustomobject]@{ ok = $false; status = $lastStatus; attempts = 3; runtimeClosureVerified = $false } + [pscustomobject]@{ + ok = $false + status = $lastStatus + attempts = $attemptsMade + runtimeClosureVerified = $false + httpStatus = if ($lastFailure) { $lastFailure.httpStatus } else { $null } + errorCode = if ($lastFailure) { $lastFailure.errorCode } else { "http_error" } + errorType = if ($lastFailure) { $lastFailure.errorType } else { "unknown" } + retryable = [bool]($lastFailure -and $lastFailure.retryable) + rawResponseStored = $false + secretValueLogged = $false + } } function Get-HostSshUser { @@ -201,6 +276,124 @@ function Get-AgentSshTransportConfig { } } +function New-AgentRecoveryTransportMutex { + [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1") +} + +function Test-AgentRecoveryTransportLease { + $mutex = $null + $acquired = $false + $abandoned = $false + try { + $mutex = New-AgentRecoveryTransportMutex + try { + $acquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $acquired = $true + $abandoned = $true + } + [pscustomobject]@{ + active = [bool](-not $acquired) + available = $true + abandonedRecovered = $abandoned + reason = if ($acquired) { "available" } else { "recovery_active" } + mutexName = "Global\WoooAgent99RecoveryTransportV1" + } + } catch { + [pscustomobject]@{ + active = $true + available = $false + abandonedRecovered = $false + reason = "recovery_lease_probe_failed" + mutexName = "Global\WoooAgent99RecoveryTransportV1" + } + } finally { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { try { $mutex.Dispose() } catch {} } + } +} + +function Enter-AgentRecoveryTransportLease { + param([string]$RunId) + + $mutex = $null + $acquired = $false + $abandoned = $false + try { + $mutex = New-AgentRecoveryTransportMutex + try { + $acquired = [bool]$mutex.WaitOne(0) + } catch [Threading.AbandonedMutexException] { + $acquired = $true + $abandoned = $true + } + if (-not $acquired) { + $mutex.Dispose() + return [pscustomobject]@{ + acquired = $false + reason = "recovery_already_active" + runId = $RunId + mutex = $null + metadataPath = $null + metadataWritten = $false + abandonedRecovered = $false + } + } + + $stateDir = Join-Path $Config.agentRoot "state" + $metadataPath = Join-Path $stateDir "recovery-transport-lease.json" + New-Item -ItemType Directory -Force -Path $stateDir | Out-Null + $metadataWritten = $false + try { + [pscustomobject]@{ + schemaVersion = "agent99_recovery_transport_lease_v1" + runId = $RunId + traceId = $TraceId + workItemId = $WorkItemId + processId = $PID + acquiredAt = (Get-Date -Format o) + purpose = "exclusive_recover_ssh_transport" + storesSecret = $false + } | ConvertTo-Json -Depth 5 | Set-Content -LiteralPath $metadataPath -Encoding UTF8 + $metadataWritten = $true + } catch { + Write-AgentLog "recovery_transport_lease_metadata_failed runId=$RunId errorType=$($_.Exception.GetType().Name)" + } + [pscustomobject]@{ + acquired = $true + reason = "acquired" + runId = $RunId + mutex = $mutex + metadataPath = $metadataPath + metadataWritten = $metadataWritten + abandonedRecovered = $abandoned + } + } catch { + if ($acquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} } + if ($mutex) { try { $mutex.Dispose() } catch {} } + [pscustomobject]@{ + acquired = $false + reason = "recovery_lease_acquire_failed" + runId = $RunId + mutex = $null + metadataPath = $null + metadataWritten = $false + abandonedRecovered = $false + } + } +} + +function Exit-AgentRecoveryTransportLease { + param([object]$Lease) + if (-not $Lease -or -not $Lease.acquired -or -not $Lease.mutex) { return $false } + if ($Lease.metadataPath) { + Remove-Item -LiteralPath $Lease.metadataPath -Force -ErrorAction SilentlyContinue | Out-Null + } + try { $Lease.mutex.ReleaseMutex() } catch {} + try { $Lease.mutex.Dispose() } catch {} + return $true +} + function Enter-AgentSshTransportLock { $transport = Get-AgentSshTransportConfig if (-not $transport.serialized) { @@ -830,6 +1023,7 @@ function Get-AgentIncidentCardModel { $targetName = [string](Get-AgentObjectValue $Data "target" (Get-AgentObjectValue $Data "alertService" "")) $alertId = [string](Get-AgentObjectValue $Data "alertId" "") $correlationKey = [string](Get-AgentObjectValue $Data "correlationKey" "") + $runKey = [string](Get-AgentObjectValue $Data "runId" (Get-AgentObjectValue $Data "automationRunId" (Get-AgentObjectValue $Data "id" ""))) $problem = if ($Data -and $Data.PSObject.Properties["problem"]) { $Data.problem } else { $null } $outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null } $problemKey = [string](Get-AgentObjectValue $problem "key" "") @@ -1092,6 +1286,7 @@ function Get-AgentIncidentCardModel { lifecycle = $lifecycle lifecycleLabel = $lifecycleLabel stateKey = "$lifecycle|$Severity" + runKey = $runKey title = Limit-AgentTextLine $title 96 target = Limit-AgentTextLine $target 96 impact = Limit-AgentTextLine $impact 240 @@ -1193,6 +1388,7 @@ function Save-AgentTelegramIncidentState { incidentId = $Card.incidentId fingerprint = $Card.fingerprint lastStateKey = $Card.stateKey + lastRunKey = $Card.runKey lastLifecycle = $Card.lifecycle lastSeverity = $Card.severity rootMessageId = $rootMessageId @@ -1971,7 +2167,13 @@ function Record-AgentEvent { $incidentCard = Get-AgentIncidentCardModel $Severity $EventType $Message $Data $incidentState = Get-AgentTelegramIncidentState $incidentCard $forceLifecycleNotification = [bool]($Data -and $Data.PSObject.Properties["forceLifecycleNotification"] -and (Convert-AgentBool $Data.forceLifecycleNotification)) - if (-not $forceLifecycleNotification -and $incidentState.value -and [string]$incidentState.value.lastStateKey -eq $incidentCard.stateKey) { + $previousRunKey = if ($incidentState.value -and $incidentState.value.PSObject.Properties["lastRunKey"]) { [string]$incidentState.value.lastRunKey } else { "" } + $sameLifecycleRun = [bool]( + $incidentState.value -and + [string]$incidentState.value.lastStateKey -eq $incidentCard.stateKey -and + (-not $incidentCard.runKey -or $previousRunKey -eq [string]$incidentCard.runKey) + ) + if (-not $forceLifecycleNotification -and $sameLifecycleRun) { Write-AgentLog "telegram_lifecycle_suppressed incident=$($incidentCard.incidentId) state=$($incidentCard.stateKey)" $script:TelegramAttempts += [pscustomobject]@{ timestamp = (Get-Date -Format o) @@ -1994,6 +2196,7 @@ function Record-AgentEvent { } $dedupeParts = @($EventType, $Severity) $dedupeParts += "lifecycle=$($incidentCard.stateKey)" + if ($incidentCard.runKey) { $dedupeParts += "run=$($incidentCard.runKey)" } if ($Data) { foreach ($propName in @("host", "name", "path", "reason", "target", "status", "mode", "alertKind", "source", "scope", "completed", "withinSlo", "rebootSloClaimed")) { if ($Data.PSObject.Properties[$propName]) { @@ -6990,6 +7193,62 @@ if ($SelfTestBackupReadbackContract) { exit 0 } +$recoveryRunId = if ($Mode -eq "Recover") { + if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } +} else { + "" +} +$recoveryTransportLease = $null +$recoveryTransportLeaseReceipt = $null +if ($Mode -eq "Recover") { + $recoveryTransportLease = Enter-AgentRecoveryTransportLease $recoveryRunId + $recoveryTransportLeaseReceipt = [pscustomobject]@{ + schemaVersion = "agent99_recovery_transport_lease_receipt_v1" + acquired = [bool]$recoveryTransportLease.acquired + reason = [string]$recoveryTransportLease.reason + runId = $recoveryRunId + metadataWritten = [bool]$recoveryTransportLease.metadataWritten + metadataRef = if ($recoveryTransportLease.metadataPath) { Convert-AgentPublicReceiptLeaf $recoveryTransportLease.metadataPath } else { $null } + abandonedRecovered = [bool]$recoveryTransportLease.abandonedRecovered + released = $false + } + if (-not $recoveryTransportLease.acquired) { + $deferred = [pscustomobject]@{ + schemaVersion = "agent99_recovery_transport_deferred_v1" + timestamp = (Get-Date -Format o) + mode = $Mode + controlledApply = [bool]$ControlledApply + deferred = $true + reason = [string]$recoveryTransportLease.reason + recoveryTransportLease = $recoveryTransportLeaseReceipt + evidenceLog = $logPath + } + Write-AgentLog "recovery_transport_deferred runId=$recoveryRunId reason=$($recoveryTransportLease.reason)" + $deferred | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath + $deferred | ConvertTo-Json -Depth 6 + exit 0 + } +} elseif ($Mode -in @("Status", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck")) { + $recoveryLeaseObservation = Test-AgentRecoveryTransportLease + if ($recoveryLeaseObservation.active) { + $deferred = [pscustomobject]@{ + schemaVersion = "agent99_background_mode_deferred_v1" + timestamp = (Get-Date -Format o) + mode = $Mode + controlledApply = [bool]$ControlledApply + deferred = $true + reason = [string]$recoveryLeaseObservation.reason + recoveryTransportLease = $recoveryLeaseObservation + runtimeWritePerformed = $false + evidenceLog = $logPath + } + Write-AgentLog "background_mode_deferred mode=$Mode reason=$($recoveryLeaseObservation.reason)" + $deferred | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath + $deferred | ConvertTo-Json -Depth 6 + exit 0 + } +} + Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" if ($ReconcileOnly -or $ReconcileQueueOnly) { Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -NoAlert @@ -7020,11 +7279,6 @@ $recoverySlo = $null $recoveryCoordinator = $null $recoveryPhases = @() $recoveryStartedAt = if ($Mode -eq "Recover") { Get-Date } else { $null } -$recoveryRunId = if ($Mode -eq "Recover") { - if ($AutomationRunId) { $AutomationRunId } else { "agent99-recovery-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) } -} else { - "" -} $recoveryConfig = Get-AgentRecoverySloConfig if ($Mode -in @("Status", "Recover")) { @@ -7269,12 +7523,19 @@ $result = [pscustomobject]@{ providerFreshness = $providerFreshness securityTriage = $securityTriage controlTick = $controlTick + recoveryTransportLease = $recoveryTransportLeaseReceipt events = $script:Events telegram = $script:TelegramAttempts evidenceLog = $logPath } $result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath +if ($recoveryTransportLease -and $recoveryTransportLease.acquired) { + $released = Exit-AgentRecoveryTransportLease $recoveryTransportLease + $recoveryTransportLeaseReceipt | Add-Member -NotePropertyName released -NotePropertyValue ([bool]$released) -Force + $result | Add-Member -NotePropertyName recoveryTransportLease -NotePropertyValue $recoveryTransportLeaseReceipt -Force + $result | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $jsonPath +} Write-AgentLog "agent99_complete json=$jsonPath" exit 0 diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index 3f688d80c..b5136d97c 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -78,6 +78,12 @@ def test_agent99_posts_fenced_outcome_to_production_ingestion() -> None: assert "agent99_outcome_receipt_id" in source assert "post_verifier_evidence_ref" in source assert "source_event_evidence_ref" in source + assert "function Get-AgentSafeHttpFailureReceipt" in source + assert "httpStatus = $httpStatus" in source + assert "errorCode = $detailCode" in source + assert "rawResponseStored = $false" in source + assert "secretValueLogged = $false" in source + assert "$lastFailure.retryable" in source for field in ( "projectId", "sourceFingerprint", diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 7ea4532d8..d2c99cc12 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -107,6 +107,9 @@ def test_agent99_telegram_threads_source_alert_and_persists_lifecycle() -> None: assert 'rootMessageId = $rootMessageId' in source assert 'reason = "same_lifecycle_state"' in source assert 'reason = "dedupe_window"' in source + assert "lastRunKey = $Card.runKey" in source + assert "$previousRunKey -eq [string]$incidentCard.runKey" in source + assert '$dedupeParts += "run=$($incidentCard.runKey)"' in source def test_agent99_records_deduped_telegram_as_an_explicit_attempt() -> None: @@ -410,6 +413,20 @@ def test_agent99_serializes_ssh_across_scheduled_task_processes() -> None: } +def test_agent99_recover_owns_transport_and_background_modes_defer() -> None: + source = CONTROL.read_text(encoding="utf-8") + + assert "function Enter-AgentRecoveryTransportLease" in source + assert "function Test-AgentRecoveryTransportLease" in source + assert "function Exit-AgentRecoveryTransportLease" in source + assert '"Global\\WoooAgent99RecoveryTransportV1"' in source + assert 'schemaVersion = "agent99_recovery_transport_lease_v1"' in source + assert 'schemaVersion = "agent99_background_mode_deferred_v1"' in source + assert 'reason = "recovery_already_active"' in source + assert 'purpose = "exclusive_recover_ssh_transport"' in source + assert "Exit-AgentRecoveryTransportLease $recoveryTransportLease" in source + + def test_agent99_emits_transport_wait_and_keeps_running_relay_healthy() -> None: source = CONTROL.read_text(encoding="utf-8") config = json.loads((ROOT / "agent99.config.99.example.json").read_text())