29072e4004
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m8s
CD Pipeline / build-and-deploy (push) Successful in 13m33s
CD Pipeline / post-deploy-checks (push) Has been cancelled
1081 lines
49 KiB
PowerShell
1081 lines
49 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[string]$Prefix = "http://+:8787/agent99/sre-alert/",
|
|
[string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN",
|
|
[string[]]$AllowedRemotePrefixes = @("127.", "::1", "192.168.0."),
|
|
[int]$MaxBodyBytes = 262144,
|
|
[switch]$RunOnce
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$AlertRoot = Join-Path $AgentRoot "alerts"
|
|
$IncomingDir = Join-Path $AlertRoot "incoming"
|
|
$LogDir = Join-Path $AgentRoot "logs"
|
|
$BinDir = Join-Path $AgentRoot "bin"
|
|
$QueueDir = Join-Path $AgentRoot "queue"
|
|
$ProcessedDir = Join-Path $QueueDir "processed"
|
|
$SameRunStateDir = Join-Path $AgentRoot "state\same-run-reconcile"
|
|
$SameRunRetryAuditDir = Join-Path $SameRunStateDir "retry-audit"
|
|
$AcceptedIgnoredDir = Join-Path $AlertRoot "ignored"
|
|
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $QueueDir, $ProcessedDir, $SameRunStateDir, $SameRunRetryAuditDir | Out-Null
|
|
|
|
function Convert-AgentSafeFileToken {
|
|
param([string]$Value)
|
|
$safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
|
|
if (-not $safe) { return "relay-alert" }
|
|
if ($safe.Length -gt 96) { return $safe.Substring(0, 96) }
|
|
return $safe
|
|
}
|
|
|
|
function Get-AgentField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[object]$Default = $null
|
|
)
|
|
if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) {
|
|
return $Object[$Name]
|
|
}
|
|
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
|
return $Object.PSObject.Properties[$Name].Value
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Get-AgentEnvValue {
|
|
param([string]$Name)
|
|
foreach ($target in @("Process", "User", "Machine")) {
|
|
$value = [Environment]::GetEnvironmentVariable($Name, $target)
|
|
if ($value) { return $value }
|
|
}
|
|
return ""
|
|
}
|
|
|
|
function Test-AgentPublicReceiptRef {
|
|
param([string]$Value)
|
|
if (-not $Value -or $Value.Length -gt 256) { return $false }
|
|
if ($Value -match "(?i)(authorization|bearer|password|passwd|token|secret|api[_-]?key|private[_-]?key|cookie|session)(\s|:|=)|-----BEGIN.*PRIVATE KEY-----|(^|[/\.])\.env($|[/\.])") { return $false }
|
|
if ($Value -match "\.\.|://|[\\?&#=\s]") { return $false }
|
|
return [bool]($Value -match "^[A-Za-z0-9][A-Za-z0-9._:/@+%-]{0,255}$")
|
|
}
|
|
|
|
function Get-AgentCanonicalOutcomeIdentity {
|
|
param([object]$Result, [object]$Outcome)
|
|
$outer = Get-AgentField $Result "identity" $null
|
|
$nested = Get-AgentField $Outcome "identity" $null
|
|
$required = @(
|
|
"schema_version", "project_id", "incident_id", "source_fingerprint",
|
|
"route_id", "execution_generation", "approval_id", "run_id", "trace_id",
|
|
"work_item_id", "idempotency_key", "single_flight_key", "lock_owner",
|
|
"canonical_digest"
|
|
)
|
|
if (-not $outer -or -not $nested) { return $null }
|
|
foreach ($name in $required) {
|
|
$outerValue = [string](Get-AgentField $outer $name "")
|
|
$nestedValue = [string](Get-AgentField $nested $name "")
|
|
if ($name -ne "approval_id" -and (-not $outerValue -or -not $nestedValue)) {
|
|
return $null
|
|
}
|
|
if ($outerValue -ne $nestedValue) { return $null }
|
|
}
|
|
return $outer
|
|
}
|
|
|
|
function Get-AgentOutcomeReadback {
|
|
param([string]$RunId)
|
|
|
|
$parsedRunId = [guid]::Empty
|
|
if (-not [guid]::TryParse($RunId, [ref]$parsedRunId)) {
|
|
return @{ ok = $false; found = $false; error = "invalid_run_id" }
|
|
}
|
|
|
|
foreach ($file in @(Get-ChildItem -Path $ProcessedDir -Filter "processed-*.json" -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) {
|
|
try {
|
|
$result = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
|
if ([string](Get-AgentField $result "automationRunId" "") -ne $RunId) { continue }
|
|
$outcome = Get-AgentField $result "outcome" $null
|
|
$canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $result $outcome
|
|
if (-not $canonicalIdentity) { continue }
|
|
$safeChecks = @()
|
|
foreach ($check in @((Get-AgentField $outcome "checks" @()) | Select-Object -First 32)) {
|
|
$safeChecks += @{
|
|
name = [string](Get-AgentField $check "name" "")
|
|
required = [bool](Get-AgentField $check "required" $false)
|
|
passed = [bool](Get-AgentField $check "passed" $false)
|
|
}
|
|
}
|
|
$safeEvidenceRefs = @{}
|
|
$outcomeEvidenceRefs = Get-AgentField $outcome "evidenceRefs" $null
|
|
foreach ($name in @(
|
|
"backup_status_evidence_ref",
|
|
"freshness_evidence_ref",
|
|
"offsite_verify_evidence_ref",
|
|
"escrow_evidence_ref",
|
|
"restore_drill_evidence_ref",
|
|
"source_resolution_receipt_ref"
|
|
)) {
|
|
$value = [string](Get-AgentField $outcomeEvidenceRefs $name "")
|
|
if (Test-AgentPublicReceiptRef $value) { $safeEvidenceRefs[$name] = $value }
|
|
}
|
|
$sourceEventEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "")
|
|
if (-not (Test-AgentPublicReceiptRef $sourceEventEvidence)) {
|
|
$sourceEventEvidence = ""
|
|
}
|
|
$evidenceReceiptId = [string](Get-AgentField $result "evidenceReceiptId" "")
|
|
if ([bool](Get-AgentField $result "reconcileOnly" $false)) {
|
|
$digest = [string](Get-AgentField $canonicalIdentity "canonical_digest" "")
|
|
$expectedEvidenceReceiptId = if ($digest -match "^[0-9a-f]{64}$") {
|
|
"same-run-status-$RunId-$($digest.Substring(0, 12))"
|
|
} else {
|
|
""
|
|
}
|
|
if ($evidenceReceiptId -ne $expectedEvidenceReceiptId) {
|
|
$evidenceReceiptId = ""
|
|
}
|
|
}
|
|
$topLevelStringFieldsComplete = $true
|
|
foreach ($name in @(
|
|
"automationRunId", "traceId", "workItemId", "projectId", "incidentId",
|
|
"routeId", "executionGeneration", "approvalId", "evidenceReceiptId", "mode"
|
|
)) {
|
|
if (-not $result.PSObject.Properties[$name] -or (Get-AgentField $result $name $null) -isnot [string]) {
|
|
$topLevelStringFieldsComplete = $false
|
|
break
|
|
}
|
|
}
|
|
$outcomeStringFieldsComplete = $true
|
|
foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) {
|
|
if (-not $outcome -or -not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) {
|
|
$outcomeStringFieldsComplete = $false
|
|
break
|
|
}
|
|
}
|
|
$sameRunContractFieldsComplete = [bool](
|
|
$topLevelStringFieldsComplete -and
|
|
$outcomeStringFieldsComplete -and
|
|
$result.PSObject.Properties["automationRunId"] -and
|
|
$result.PSObject.Properties["traceId"] -and
|
|
$result.PSObject.Properties["workItemId"] -and
|
|
$result.PSObject.Properties["projectId"] -and
|
|
$result.PSObject.Properties["incidentId"] -and
|
|
$result.PSObject.Properties["routeId"] -and
|
|
$result.PSObject.Properties["executionGeneration"] -and
|
|
$result.PSObject.Properties["approvalId"] -and
|
|
$result.PSObject.Properties["evidenceReceiptId"] -and
|
|
$result.PSObject.Properties["controlledApply"] -and
|
|
(Get-AgentField $result "controlledApply" $null) -is [bool] -and
|
|
$result.PSObject.Properties["reconcileOnly"] -and
|
|
(Get-AgentField $result "reconcileOnly" $null) -is [bool] -and
|
|
$result.PSObject.Properties["runtimeWritePerformed"] -and
|
|
(Get-AgentField $result "runtimeWritePerformed" $null) -is [bool] -and
|
|
$result.PSObject.Properties["mode"] -and
|
|
$outcome -and
|
|
$outcome.PSObject.Properties["schemaVersion"] -and
|
|
$outcome.PSObject.Properties["state"] -and
|
|
$outcome.PSObject.Properties["resolved"] -and
|
|
(Get-AgentField $outcome "resolved" $null) -is [bool] -and
|
|
$outcome.PSObject.Properties["transportOk"] -and
|
|
(Get-AgentField $outcome "transportOk" $null) -is [bool] -and
|
|
$outcome.PSObject.Properties["verifierName"] -and
|
|
$outcome.PSObject.Properties["verifierPassed"] -and
|
|
(Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and
|
|
$outcome.PSObject.Properties["sourceEventResolved"] -and
|
|
(Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and
|
|
$outcome.PSObject.Properties["sourceEventEvidence"] -and
|
|
$outcome.PSObject.Properties["identity"]
|
|
)
|
|
return @{
|
|
ok = $true
|
|
found = $true
|
|
schemaVersion = "agent99_outcome_readback_v1"
|
|
automationRunId = $RunId
|
|
traceId = [string](Get-AgentField $result "traceId" "")
|
|
workItemId = [string](Get-AgentField $result "workItemId" "")
|
|
projectId = [string](Get-AgentField $result "projectId" "")
|
|
idempotencyKey = [string](Get-AgentField $result "idempotencyKey" "")
|
|
executionGeneration = [string](Get-AgentField $result "executionGeneration" "1")
|
|
incidentId = [string](Get-AgentField $result "incidentId" "")
|
|
approvalId = [string](Get-AgentField $result "approvalId" "")
|
|
routeId = [string](Get-AgentField $result "routeId" "")
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
sameRunContractFieldsComplete = $sameRunContractFieldsComplete
|
|
identity = $canonicalIdentity
|
|
controlledApply = [bool](Get-AgentField $result "controlledApply" $false)
|
|
reconcileOnly = [bool](Get-AgentField $result "reconcileOnly" $false)
|
|
runtimeWritePerformed = [bool](Get-AgentField $result "runtimeWritePerformed" $false)
|
|
mode = [string](Get-AgentField $result "mode" "")
|
|
outcome = @{
|
|
schemaVersion = [string](Get-AgentField $outcome "schemaVersion" "")
|
|
state = [string](Get-AgentField $outcome "state" "unknown")
|
|
resolved = [bool](Get-AgentField $outcome "resolved" $false)
|
|
transportOk = [bool](Get-AgentField $outcome "transportOk" $false)
|
|
verifierName = [string](Get-AgentField $outcome "verifierName" "")
|
|
verifierPassed = [bool](Get-AgentField $outcome "verifierPassed" $false)
|
|
sourceEventResolved = [bool](Get-AgentField $outcome "sourceEventResolved" $false)
|
|
sourceEventEvidence = $sourceEventEvidence
|
|
identity = $canonicalIdentity
|
|
failedChecks = @((Get-AgentField $outcome "failedChecks" @()) | Select-Object -First 32)
|
|
checks = $safeChecks
|
|
evidenceRefs = $safeEvidenceRefs
|
|
verifiedAt = [string](Get-AgentField $outcome "verifiedAt" "")
|
|
}
|
|
storesRawEvidence = $false
|
|
}
|
|
} catch {
|
|
continue
|
|
}
|
|
}
|
|
return @{
|
|
ok = $true
|
|
found = $false
|
|
schemaVersion = "agent99_outcome_readback_v1"
|
|
automationRunId = $RunId
|
|
storesRawEvidence = $false
|
|
}
|
|
}
|
|
|
|
function Write-AgentRelayEvent {
|
|
param([hashtable]$Event)
|
|
$path = Join-Path $LogDir "agent99-sre-alert-relay.jsonl"
|
|
$Event["timestamp"] = (Get-Date).ToString("o")
|
|
($Event | ConvertTo-Json -Compress -Depth 8) | Add-Content -Encoding UTF8 $path
|
|
}
|
|
|
|
function Send-AgentRelayResponse {
|
|
param(
|
|
[System.Net.HttpListenerContext]$Context,
|
|
[int]$StatusCode,
|
|
[hashtable]$Body
|
|
)
|
|
$json = $Body | ConvertTo-Json -Compress -Depth 8
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
|
$Context.Response.StatusCode = $StatusCode
|
|
$Context.Response.ContentType = "application/json; charset=utf-8"
|
|
$Context.Response.ContentLength64 = $bytes.Length
|
|
$Context.Response.OutputStream.Write($bytes, 0, $bytes.Length)
|
|
$Context.Response.Close()
|
|
}
|
|
|
|
function Test-AgentRelayRemoteAllowed {
|
|
param([System.Net.HttpListenerContext]$Context)
|
|
$remote = [string]$Context.Request.RemoteEndPoint.Address
|
|
foreach ($prefix in $AllowedRemotePrefixes) {
|
|
if ($remote.StartsWith($prefix)) { return $true }
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Start-AgentSreAlertInbox {
|
|
if (-not (Test-Path $SreAlertInboxScript)) {
|
|
return [pscustomobject]@{ started = $false; reason = "sre_alert_inbox_script_missing" }
|
|
}
|
|
$ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$stdout = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.out"
|
|
$stderr = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.err"
|
|
$args = @(
|
|
"-NoProfile",
|
|
"-ExecutionPolicy", "Bypass",
|
|
"-File", $SreAlertInboxScript,
|
|
"-AgentRoot", $AgentRoot,
|
|
"-RunNow"
|
|
)
|
|
$proc = Start-Process -FilePath $ps -ArgumentList $args -PassThru -WindowStyle Hidden -RedirectStandardOutput $stdout -RedirectStandardError $stderr
|
|
return [pscustomobject]@{
|
|
started = $true
|
|
processId = $proc.Id
|
|
stdout = $stdout
|
|
stderr = $stderr
|
|
}
|
|
}
|
|
|
|
function Get-AgentSha256Text {
|
|
param([string]$Value)
|
|
$sha = [Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
$bytes = [Text.Encoding]::UTF8.GetBytes($Value)
|
|
return ([BitConverter]::ToString($sha.ComputeHash($bytes))).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
}
|
|
|
|
function Test-AgentSameRunDispatchIdentity {
|
|
param([object]$Identity)
|
|
|
|
$required = @(
|
|
"schema_version", "project_id", "incident_id", "source_fingerprint",
|
|
"route_id", "execution_generation", "approval_id", "run_id", "trace_id",
|
|
"work_item_id", "idempotency_key", "single_flight_key", "lock_owner",
|
|
"canonical_digest"
|
|
)
|
|
if (-not $Identity) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_missing" }
|
|
}
|
|
foreach ($name in $required) {
|
|
if (-not $Identity.PSObject.Properties[$name]) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_field_missing_$name" }
|
|
}
|
|
if ($name -ne "approval_id" -and -not [string](Get-AgentField $Identity $name "")) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_field_empty_$name" }
|
|
}
|
|
}
|
|
$runId = [string](Get-AgentField $Identity "run_id" "")
|
|
$parsedRunId = [guid]::Empty
|
|
if (-not [guid]::TryParse($runId, [ref]$parsedRunId)) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_run_id_invalid" }
|
|
}
|
|
if ([string](Get-AgentField $Identity "schema_version" "") -ne "agent99_controlled_dispatch_identity_v1") {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_schema_invalid" }
|
|
}
|
|
if ([string](Get-AgentField $Identity "project_id" "") -ne "awoooi") {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_project_not_eligible" }
|
|
}
|
|
if ([string](Get-AgentField $Identity "trace_id" "") -notmatch "^00-[0-9a-f]{32}-[0-9a-f]{16}-01$") {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_trace_not_eligible" }
|
|
}
|
|
if ([string](Get-AgentField $Identity "route_id" "") -ne "agent99:host_recovery:Recover") {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_route_not_eligible" }
|
|
}
|
|
if ([string](Get-AgentField $Identity "lock_owner" "") -ne $runId) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_lock_owner_mismatch" }
|
|
}
|
|
$incidentId = [string](Get-AgentField $Identity "incident_id" "")
|
|
$workItemId = [string](Get-AgentField $Identity "work_item_id" "")
|
|
$sourceFingerprint = [string](Get-AgentField $Identity "source_fingerprint" "")
|
|
$executionGeneration = [string](Get-AgentField $Identity "execution_generation" "")
|
|
$legacyIdentity = [bool](
|
|
$incidentId -eq "INC-20260711-11C751" -and
|
|
$workItemId -eq "agent99-dispatch:awoooi:INC-20260711-11C751:legacy-cold-start" -and
|
|
$sourceFingerprint -match "^legacy-cold-start:[0-9a-f]{64}$" -and
|
|
$executionGeneration -eq "1"
|
|
)
|
|
$liveIdentity = [bool](
|
|
$incidentId -match "^INC-[0-9]{8}-[0-9A-F]{6}$" -and
|
|
$workItemId -eq "agent99-dispatch:awoooi:$incidentId`:agent99:host_recovery:Recover" -and
|
|
$sourceFingerprint -match "^[0-9a-f]{32,64}$" -and
|
|
$executionGeneration -match "^[1-3]$"
|
|
)
|
|
if (-not ($legacyIdentity -or $liveIdentity)) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_cold_start_scope_not_eligible" }
|
|
}
|
|
$canonical = [ordered]@{
|
|
approval_id = [string](Get-AgentField $Identity "approval_id" "")
|
|
execution_generation = [string](Get-AgentField $Identity "execution_generation" "")
|
|
idempotency_key = [string](Get-AgentField $Identity "idempotency_key" "")
|
|
incident_id = [string](Get-AgentField $Identity "incident_id" "")
|
|
lock_owner = [string](Get-AgentField $Identity "lock_owner" "")
|
|
project_id = [string](Get-AgentField $Identity "project_id" "")
|
|
route_id = [string](Get-AgentField $Identity "route_id" "")
|
|
run_id = $runId
|
|
schema_version = [string](Get-AgentField $Identity "schema_version" "")
|
|
single_flight_key = [string](Get-AgentField $Identity "single_flight_key" "")
|
|
source_fingerprint = [string](Get-AgentField $Identity "source_fingerprint" "")
|
|
trace_id = [string](Get-AgentField $Identity "trace_id" "")
|
|
work_item_id = [string](Get-AgentField $Identity "work_item_id" "")
|
|
}
|
|
$expectedDigest = Get-AgentSha256Text ($canonical | ConvertTo-Json -Compress -Depth 4)
|
|
if ($expectedDigest -ne [string](Get-AgentField $Identity "canonical_digest" "")) {
|
|
return [pscustomobject]@{ ok = $false; reason = "identity_canonical_digest_mismatch" }
|
|
}
|
|
return [pscustomobject]@{ ok = $true; reason = "identity_valid" }
|
|
}
|
|
|
|
function Convert-AgentStoredDispatchIdentity {
|
|
param([object]$Stored)
|
|
|
|
$identity = Get-AgentField $Stored "identity" $null
|
|
if ($identity) { return $identity }
|
|
$awoooi = Get-AgentField $Stored "awoooi" $null
|
|
$acceptedAlertIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null
|
|
if ($acceptedAlertIdentity) { return $acceptedAlertIdentity }
|
|
return [pscustomobject]@{
|
|
schema_version = "agent99_controlled_dispatch_identity_v1"
|
|
project_id = [string](Get-AgentField $Stored "projectId" "")
|
|
incident_id = [string](Get-AgentField $Stored "incidentId" "")
|
|
source_fingerprint = [string](Get-AgentField $Stored "sourceFingerprint" "")
|
|
route_id = [string](Get-AgentField $Stored "dispatchRouteId" "")
|
|
execution_generation = [string](Get-AgentField $Stored "executionGeneration" "")
|
|
approval_id = [string](Get-AgentField $Stored "approvalId" "")
|
|
run_id = [string](Get-AgentField $Stored "automationRunId" "")
|
|
trace_id = [string](Get-AgentField $Stored "traceId" "")
|
|
work_item_id = [string](Get-AgentField $Stored "workItemId" "")
|
|
idempotency_key = [string](Get-AgentField $Stored "idempotencyKey" "")
|
|
single_flight_key = [string](Get-AgentField $Stored "singleFlightKey" "")
|
|
lock_owner = [string](Get-AgentField $Stored "lockOwner" "")
|
|
canonical_digest = [string](Get-AgentField $Stored "canonicalDigest" "")
|
|
}
|
|
}
|
|
|
|
function Test-AgentStoredColdStartRecoverEnvelope {
|
|
param([object]$Stored)
|
|
|
|
if (-not $Stored -or [bool](Get-AgentField $Stored "reconcileOnly" $false)) {
|
|
return $false
|
|
}
|
|
$mode = [string](Get-AgentField $Stored "mode" (Get-AgentField $Stored "suggestedMode" ""))
|
|
$source = [string](Get-AgentField $Stored "alertSource" (Get-AgentField $Stored "source" ""))
|
|
$kind = [string](Get-AgentField $Stored "alertKind" (Get-AgentField $Stored "kind" ""))
|
|
$service = [string](Get-AgentField $Stored "alertService" (Get-AgentField $Stored "service" ""))
|
|
$controlled = Get-AgentField $Stored "controlledApply" $null
|
|
return [bool](
|
|
$mode -eq "Recover" -and
|
|
$source -eq "awoooi-api-alertmanager" -and
|
|
$kind -eq "host_recovery" -and
|
|
$service -eq "cold-start-gate" -and
|
|
$controlled -is [bool] -and
|
|
$controlled
|
|
)
|
|
}
|
|
|
|
function Get-AgentExistingRecoverIdentity {
|
|
param([string]$RunId)
|
|
|
|
$files = @()
|
|
$files += @(Get-ChildItem -LiteralPath $ProcessedDir -Filter "processed-*.json" -File -ErrorAction SilentlyContinue)
|
|
$files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "running-*.json" -File -ErrorAction SilentlyContinue)
|
|
$files += @(Get-ChildItem -LiteralPath $QueueDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
|
|
Where-Object { $_.Name -notmatch "^(processed|failed|running|same-run-status)-" })
|
|
# One legacy envelope was moved to ignored by an older inbox build. Current
|
|
# runs are admitted only from an immutable cold-start Recover envelope; a
|
|
# generic ignored, queue, or reconcile payload can never authorize Status.
|
|
$acceptedIgnoredPattern = "ignored-awoooi-agent99-$RunId-*.json"
|
|
$files = @($files | Sort-Object LastWriteTime -Descending | Select-Object -First 256)
|
|
$files += @(Get-ChildItem -LiteralPath $AcceptedIgnoredDir -Filter $acceptedIgnoredPattern -File -ErrorAction SilentlyContinue)
|
|
foreach ($file in $files) {
|
|
try {
|
|
$stored = Get-Content -LiteralPath $file.FullName -Raw | ConvertFrom-Json
|
|
if (-not (Test-AgentStoredColdStartRecoverEnvelope $stored)) { continue }
|
|
$storedMode = [string](Get-AgentField $stored "mode" (Get-AgentField $stored "suggestedMode" ""))
|
|
if ($storedMode -ne "Recover") { continue }
|
|
$identity = Convert-AgentStoredDispatchIdentity $stored
|
|
if ([string](Get-AgentField $identity "run_id" "") -ne $RunId) { continue }
|
|
$check = Test-AgentSameRunDispatchIdentity $identity
|
|
if ($check.ok) { return $identity }
|
|
} catch {
|
|
continue
|
|
}
|
|
}
|
|
return $null
|
|
}
|
|
|
|
function Test-AgentSameRunIdentityMatchesExisting {
|
|
param([object]$Requested, [object]$Existing)
|
|
|
|
foreach ($name in @(
|
|
"schema_version", "project_id", "incident_id", "source_fingerprint",
|
|
"route_id", "execution_generation", "approval_id", "run_id", "trace_id",
|
|
"work_item_id", "idempotency_key", "single_flight_key", "lock_owner",
|
|
"canonical_digest"
|
|
)) {
|
|
if ([string](Get-AgentField $Requested $name "") -ne [string](Get-AgentField $Existing $name "")) {
|
|
return $false
|
|
}
|
|
}
|
|
return $true
|
|
}
|
|
|
|
function Test-AgentSameRunOutcomeEligible {
|
|
param(
|
|
[object]$Existing,
|
|
[object]$Identity,
|
|
[string]$SourceEvidence,
|
|
[string]$EvidenceReceiptId
|
|
)
|
|
|
|
if (-not $Existing -or -not [bool](Get-AgentField $Existing "found" $false)) { return $false }
|
|
$outcome = Get-AgentField $Existing "outcome" $null
|
|
$existingIdentity = Get-AgentField $Existing "identity" $null
|
|
$nestedIdentity = Get-AgentField $outcome "identity" $null
|
|
return [bool](
|
|
[string](Get-AgentField $Existing "schemaVersion" "") -eq "agent99_outcome_readback_v1" -and
|
|
[bool](Get-AgentField $Existing "sameRunContractFieldsComplete" $false) -and
|
|
[string](Get-AgentField $Existing "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and
|
|
[string](Get-AgentField $Existing "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and
|
|
[string](Get-AgentField $Existing "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and
|
|
[string](Get-AgentField $Existing "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and
|
|
[string](Get-AgentField $Existing "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and
|
|
[string](Get-AgentField $Existing "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and
|
|
[string](Get-AgentField $Existing "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and
|
|
[string](Get-AgentField $Existing "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and
|
|
[string](Get-AgentField $Existing "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
|
|
(Test-AgentSameRunIdentityMatchesExisting $existingIdentity $Identity) -and
|
|
(Test-AgentSameRunIdentityMatchesExisting $nestedIdentity $Identity) -and
|
|
[string](Get-AgentField $Existing "mode" "") -eq "Status" -and
|
|
(Get-AgentField $Existing "controlledApply" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $Existing "controlledApply" $true) -and
|
|
(Get-AgentField $Existing "reconcileOnly" $null) -is [bool] -and
|
|
[bool](Get-AgentField $Existing "reconcileOnly" $false) -and
|
|
(Get-AgentField $Existing "runtimeWritePerformed" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $Existing "runtimeWritePerformed" $true) -and
|
|
[string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and
|
|
[string](Get-AgentField $outcome "state" "") -eq "resolved" -and
|
|
(Get-AgentField $outcome "resolved" $null) -is [bool] -and
|
|
[bool](Get-AgentField $outcome "resolved" $false) -and
|
|
(Get-AgentField $outcome "transportOk" $null) -is [bool] -and
|
|
[bool](Get-AgentField $outcome "transportOk" $false) -and
|
|
[string](Get-AgentField $outcome "verifierName" "") -eq "agent99-status-same-run-no-write-post-condition-v1" -and
|
|
(Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and
|
|
[bool](Get-AgentField $outcome "verifierPassed" $false) -and
|
|
(Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and
|
|
[bool](Get-AgentField $outcome "sourceEventResolved" $false) -and
|
|
[string](Get-AgentField $outcome "sourceEventEvidence" "") -eq $SourceEvidence
|
|
)
|
|
}
|
|
|
|
function Test-AgentSameRunFailedNoWriteOutcomeRetryEligible {
|
|
param(
|
|
[object]$Result,
|
|
[object]$Identity,
|
|
[string]$SourceEvidence,
|
|
[string]$EvidenceReceiptId,
|
|
[string]$QueueId
|
|
)
|
|
|
|
if (-not $Result) { return $false }
|
|
$outcome = Get-AgentField $Result "outcome" $null
|
|
$canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $Result $outcome
|
|
if (-not $outcome -or -not $canonicalIdentity) { return $false }
|
|
foreach ($name in @(
|
|
"id", "source", "reason", "mode", "reconcileEvidenceId", "evidenceReceiptId",
|
|
"automationRunId", "traceId", "workItemId", "idempotencyKey",
|
|
"executionGeneration", "incidentId", "approvalId", "projectId", "routeId",
|
|
"sourceEventResolutionPolicy"
|
|
)) {
|
|
if (-not $Result.PSObject.Properties[$name] -or (Get-AgentField $Result $name $null) -isnot [string]) {
|
|
return $false
|
|
}
|
|
}
|
|
foreach ($name in @("schemaVersion", "state", "verifierName", "sourceEventEvidence")) {
|
|
if (-not $outcome.PSObject.Properties[$name] -or (Get-AgentField $outcome $name $null) -isnot [string]) {
|
|
return $false
|
|
}
|
|
}
|
|
if (-not [string](Get-AgentField $outcome "verifierName" "")) { return $false }
|
|
$previousSourceEvidence = [string](Get-AgentField $outcome "sourceEventEvidence" "")
|
|
if (
|
|
-not (Test-AgentPublicReceiptRef $SourceEvidence) -or
|
|
-not $SourceEvidence.StartsWith("prometheus:cold-start:") -or
|
|
-not (Test-AgentPublicReceiptRef $previousSourceEvidence) -or
|
|
-not $previousSourceEvidence.StartsWith("prometheus:cold-start:")
|
|
) { return $false }
|
|
|
|
return [bool](
|
|
[string](Get-AgentField $Result "id" "") -eq $QueueId -and
|
|
[string](Get-AgentField $Result "source" "") -eq "agent99-same-run-reconcile" -and
|
|
[string](Get-AgentField $Result "reason" "") -eq "source_event_resolved_status_reconcile" -and
|
|
[string](Get-AgentField $Result "mode" "") -eq "Status" -and
|
|
(Get-AgentField $Result "controlledApply" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $Result "controlledApply" $true) -and
|
|
(Get-AgentField $Result "reconcileOnly" $null) -is [bool] -and
|
|
[bool](Get-AgentField $Result "reconcileOnly" $false) -and
|
|
(Get-AgentField $Result "runtimeWritePerformed" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $Result "runtimeWritePerformed" $true) -and
|
|
(Get-AgentField $Result "suppressTelegram" $null) -is [bool] -and
|
|
[bool](Get-AgentField $Result "suppressTelegram" $false) -and
|
|
[string](Get-AgentField $Result "reconcileEvidenceId" "") -eq $EvidenceReceiptId -and
|
|
[string](Get-AgentField $Result "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
|
|
[string](Get-AgentField $Result "automationRunId" "") -eq [string](Get-AgentField $Identity "run_id" "") -and
|
|
[string](Get-AgentField $Result "traceId" "") -eq [string](Get-AgentField $Identity "trace_id" "") -and
|
|
[string](Get-AgentField $Result "workItemId" "") -eq [string](Get-AgentField $Identity "work_item_id" "") -and
|
|
[string](Get-AgentField $Result "idempotencyKey" "") -eq [string](Get-AgentField $Identity "idempotency_key" "") -and
|
|
[string](Get-AgentField $Result "executionGeneration" "") -eq [string](Get-AgentField $Identity "execution_generation" "") -and
|
|
[string](Get-AgentField $Result "incidentId" "") -eq [string](Get-AgentField $Identity "incident_id" "") -and
|
|
[string](Get-AgentField $Result "approvalId" "") -eq [string](Get-AgentField $Identity "approval_id" "") -and
|
|
[string](Get-AgentField $Result "projectId" "") -eq [string](Get-AgentField $Identity "project_id" "") -and
|
|
[string](Get-AgentField $Result "routeId" "") -eq [string](Get-AgentField $Identity "route_id" "") -and
|
|
(Test-AgentSameRunIdentityMatchesExisting $canonicalIdentity $Identity) -and
|
|
[string](Get-AgentField $Result "sourceEventResolutionPolicy" "") -eq "external_source_receipt_required" -and
|
|
[string](Get-AgentField $outcome "schemaVersion" "") -eq "agent99_outcome_contract_v1" -and
|
|
[string](Get-AgentField $outcome "state" "") -eq "failed" -and
|
|
(Get-AgentField $outcome "resolved" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $outcome "resolved" $true) -and
|
|
(Get-AgentField $outcome "transportOk" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $outcome "transportOk" $true) -and
|
|
(Get-AgentField $outcome "verifierPassed" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $outcome "verifierPassed" $true) -and
|
|
(Get-AgentField $outcome "sourceEventResolved" $null) -is [bool] -and
|
|
[bool](Get-AgentField $outcome "sourceEventResolved" $false)
|
|
)
|
|
}
|
|
|
|
function Move-AgentSameRunFailedNoWriteOutcomeForRetry {
|
|
param(
|
|
[string]$ProcessedPath,
|
|
[string]$LockPath,
|
|
[string]$RetryMarkerPath,
|
|
[string]$QueueId,
|
|
[object]$Identity,
|
|
[string]$SourceEvidence,
|
|
[string]$EvidenceReceiptId
|
|
)
|
|
|
|
if (Test-Path -LiteralPath $RetryMarkerPath) {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
|
|
}
|
|
if (-not (Test-Path -LiteralPath $ProcessedPath) -or -not (Test-Path -LiteralPath $LockPath)) {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_missing_fail_closed" }
|
|
}
|
|
|
|
try {
|
|
$failedResult = Get-Content -LiteralPath $ProcessedPath -Raw | ConvertFrom-Json
|
|
$lockReceipt = Get-Content -LiteralPath $LockPath -Raw | ConvertFrom-Json
|
|
$failedOutcomeSha256 = (Get-FileHash -LiteralPath $ProcessedPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$previousLockSha256 = (Get-FileHash -LiteralPath $LockPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
} catch {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_invalid_fail_closed" }
|
|
}
|
|
if ($failedOutcomeSha256 -notmatch "^[0-9a-f]{64}$" -or $previousLockSha256 -notmatch "^[0-9a-f]{64}$") {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_artifact_digest_invalid_fail_closed" }
|
|
}
|
|
if (-not (Test-AgentSameRunFailedNoWriteOutcomeRetryEligible $failedResult $Identity $SourceEvidence $EvidenceReceiptId $QueueId)) {
|
|
return [pscustomobject]@{ ok = $false; reason = "existing_same_run_status_outcome_invalid_fail_closed" }
|
|
}
|
|
|
|
$runId = [string](Get-AgentField $Identity "run_id" "")
|
|
$canonicalDigest = [string](Get-AgentField $Identity "canonical_digest" "")
|
|
$lockValid = [bool](
|
|
[string](Get-AgentField $lockReceipt "schemaVersion" "") -eq "agent99_same_run_single_flight_lock_v1" -and
|
|
[string](Get-AgentField $lockReceipt "automationRunId" "") -eq $runId -and
|
|
[string](Get-AgentField $lockReceipt "evidenceReceiptId" "") -eq $EvidenceReceiptId -and
|
|
[string](Get-AgentField $lockReceipt "canonicalDigest" "") -eq $canonicalDigest -and
|
|
(Get-AgentField $lockReceipt "storesRawEvidence" $null) -is [bool] -and
|
|
-not [bool](Get-AgentField $lockReceipt "storesRawEvidence" $true)
|
|
)
|
|
if (-not $lockValid) {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_lock_invalid_fail_closed" }
|
|
}
|
|
|
|
$outcomeAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.processed.json"
|
|
$lockAuditPath = Join-Path $SameRunRetryAuditDir "$QueueId.attempt-1.lock.json"
|
|
if ((Test-Path -LiteralPath $outcomeAuditPath) -or (Test-Path -LiteralPath $lockAuditPath)) {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_audit_collision_fail_closed" }
|
|
}
|
|
|
|
try {
|
|
$markerStream = [IO.File]::Open($RetryMarkerPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
|
try {
|
|
$markerReceipt = @{
|
|
schemaVersion = "agent99_same_run_status_retry_v1"
|
|
status = "retry_reserved"
|
|
retryOrdinal = 1
|
|
automationRunId = $runId
|
|
evidenceReceiptId = $EvidenceReceiptId
|
|
canonicalDigest = $canonicalDigest
|
|
failedOutcomeAuditFile = (Split-Path -Leaf $outcomeAuditPath)
|
|
failedOutcomeSha256 = $failedOutcomeSha256
|
|
previousLockAuditFile = (Split-Path -Leaf $lockAuditPath)
|
|
previousLockSha256 = $previousLockSha256
|
|
preservesFailedOutcome = $true
|
|
controlledApply = $false
|
|
reconcileOnly = $true
|
|
runtimeWriteAuthorized = $false
|
|
storesRawEvidence = $false
|
|
createdAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Compress
|
|
$markerBytes = [Text.Encoding]::UTF8.GetBytes($markerReceipt)
|
|
$markerStream.Write($markerBytes, 0, $markerBytes.Length)
|
|
$markerStream.Flush($true)
|
|
} finally {
|
|
$markerStream.Dispose()
|
|
}
|
|
} catch [System.IO.IOException] {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_already_consumed_fail_closed" }
|
|
}
|
|
|
|
try {
|
|
Move-Item -LiteralPath $ProcessedPath -Destination $outcomeAuditPath
|
|
Move-Item -LiteralPath $LockPath -Destination $lockAuditPath
|
|
} catch {
|
|
return [pscustomobject]@{ ok = $false; reason = "same_run_status_retry_archive_failed_fail_closed" }
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
reason = "same_run_status_failed_no_write_outcome_archived_for_single_retry"
|
|
retryOrdinal = 1
|
|
}
|
|
}
|
|
|
|
function Start-AgentSameRunStatusReconcile {
|
|
param([object]$Request)
|
|
|
|
$identity = Get-AgentField $Request "identity" $null
|
|
$identityCheck = Test-AgentSameRunDispatchIdentity $identity
|
|
if (-not $identityCheck.ok) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = $identityCheck.reason; httpStatus = 400 }
|
|
}
|
|
$controlledApply = Get-AgentField $Request "controlledApply" $null
|
|
$reconcileOnly = Get-AgentField $Request "reconcileOnly" $null
|
|
$sourceEventResolved = Get-AgentField $Request "sourceEventResolved" $null
|
|
$suppressTelegram = Get-AgentField $Request "suppressTelegram" $null
|
|
if (
|
|
[string](Get-AgentField $Request "action" "") -ne "same_run_status_reconcile" -or
|
|
[string](Get-AgentField $Request "mode" "") -ne "Status" -or
|
|
$controlledApply -isnot [bool] -or $controlledApply -or
|
|
$reconcileOnly -isnot [bool] -or -not $reconcileOnly -or
|
|
$sourceEventResolved -isnot [bool] -or -not $sourceEventResolved -or
|
|
[string](Get-AgentField $Request "sourceEventResolutionPolicy" "") -ne "external_source_receipt_required" -or
|
|
$suppressTelegram -isnot [bool] -or -not $suppressTelegram
|
|
) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_reconcile_contract_invalid"; httpStatus = 400 }
|
|
}
|
|
$sourceEvidence = [string](Get-AgentField $Request "sourceEventEvidence" "")
|
|
if (-not (Test-AgentPublicReceiptRef $sourceEvidence) -or -not $sourceEvidence.StartsWith("prometheus:cold-start:")) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "source_event_evidence_invalid"; httpStatus = 400 }
|
|
}
|
|
|
|
$runId = [string](Get-AgentField $identity "run_id" "")
|
|
$canonicalDigest = [string](Get-AgentField $identity "canonical_digest" "")
|
|
$evidenceReceiptId = [string](Get-AgentField $Request "evidenceReceiptId" "")
|
|
$expectedEvidenceReceiptId = if ($canonicalDigest -match "^[0-9a-f]{64}$") {
|
|
"same-run-status-$runId-$($canonicalDigest.Substring(0, 12))"
|
|
} else {
|
|
""
|
|
}
|
|
if (-not $expectedEvidenceReceiptId -or $evidenceReceiptId -ne $expectedEvidenceReceiptId) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_evidence_receipt_id_invalid"; httpStatus = 400 }
|
|
}
|
|
|
|
$existingRecoverIdentity = Get-AgentExistingRecoverIdentity $runId
|
|
if (-not $existingRecoverIdentity) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_not_found"; httpStatus = 409 }
|
|
}
|
|
if (-not (Test-AgentSameRunIdentityMatchesExisting $identity $existingRecoverIdentity)) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "existing_dispatch_identity_mismatch"; httpStatus = 409 }
|
|
}
|
|
|
|
$safeRunId = Convert-AgentSafeFileToken $runId
|
|
$queueId = "same-run-status-$safeRunId"
|
|
$queuePath = Join-Path $QueueDir "$queueId.json"
|
|
$runningPath = Join-Path $QueueDir "running-$queueId.json"
|
|
$processedPath = Join-Path $ProcessedDir "processed-$queueId.json"
|
|
$lockPath = Join-Path $SameRunStateDir "$queueId.lock.json"
|
|
$retryMarkerPath = Join-Path $SameRunStateDir "$queueId.retry-1.json"
|
|
$singleRetryPrepared = $false
|
|
|
|
$existing = Get-AgentOutcomeReadback $runId
|
|
if (Test-AgentSameRunOutcomeEligible $existing $identity $sourceEvidence $evidenceReceiptId) {
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status = "same_run_status_outcome_already_available"
|
|
httpStatus = 200
|
|
automationRunId = $runId
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
queued = $false
|
|
}
|
|
}
|
|
if ($existing.found -and [bool](Get-AgentField $existing "reconcileOnly" $false)) {
|
|
$retryPreparation = Move-AgentSameRunFailedNoWriteOutcomeForRetry `
|
|
$processedPath $lockPath $retryMarkerPath $queueId $identity $sourceEvidence $evidenceReceiptId
|
|
if (-not $retryPreparation.ok) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = [string]$retryPreparation.reason; httpStatus = 409 }
|
|
}
|
|
$singleRetryPrepared = $true
|
|
}
|
|
$queuePending = Test-Path -LiteralPath $queuePath -PathType Leaf
|
|
$runningPending = Test-Path -LiteralPath $runningPath -PathType Leaf
|
|
$lockPending = Test-Path -LiteralPath $lockPath -PathType Leaf
|
|
if (-not $singleRetryPrepared -and (Test-Path -LiteralPath $retryMarkerPath -PathType Leaf)) {
|
|
if (-not (($queuePending -or $runningPending) -and $lockPending)) {
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "same_run_status_retry_already_consumed_fail_closed"; httpStatus = 409 }
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status = "same_run_status_reconcile_pending"
|
|
httpStatus = 202
|
|
automationRunId = $runId
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
queued = $false
|
|
}
|
|
}
|
|
|
|
if ($queuePending -or $runningPending -or $lockPending) {
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status = "same_run_status_reconcile_pending"
|
|
httpStatus = 202
|
|
automationRunId = $runId
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
queued = $false
|
|
}
|
|
}
|
|
|
|
try {
|
|
$lockStream = [IO.File]::Open($lockPath, [IO.FileMode]::CreateNew, [IO.FileAccess]::Write, [IO.FileShare]::None)
|
|
try {
|
|
$lockReceipt = @{
|
|
schemaVersion = "agent99_same_run_single_flight_lock_v1"
|
|
automationRunId = $runId
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
canonicalDigest = $canonicalDigest
|
|
createdAt = (Get-Date -Format o)
|
|
storesRawEvidence = $false
|
|
} | ConvertTo-Json -Compress
|
|
$lockBytes = [Text.Encoding]::UTF8.GetBytes($lockReceipt)
|
|
$lockStream.Write($lockBytes, 0, $lockBytes.Length)
|
|
$lockStream.Flush($true)
|
|
} finally {
|
|
$lockStream.Dispose()
|
|
}
|
|
} catch [System.IO.IOException] {
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status = "same_run_status_reconcile_pending"
|
|
httpStatus = 202
|
|
automationRunId = $runId
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
queued = $false
|
|
}
|
|
}
|
|
|
|
$command = [ordered]@{
|
|
id = $queueId
|
|
mode = "Status"
|
|
controlledApply = $false
|
|
reconcileOnly = $true
|
|
reconcileEvidenceId = $evidenceReceiptId
|
|
requestedBy = "awoooi-agent99-controlled-dispatch-reconciler"
|
|
source = "agent99-same-run-reconcile"
|
|
reason = "source_event_resolved_status_reconcile"
|
|
instruction = "Read-only Status verifier for the existing run; do not recover, advance generation, page Telegram, or write runtime state."
|
|
correlationKey = ([string](Get-AgentField $identity "idempotency_key" "")) + ":same-run-status"
|
|
alertId = "same-run-reconcile:$runId"
|
|
alertSource = "awoooi-production-source-verifier"
|
|
alertKind = "host_recovery"
|
|
alertSeverity = "info"
|
|
alertService = "cold-start-gate"
|
|
alertHost = ""
|
|
suppressTelegram = $true
|
|
sourceEventResolutionRequired = $true
|
|
sourceEventResolutionPolicy = "external_source_receipt_required"
|
|
sourceEventResolved = $true
|
|
sourceEventEvidence = $sourceEvidence
|
|
automationRunId = $runId
|
|
traceId = [string](Get-AgentField $identity "trace_id" "")
|
|
workItemId = [string](Get-AgentField $identity "work_item_id" "")
|
|
idempotencyKey = [string](Get-AgentField $identity "idempotency_key" "")
|
|
executionGeneration = [string](Get-AgentField $identity "execution_generation" "")
|
|
incidentId = [string](Get-AgentField $identity "incident_id" "")
|
|
approvalId = [string](Get-AgentField $identity "approval_id" "")
|
|
projectId = [string](Get-AgentField $identity "project_id" "")
|
|
sourceFingerprint = [string](Get-AgentField $identity "source_fingerprint" "")
|
|
dispatchRouteId = [string](Get-AgentField $identity "route_id" "")
|
|
singleFlightKey = [string](Get-AgentField $identity "single_flight_key" "")
|
|
lockOwner = [string](Get-AgentField $identity "lock_owner" "")
|
|
canonicalDigest = [string](Get-AgentField $identity "canonical_digest" "")
|
|
createdAt = (Get-Date -Format o)
|
|
}
|
|
$tmpPath = Join-Path $QueueDir ".$queueId-$([guid]::NewGuid().ToString("N")).tmp"
|
|
$command | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $tmpPath
|
|
Move-Item -Force $tmpPath $queuePath
|
|
|
|
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
|
|
if (-not (Test-Path $launcher)) {
|
|
Remove-Item -Force $queuePath -ErrorAction SilentlyContinue
|
|
Remove-Item -Force $lockPath -ErrorAction SilentlyContinue
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "agent99_launcher_missing"; httpStatus = 503 }
|
|
}
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$stdout = Join-Path $LogDir "agent99-same-run-status-$stamp.out"
|
|
$stderr = Join-Path $LogDir "agent99-same-run-status-$stamp.err"
|
|
try {
|
|
$process = Start-Process -FilePath "powershell.exe" -ArgumentList @(
|
|
"-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher,
|
|
"-Mode", "ControlTick",
|
|
"-ReconcileQueueOnly", "-ReconcileQueueId", $queueId,
|
|
"-SuppressAlerts", "-SuppressReason", "same_run_reconcile_no_telegram"
|
|
) -WindowStyle Hidden -RedirectStandardOutput $stdout -RedirectStandardError $stderr -PassThru
|
|
} catch {
|
|
Remove-Item -Force $queuePath -ErrorAction SilentlyContinue
|
|
Remove-Item -Force $lockPath -ErrorAction SilentlyContinue
|
|
return [pscustomobject]@{ ok = $false; status = "rejected"; reason = "control_tick_start_failed"; httpStatus = 503 }
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = $true
|
|
status = "same_run_status_reconcile_queued"
|
|
httpStatus = 202
|
|
automationRunId = $runId
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
evidenceReceiptId = $evidenceReceiptId
|
|
queued = $true
|
|
processId = $process.Id
|
|
}
|
|
}
|
|
|
|
if (-not $Prefix.EndsWith("/")) {
|
|
$Prefix = "$Prefix/"
|
|
}
|
|
|
|
$listener = [System.Net.HttpListener]::new()
|
|
$listener.Prefixes.Add($Prefix)
|
|
|
|
$startupStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$startupEvidence = Join-Path $EvidenceDir "agent99-SreAlertRelay-start-$startupStamp.json"
|
|
@{
|
|
ok = $true
|
|
prefix = $Prefix
|
|
tokenEnv = $TokenEnv
|
|
tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv)
|
|
allowedRemotePrefixes = $AllowedRemotePrefixes
|
|
maxBodyBytes = $MaxBodyBytes
|
|
runOnce = [bool]$RunOnce
|
|
startedAt = (Get-Date).ToString("o")
|
|
} | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $startupEvidence
|
|
|
|
try {
|
|
$listener.Start()
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_started"
|
|
ok = $true
|
|
prefix = $Prefix
|
|
tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv)
|
|
evidence = $startupEvidence
|
|
}
|
|
|
|
do {
|
|
$context = $listener.GetContext()
|
|
$remote = [string]$context.Request.RemoteEndPoint.Address
|
|
try {
|
|
if (-not (Test-AgentRelayRemoteAllowed $context)) {
|
|
Send-AgentRelayResponse $context 403 @{ ok = $false; error = "remote_not_allowed" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "remote_not_allowed" }
|
|
continue
|
|
}
|
|
|
|
$expectedToken = Get-AgentEnvValue $TokenEnv
|
|
if (-not $expectedToken) {
|
|
Send-AgentRelayResponse $context 503 @{ ok = $false; error = "relay_auth_not_configured" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "relay_auth_not_configured" }
|
|
continue
|
|
}
|
|
$actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"]
|
|
if ($actualToken -ne $expectedToken) {
|
|
Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" }
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.HttpMethod -eq "GET") {
|
|
$runId = [string]$context.Request.QueryString["run_id"]
|
|
$readback = Get-AgentOutcomeReadback $runId
|
|
$statusCode = if ($readback.ok) { 200 } else { 400 }
|
|
Send-AgentRelayResponse $context $statusCode $readback
|
|
Write-AgentRelayEvent @{
|
|
event = "outcome_readback"
|
|
ok = [bool]$readback.ok
|
|
found = [bool]$readback.found
|
|
remote = $remote
|
|
runId = if ($readback.ok) { $runId } else { "invalid" }
|
|
storesRawEvidence = $false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.HttpMethod -ne "POST") {
|
|
Send-AgentRelayResponse $context 405 @{ ok = $false; error = "method_not_allowed" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "method_not_allowed" }
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.ContentLength64 -gt $MaxBodyBytes) {
|
|
Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large" }
|
|
continue
|
|
}
|
|
|
|
$reader = [System.IO.StreamReader]::new($context.Request.InputStream, $context.Request.ContentEncoding)
|
|
$body = $reader.ReadToEnd()
|
|
if ($body.Length -gt $MaxBodyBytes) {
|
|
Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large_readback" }
|
|
continue
|
|
}
|
|
|
|
$requestPayload = $body | ConvertFrom-Json
|
|
if ([string](Get-AgentField $requestPayload "schemaVersion" "") -eq "agent99_same_run_status_reconcile_v1") {
|
|
$reconcile = Start-AgentSameRunStatusReconcile $requestPayload
|
|
$reconcileBody = @{
|
|
ok = [bool]$reconcile.ok
|
|
status = [string]$reconcile.status
|
|
reason = [string](Get-AgentField $reconcile "reason" "")
|
|
automationRunId = [string](Get-AgentField $reconcile "automationRunId" "")
|
|
reconcileOnly = [bool](Get-AgentField $reconcile "reconcileOnly" $true)
|
|
controlledApply = $false
|
|
evidenceReceiptId = [string](Get-AgentField $reconcile "evidenceReceiptId" "")
|
|
queued = [bool](Get-AgentField $reconcile "queued" $false)
|
|
}
|
|
Send-AgentRelayResponse $context ([int]$reconcile.httpStatus) $reconcileBody
|
|
Write-AgentRelayEvent @{
|
|
event = "same_run_status_reconcile"
|
|
ok = [bool]$reconcile.ok
|
|
status = [string]$reconcile.status
|
|
reason = [string](Get-AgentField $reconcile "reason" "")
|
|
httpStatus = [int]$reconcile.httpStatus
|
|
remote = $remote
|
|
runId = if ($reconcile.ok) { [string]$reconcile.automationRunId } else { "rejected" }
|
|
reconcileOnly = $true
|
|
controlledApply = $false
|
|
storesRawEvidence = $false
|
|
}
|
|
continue
|
|
}
|
|
|
|
$alert = $requestPayload
|
|
$alertId = [string](Get-AgentField $alert "id" ("relay-" + [guid]::NewGuid().ToString("N")))
|
|
$safeId = Convert-AgentSafeFileToken $alertId
|
|
$fileStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$tmpPath = Join-Path $IncomingDir ".$safeId-$fileStamp-$([guid]::NewGuid().ToString("N")).tmp"
|
|
$finalPath = Join-Path $IncomingDir "$safeId-$fileStamp.json"
|
|
$alert | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $tmpPath
|
|
Move-Item -Force $tmpPath $finalPath
|
|
|
|
$trigger = Start-AgentSreAlertInbox
|
|
Send-AgentRelayResponse $context 202 @{
|
|
ok = $true
|
|
alertId = $alertId
|
|
incomingFile = $finalPath
|
|
inboxTriggered = [bool]$trigger.started
|
|
inboxProcessId = $trigger.processId
|
|
}
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_accepted"
|
|
ok = $true
|
|
remote = $remote
|
|
alertId = $alertId
|
|
incomingFile = $finalPath
|
|
inboxTriggered = [bool]$trigger.started
|
|
inboxProcessId = $trigger.processId
|
|
}
|
|
} catch {
|
|
try {
|
|
Send-AgentRelayResponse $context 500 @{ ok = $false; error = "relay_exception"; errorType = $_.Exception.GetType().Name }
|
|
} catch {
|
|
}
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_exception"
|
|
ok = $false
|
|
remote = $remote
|
|
errorType = $_.Exception.GetType().Name
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
} while (-not $RunOnce)
|
|
} finally {
|
|
if ($listener.IsListening) { $listener.Stop() }
|
|
$listener.Close()
|
|
}
|