6cf8429d17
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m32s
CD Pipeline / build-and-deploy (push) Successful in 8m2s
CD Pipeline / post-deploy-checks (push) Successful in 2m22s
Restore D037 durable incident readback without weakening database failure semantics. Fence Agent99 dispatch and terminal learning to canonical identities, durable leases, verifier receipts, and reconciler-owned checkpoints. Drain backup and restore legacy receipts in bounded cohorts with strict transport, claim, projection, and no-false-green controls. Keep SSH service refusal visible while separating it from broker network-policy reachability.
351 lines
13 KiB
PowerShell
351 lines
13 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[string]$Prefix = "http://+:8787/agent99/sre-alert/",
|
|
[string]$TokenEnv = "AGENT99_SRE_RELAY_TOKEN",
|
|
[string[]]$AllowedRemotePrefixes = @("127.", "::1", "192.168.0."),
|
|
[int]$MaxBodyBytes = 262144,
|
|
[switch]$RunOnce
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$AlertRoot = Join-Path $AgentRoot "alerts"
|
|
$IncomingDir = Join-Path $AlertRoot "incoming"
|
|
$LogDir = Join-Path $AgentRoot "logs"
|
|
$BinDir = Join-Path $AgentRoot "bin"
|
|
$ProcessedDir = Join-Path (Join-Path $AgentRoot "queue") "processed"
|
|
$SreAlertInboxScript = Join-Path $BinDir "agent99-sre-alert-inbox.ps1"
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $LogDir, $ProcessedDir | Out-Null
|
|
|
|
function Convert-AgentSafeFileToken {
|
|
param([string]$Value)
|
|
$safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
|
|
if (-not $safe) { return "relay-alert" }
|
|
if ($safe.Length -gt 96) { return $safe.Substring(0, 96) }
|
|
return $safe
|
|
}
|
|
|
|
function Get-AgentField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[object]$Default = $null
|
|
)
|
|
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
|
return $Object.PSObject.Properties[$Name].Value
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Get-AgentEnvValue {
|
|
param([string]$Name)
|
|
foreach ($target in @("Process", "User", "Machine")) {
|
|
$value = [Environment]::GetEnvironmentVariable($Name, $target)
|
|
if ($value) { return $value }
|
|
}
|
|
return ""
|
|
}
|
|
|
|
function Test-AgentPublicReceiptRef {
|
|
param([string]$Value)
|
|
if (-not $Value -or $Value.Length -gt 256) { return $false }
|
|
if ($Value -match "(?i)(authorization|bearer|password|passwd|token|secret|api[_-]?key|private[_-]?key|cookie|session)(\s|:|=)|-----BEGIN.*PRIVATE KEY-----|(^|[/\.])\.env($|[/\.])") { return $false }
|
|
if ($Value -match "\.\.|://|[\\?&#=\s]") { return $false }
|
|
return [bool]($Value -match "^[A-Za-z0-9][A-Za-z0-9._:/@+%-]{0,255}$")
|
|
}
|
|
|
|
function Get-AgentCanonicalOutcomeIdentity {
|
|
param([object]$Result, [object]$Outcome)
|
|
$outer = Get-AgentField $Result "identity" $null
|
|
$nested = Get-AgentField $Outcome "identity" $null
|
|
$required = @(
|
|
"schema_version", "project_id", "incident_id", "source_fingerprint",
|
|
"route_id", "execution_generation", "approval_id", "run_id", "trace_id",
|
|
"work_item_id", "idempotency_key", "single_flight_key", "lock_owner",
|
|
"canonical_digest"
|
|
)
|
|
if (-not $outer -or -not $nested) { return $null }
|
|
foreach ($name in $required) {
|
|
$outerValue = [string](Get-AgentField $outer $name "")
|
|
$nestedValue = [string](Get-AgentField $nested $name "")
|
|
if ($name -ne "approval_id" -and (-not $outerValue -or -not $nestedValue)) {
|
|
return $null
|
|
}
|
|
if ($outerValue -ne $nestedValue) { return $null }
|
|
}
|
|
return $outer
|
|
}
|
|
|
|
function Get-AgentOutcomeReadback {
|
|
param([string]$RunId)
|
|
|
|
$parsedRunId = [guid]::Empty
|
|
if (-not [guid]::TryParse($RunId, [ref]$parsedRunId)) {
|
|
return @{ ok = $false; found = $false; error = "invalid_run_id" }
|
|
}
|
|
|
|
foreach ($file in @(Get-ChildItem -Path $ProcessedDir -Filter "processed-*.json" -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 256)) {
|
|
try {
|
|
$result = Get-Content $file.FullName -Raw | ConvertFrom-Json
|
|
if ([string](Get-AgentField $result "automationRunId" "") -ne $RunId) { continue }
|
|
$outcome = Get-AgentField $result "outcome" $null
|
|
$canonicalIdentity = Get-AgentCanonicalOutcomeIdentity $result $outcome
|
|
if (-not $canonicalIdentity) { continue }
|
|
$safeChecks = @()
|
|
foreach ($check in @((Get-AgentField $outcome "checks" @()) | Select-Object -First 32)) {
|
|
$safeChecks += @{
|
|
name = [string](Get-AgentField $check "name" "")
|
|
required = [bool](Get-AgentField $check "required" $false)
|
|
passed = [bool](Get-AgentField $check "passed" $false)
|
|
}
|
|
}
|
|
$safeEvidenceRefs = @{}
|
|
$outcomeEvidenceRefs = Get-AgentField $outcome "evidenceRefs" $null
|
|
foreach ($name in @(
|
|
"backup_status_evidence_ref",
|
|
"freshness_evidence_ref",
|
|
"offsite_verify_evidence_ref",
|
|
"escrow_evidence_ref",
|
|
"restore_drill_evidence_ref",
|
|
"source_resolution_receipt_ref"
|
|
)) {
|
|
$value = [string](Get-AgentField $outcomeEvidenceRefs $name "")
|
|
if (Test-AgentPublicReceiptRef $value) { $safeEvidenceRefs[$name] = $value }
|
|
}
|
|
return @{
|
|
ok = $true
|
|
found = $true
|
|
schemaVersion = "agent99_outcome_readback_v1"
|
|
automationRunId = $RunId
|
|
traceId = [string](Get-AgentField $result "traceId" "")
|
|
workItemId = [string](Get-AgentField $result "workItemId" "")
|
|
idempotencyKey = [string](Get-AgentField $result "idempotencyKey" "")
|
|
executionGeneration = [string](Get-AgentField $result "executionGeneration" "1")
|
|
incidentId = [string](Get-AgentField $result "incidentId" "")
|
|
approvalId = [string](Get-AgentField $result "approvalId" "")
|
|
identity = $canonicalIdentity
|
|
controlledApply = [bool](Get-AgentField $result "controlledApply" $false)
|
|
mode = [string](Get-AgentField $result "mode" "")
|
|
outcome = @{
|
|
schemaVersion = [string](Get-AgentField $outcome "schemaVersion" "")
|
|
state = [string](Get-AgentField $outcome "state" "unknown")
|
|
resolved = [bool](Get-AgentField $outcome "resolved" $false)
|
|
transportOk = [bool](Get-AgentField $outcome "transportOk" $false)
|
|
verifierName = [string](Get-AgentField $outcome "verifierName" "")
|
|
verifierPassed = [bool](Get-AgentField $outcome "verifierPassed" $false)
|
|
sourceEventResolved = [bool](Get-AgentField $outcome "sourceEventResolved" $false)
|
|
identity = $canonicalIdentity
|
|
failedChecks = @((Get-AgentField $outcome "failedChecks" @()) | Select-Object -First 32)
|
|
checks = $safeChecks
|
|
evidenceRefs = $safeEvidenceRefs
|
|
verifiedAt = [string](Get-AgentField $outcome "verifiedAt" "")
|
|
}
|
|
storesRawEvidence = $false
|
|
}
|
|
} catch {
|
|
continue
|
|
}
|
|
}
|
|
return @{
|
|
ok = $true
|
|
found = $false
|
|
schemaVersion = "agent99_outcome_readback_v1"
|
|
automationRunId = $RunId
|
|
storesRawEvidence = $false
|
|
}
|
|
}
|
|
|
|
function Write-AgentRelayEvent {
|
|
param([hashtable]$Event)
|
|
$path = Join-Path $LogDir "agent99-sre-alert-relay.jsonl"
|
|
$Event["timestamp"] = (Get-Date).ToString("o")
|
|
($Event | ConvertTo-Json -Compress -Depth 8) | Add-Content -Encoding UTF8 $path
|
|
}
|
|
|
|
function Send-AgentRelayResponse {
|
|
param(
|
|
[System.Net.HttpListenerContext]$Context,
|
|
[int]$StatusCode,
|
|
[hashtable]$Body
|
|
)
|
|
$json = $Body | ConvertTo-Json -Compress -Depth 8
|
|
$bytes = [System.Text.Encoding]::UTF8.GetBytes($json)
|
|
$Context.Response.StatusCode = $StatusCode
|
|
$Context.Response.ContentType = "application/json; charset=utf-8"
|
|
$Context.Response.ContentLength64 = $bytes.Length
|
|
$Context.Response.OutputStream.Write($bytes, 0, $bytes.Length)
|
|
$Context.Response.Close()
|
|
}
|
|
|
|
function Test-AgentRelayRemoteAllowed {
|
|
param([System.Net.HttpListenerContext]$Context)
|
|
$remote = [string]$Context.Request.RemoteEndPoint.Address
|
|
foreach ($prefix in $AllowedRemotePrefixes) {
|
|
if ($remote.StartsWith($prefix)) { return $true }
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Start-AgentSreAlertInbox {
|
|
if (-not (Test-Path $SreAlertInboxScript)) {
|
|
return [pscustomobject]@{ started = $false; reason = "sre_alert_inbox_script_missing" }
|
|
}
|
|
$ps = Join-Path $env:SystemRoot "System32\WindowsPowerShell\v1.0\powershell.exe"
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$stdout = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.out"
|
|
$stderr = Join-Path $LogDir "agent99-sre-alert-relay-trigger-$stamp.err"
|
|
$args = @(
|
|
"-NoProfile",
|
|
"-ExecutionPolicy", "Bypass",
|
|
"-File", $SreAlertInboxScript,
|
|
"-AgentRoot", $AgentRoot,
|
|
"-RunNow"
|
|
)
|
|
$proc = Start-Process -FilePath $ps -ArgumentList $args -PassThru -WindowStyle Hidden -RedirectStandardOutput $stdout -RedirectStandardError $stderr
|
|
return [pscustomobject]@{
|
|
started = $true
|
|
processId = $proc.Id
|
|
stdout = $stdout
|
|
stderr = $stderr
|
|
}
|
|
}
|
|
|
|
if (-not $Prefix.EndsWith("/")) {
|
|
$Prefix = "$Prefix/"
|
|
}
|
|
|
|
$listener = [System.Net.HttpListener]::new()
|
|
$listener.Prefixes.Add($Prefix)
|
|
|
|
$startupStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$startupEvidence = Join-Path $EvidenceDir "agent99-SreAlertRelay-start-$startupStamp.json"
|
|
@{
|
|
ok = $true
|
|
prefix = $Prefix
|
|
tokenEnv = $TokenEnv
|
|
tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv)
|
|
allowedRemotePrefixes = $AllowedRemotePrefixes
|
|
maxBodyBytes = $MaxBodyBytes
|
|
runOnce = [bool]$RunOnce
|
|
startedAt = (Get-Date).ToString("o")
|
|
} | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $startupEvidence
|
|
|
|
try {
|
|
$listener.Start()
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_started"
|
|
ok = $true
|
|
prefix = $Prefix
|
|
tokenConfigured = [bool](Get-AgentEnvValue $TokenEnv)
|
|
evidence = $startupEvidence
|
|
}
|
|
|
|
do {
|
|
$context = $listener.GetContext()
|
|
$remote = [string]$context.Request.RemoteEndPoint.Address
|
|
try {
|
|
if (-not (Test-AgentRelayRemoteAllowed $context)) {
|
|
Send-AgentRelayResponse $context 403 @{ ok = $false; error = "remote_not_allowed" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "remote_not_allowed" }
|
|
continue
|
|
}
|
|
|
|
$expectedToken = Get-AgentEnvValue $TokenEnv
|
|
if ($expectedToken) {
|
|
$actualToken = [string]$context.Request.Headers["X-Agent99-Relay-Token"]
|
|
if ($actualToken -ne $expectedToken) {
|
|
Send-AgentRelayResponse $context 401 @{ ok = $false; error = "invalid_relay_token" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "invalid_relay_token" }
|
|
continue
|
|
}
|
|
}
|
|
|
|
if ($context.Request.HttpMethod -eq "GET" -and -not $expectedToken) {
|
|
Send-AgentRelayResponse $context 503 @{ ok = $false; found = $false; error = "outcome_readback_auth_not_configured" }
|
|
Write-AgentRelayEvent @{ event = "outcome_readback_rejected"; ok = $false; remote = $remote; reason = "auth_not_configured" }
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.HttpMethod -eq "GET") {
|
|
$runId = [string]$context.Request.QueryString["run_id"]
|
|
$readback = Get-AgentOutcomeReadback $runId
|
|
$statusCode = if ($readback.ok) { 200 } else { 400 }
|
|
Send-AgentRelayResponse $context $statusCode $readback
|
|
Write-AgentRelayEvent @{
|
|
event = "outcome_readback"
|
|
ok = [bool]$readback.ok
|
|
found = [bool]$readback.found
|
|
remote = $remote
|
|
runId = if ($readback.ok) { $runId } else { "invalid" }
|
|
storesRawEvidence = $false
|
|
}
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.HttpMethod -ne "POST") {
|
|
Send-AgentRelayResponse $context 405 @{ ok = $false; error = "method_not_allowed" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "method_not_allowed" }
|
|
continue
|
|
}
|
|
|
|
if ($context.Request.ContentLength64 -gt $MaxBodyBytes) {
|
|
Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large" }
|
|
continue
|
|
}
|
|
|
|
$reader = [System.IO.StreamReader]::new($context.Request.InputStream, $context.Request.ContentEncoding)
|
|
$body = $reader.ReadToEnd()
|
|
if ($body.Length -gt $MaxBodyBytes) {
|
|
Send-AgentRelayResponse $context 413 @{ ok = $false; error = "payload_too_large" }
|
|
Write-AgentRelayEvent @{ event = "relay_rejected"; ok = $false; remote = $remote; reason = "payload_too_large_readback" }
|
|
continue
|
|
}
|
|
|
|
$alert = $body | ConvertFrom-Json
|
|
$alertId = [string](Get-AgentField $alert "id" ("relay-" + [guid]::NewGuid().ToString("N")))
|
|
$safeId = Convert-AgentSafeFileToken $alertId
|
|
$fileStamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$tmpPath = Join-Path $IncomingDir ".$safeId-$fileStamp-$([guid]::NewGuid().ToString("N")).tmp"
|
|
$finalPath = Join-Path $IncomingDir "$safeId-$fileStamp.json"
|
|
$alert | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $tmpPath
|
|
Move-Item -Force $tmpPath $finalPath
|
|
|
|
$trigger = Start-AgentSreAlertInbox
|
|
Send-AgentRelayResponse $context 202 @{
|
|
ok = $true
|
|
alertId = $alertId
|
|
incomingFile = $finalPath
|
|
inboxTriggered = [bool]$trigger.started
|
|
inboxProcessId = $trigger.processId
|
|
}
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_accepted"
|
|
ok = $true
|
|
remote = $remote
|
|
alertId = $alertId
|
|
incomingFile = $finalPath
|
|
inboxTriggered = [bool]$trigger.started
|
|
inboxProcessId = $trigger.processId
|
|
}
|
|
} catch {
|
|
try {
|
|
Send-AgentRelayResponse $context 500 @{ ok = $false; error = "relay_exception"; errorType = $_.Exception.GetType().Name }
|
|
} catch {
|
|
}
|
|
Write-AgentRelayEvent @{
|
|
event = "relay_exception"
|
|
ok = $false
|
|
remote = $remote
|
|
errorType = $_.Exception.GetType().Name
|
|
error = $_.Exception.Message
|
|
}
|
|
}
|
|
} while (-not $RunOnce)
|
|
} finally {
|
|
if ($listener.IsListening) { $listener.Stop() }
|
|
$listener.Close()
|
|
}
|