Files
awoooi/agent99-alertmanager-alertchain-poll.ps1
ogt 541a32a9cb
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
feat(sre): enforce typed controlled automation
2026-07-16 17:32:01 +08:00

317 lines
10 KiB
PowerShell

param(
[string]$AgentRoot = "C:\Wooo\Agent99",
[string]$ConfigPath = "",
[string]$SourceUrl = "",
[string]$SourceHost = "",
[int]$PollIntervalSeconds = 0,
[string]$IngressUrl = "",
[string]$TokenEnv = ""
)
$ErrorActionPreference = "Stop"
$CanonicalSourceHost = "192.168.0.110"
$CanonicalSourceUrl = "http://192.168.0.110:9093/api/v2/alerts"
$CanonicalIngressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency"
$CanonicalAlertName = "AlertChainBroken_Alertmanager"
$CanonicalCatalogId = "ansible:110-alertmanager-delivery-recovery"
$DefaultTokenEnv = "AGENT99_SRE_RELAY_TOKEN"
function Get-AgentField {
param(
[object]$Object,
[string]$Name,
[object]$Default = $null
)
if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) {
return $Object[$Name]
}
if ($Object -and $Object.PSObject.Properties[$Name]) {
return $Object.PSObject.Properties[$Name].Value
}
return $Default
}
function Get-AgentEnvValue {
param([string]$Name)
foreach ($target in @("Process", "User", "Machine")) {
$value = [Environment]::GetEnvironmentVariable($Name, $target)
if ($value) { return $value }
}
return ""
}
function Get-AgentSha256Text {
param([string]$Value)
$sha = [Security.Cryptography.SHA256]::Create()
try {
return [BitConverter]::ToString(
$sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Value))
).Replace("-", "").ToLowerInvariant()
} finally {
$sha.Dispose()
}
}
function Write-AgentJsonAtomically {
param(
[string]$Path,
[object]$Value
)
$temporaryPath = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
$Value | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8
Move-Item -Force $temporaryPath $Path
}
if (-not $ConfigPath) {
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
}
$configuredPoll = $null
if (Test-Path -LiteralPath $ConfigPath -PathType Leaf) {
$config = Get-Content -LiteralPath $ConfigPath -Raw | ConvertFrom-Json
$relay = Get-AgentField $config "sreAlertRelay" $null
$configuredPoll = Get-AgentField $relay "alertChainPoll" $null
}
if (-not $SourceUrl) {
$SourceUrl = [string](Get-AgentField $configuredPoll "sourceUrl" $CanonicalSourceUrl)
}
if (-not $SourceHost) {
$SourceHost = [string](Get-AgentField $configuredPoll "sourceHost" $CanonicalSourceHost)
}
if ($PollIntervalSeconds -le 0) {
$PollIntervalSeconds = [int](Get-AgentField $configuredPoll "pollIntervalSeconds" 60)
}
if (-not $IngressUrl) {
$IngressUrl = [string](Get-AgentField $configuredPoll "ingressUrl" $CanonicalIngressUrl)
}
if (-not $TokenEnv) {
$TokenEnv = [string](Get-AgentField $configuredPoll "tokenEnv" $DefaultTokenEnv)
}
$sourceUri = [Uri]$SourceUrl
$ingressUri = [Uri]$IngressUrl
if (
$SourceHost -ne $CanonicalSourceHost -or
$SourceUrl -ne $CanonicalSourceUrl -or
$sourceUri.Scheme -ne "http" -or
$sourceUri.Host -ne $CanonicalSourceHost -or
$sourceUri.Port -ne 9093 -or
$sourceUri.AbsolutePath -ne "/api/v2/alerts" -or
$sourceUri.Query -or
$sourceUri.UserInfo
) {
throw "alertmanager_poll_source_identity_not_exact"
}
if (
$IngressUrl -ne $CanonicalIngressUrl -or
$ingressUri.Scheme -ne "https" -or
$ingressUri.Host -ne "awoooi.wooo.work" -or
$ingressUri.AbsolutePath -ne "/api/v1/webhooks/agent99/alertmanager-emergency" -or
$ingressUri.Query -or
$ingressUri.UserInfo
) {
throw "alertmanager_poll_ingress_identity_not_exact"
}
if (
$PollIntervalSeconds -lt 60 -or
$PollIntervalSeconds -gt 300 -or
($PollIntervalSeconds % 60) -ne 0
) {
throw "alertmanager_poll_interval_out_of_bounds"
}
if ($TokenEnv -ne $DefaultTokenEnv) {
throw "alertmanager_poll_token_env_not_allowlisted"
}
$EvidenceDir = Join-Path $AgentRoot "evidence\alertmanager-alertchain-poll"
$ReceiptDir = Join-Path $EvidenceDir "dedupe-receipts"
New-Item -ItemType Directory -Force -Path $EvidenceDir, $ReceiptDir | Out-Null
$pollStartedAt = (Get-Date).ToUniversalTime()
$summary = [ordered]@{
schemaVersion = "agent99_alertmanager_alertchain_poll_summary_v1"
status = "poll_in_progress"
sourceUrl = $CanonicalSourceUrl
sourceHost = $CanonicalSourceHost
pollIntervalSeconds = $PollIntervalSeconds
alertname = $CanonicalAlertName
firstHopAuthenticationUsed = $false
firstHopSecretTransmitted = $false
observedCount = 0
exactFiringCount = 0
relayedCount = 0
deduplicatedCount = 0
rejectedExactCount = 0
failedCount = 0
rawPayloadStored = $false
secretValueLogged = $false
runtimeWritePerformed = $false
runtimeClosureVerified = $false
polledAt = $pollStartedAt.ToString("o")
}
try {
# First hop is deliberately unauthenticated read-only HTTP on the exact LAN
# host. No credential, bearer header, query parameter or raw payload is saved.
$alerts = @(Invoke-RestMethod `
-Method Get `
-Uri $CanonicalSourceUrl `
-TimeoutSec 15)
} catch {
$summary.status = "alertmanager_poll_source_unavailable"
$summary.failedCount = 1
$summary.errorType = $_.Exception.GetType().Name
$summary.completedAt = (Get-Date).ToUniversalTime().ToString("o")
$summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json"
Write-AgentJsonAtomically -Path $summaryPath -Value $summary
$summary | ConvertTo-Json -Compress -Depth 8
throw "alertmanager_poll_source_unavailable"
}
$summary.observedCount = $alerts.Count
foreach ($alert in $alerts) {
$labels = Get-AgentField $alert "labels" $null
if ([string](Get-AgentField $labels "alertname" "") -ne $CanonicalAlertName) {
continue
}
$status = Get-AgentField $alert "status" $null
$severity = [string](Get-AgentField $labels "severity" "")
$state = [string](Get-AgentField $status "state" "")
$fingerprint = ([string](Get-AgentField $alert "fingerprint" "")).Trim().ToLowerInvariant()
$startsAt = ([string](Get-AgentField $alert "startsAt" "")).Trim()
$parsedStartsAt = [DateTimeOffset]::MinValue
if (
$severity -ne "critical" -or
$state -ne "active" -or
$fingerprint -notmatch "^[0-9a-f]{8,64}$" -or
-not [DateTimeOffset]::TryParse($startsAt, [ref]$parsedStartsAt)
) {
$summary.rejectedExactCount++
continue
}
$summary.exactFiringCount++
$sourceStartedAt = $parsedStartsAt.UtcDateTime.ToString("o")
$receiptDigest = Get-AgentSha256Text "$fingerprint|$sourceStartedAt"
$relayReceiptId = "agent99-alertchain-$receiptDigest"
$receiptPath = Join-Path $ReceiptDir "$relayReceiptId.json"
if (Test-Path -LiteralPath $receiptPath -PathType Leaf) {
try {
$existing = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json
if (
[string](Get-AgentField $existing "relayReceiptId" "") -eq $relayReceiptId -and
[string](Get-AgentField $existing "sourceFingerprint" "") -eq $fingerprint -and
[bool](Get-AgentField $existing "candidateQueued" $false) -and
-not [bool](Get-AgentField $existing "rawPayloadStored" $true)
) {
$summary.deduplicatedCount++
continue
}
} catch {
# A malformed public receipt cannot suppress retry.
}
}
$token = Get-AgentEnvValue $TokenEnv
if (-not $token) {
$summary.failedCount++
continue
}
$envelope = [ordered]@{
schema_version = "agent99_alertmanager_emergency_relay_v1"
relay_receipt_id = $relayReceiptId
source_host = $CanonicalSourceHost
source_url = $CanonicalSourceUrl
source_fingerprint = $fingerprint
source_started_at = $sourceStartedAt
alertname = $CanonicalAlertName
status = "firing"
severity = "critical"
target_resource = "alertmanager"
namespace = "monitoring"
agent99_role = "relay_coordination_only"
executor = "host_ansible_executor"
catalog_id = $CanonicalCatalogId
runtime_write_performed = $false
raw_payload_stored = $false
}
try {
$response = Invoke-RestMethod `
-Method Post `
-Uri $CanonicalIngressUrl `
-Headers @{ "X-Agent99-Relay-Token" = $token } `
-ContentType "application/json; charset=utf-8" `
-Body ($envelope | ConvertTo-Json -Compress -Depth 6) `
-TimeoutSec 30
} catch {
$summary.failedCount++
continue
}
$identity = Get-AgentField $response "identity" $null
$runId = [string](Get-AgentField $identity "run_id" "")
$traceId = [string](Get-AgentField $identity "trace_id" "")
$workItemId = [string](Get-AgentField $identity "work_item_id" "")
$parsedRunId = [guid]::Empty
$accepted = [bool](
[string](Get-AgentField $response "schema_version" "") -eq "alert_chain_emergency_ingress_result_v1" -and
[bool](Get-AgentField $response "candidate_persisted" $false) -and
[bool](Get-AgentField $response "candidate_queued" $false) -and
[string](Get-AgentField $response "agent99_role" "") -eq "relay_coordination_only" -and
[string](Get-AgentField $response "executor" "") -eq "host_ansible_executor" -and
[string](Get-AgentField $response "catalog_id" "") -eq $CanonicalCatalogId -and
-not [bool](Get-AgentField $response "runtime_write_performed" $true) -and
-not [bool](Get-AgentField $response "runtime_closure_verified" $true) -and
[guid]::TryParse($runId, [ref]$parsedRunId) -and
$traceId -eq $runId -and
$workItemId.StartsWith("alertchain:awoooi:")
)
if (-not $accepted) {
$summary.failedCount++
continue
}
$publicReceipt = [ordered]@{
schemaVersion = "agent99_alertmanager_alertchain_poll_receipt_v1"
relayReceiptId = $relayReceiptId
sourceHost = $CanonicalSourceHost
sourceFingerprint = $fingerprint
sourceStartedAt = $sourceStartedAt
candidatePersisted = $true
candidateQueued = $true
runId = $runId
traceId = $traceId
workItemId = $workItemId
agent99Role = "relay_coordination_only"
executor = "host_ansible_executor"
catalogId = $CanonicalCatalogId
firstHopAuthenticationUsed = $false
firstHopSecretTransmitted = $false
runtimeWritePerformed = $false
runtimeClosureVerified = $false
rawPayloadStored = $false
secretValueLogged = $false
receivedAt = (Get-Date).ToUniversalTime().ToString("o")
}
Write-AgentJsonAtomically -Path $receiptPath -Value $publicReceipt
$summary.relayedCount++
}
$summary.status = if ($summary.failedCount -eq 0) {
"poll_completed"
} else {
"poll_partial_retry_required"
}
$summary.completedAt = (Get-Date).ToUniversalTime().ToString("o")
$summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json"
Write-AgentJsonAtomically -Path $summaryPath -Value $summary
$summary | ConvertTo-Json -Compress -Depth 8
if ($summary.failedCount -gt 0) {
throw "alertmanager_poll_partial_retry_required"
}