Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m38s
CD Pipeline / build-and-deploy (push) Failing after 24m19s
CD Pipeline / post-deploy-checks (push) Has been skipped
317 lines
10 KiB
PowerShell
317 lines
10 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[string]$ConfigPath = "",
|
|
[string]$SourceUrl = "",
|
|
[string]$SourceHost = "",
|
|
[int]$PollIntervalSeconds = 0,
|
|
[string]$IngressUrl = "",
|
|
[string]$TokenEnv = ""
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$CanonicalSourceHost = "192.168.0.110"
|
|
$CanonicalSourceUrl = "http://192.168.0.110:9093/api/v2/alerts"
|
|
$CanonicalIngressUrl = "https://awoooi.wooo.work/api/v1/webhooks/agent99/alertmanager-emergency"
|
|
$CanonicalAlertName = "AlertChainBroken_Alertmanager"
|
|
$CanonicalCatalogId = "ansible:110-alertmanager-delivery-recovery"
|
|
$DefaultTokenEnv = "AGENT99_SRE_RELAY_TOKEN"
|
|
|
|
function Get-AgentField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[object]$Default = $null
|
|
)
|
|
if ($Object -is [Collections.IDictionary] -and $Object.Contains($Name)) {
|
|
return $Object[$Name]
|
|
}
|
|
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
|
return $Object.PSObject.Properties[$Name].Value
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Get-AgentEnvValue {
|
|
param([string]$Name)
|
|
foreach ($target in @("Process", "User", "Machine")) {
|
|
$value = [Environment]::GetEnvironmentVariable($Name, $target)
|
|
if ($value) { return $value }
|
|
}
|
|
return ""
|
|
}
|
|
|
|
function Get-AgentSha256Text {
|
|
param([string]$Value)
|
|
$sha = [Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
return [BitConverter]::ToString(
|
|
$sha.ComputeHash([Text.Encoding]::UTF8.GetBytes($Value))
|
|
).Replace("-", "").ToLowerInvariant()
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
}
|
|
|
|
function Write-AgentJsonAtomically {
|
|
param(
|
|
[string]$Path,
|
|
[object]$Value
|
|
)
|
|
$temporaryPath = "$Path.tmp-$PID-$([guid]::NewGuid().ToString('N'))"
|
|
$Value | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8
|
|
Move-Item -Force $temporaryPath $Path
|
|
}
|
|
|
|
if (-not $ConfigPath) {
|
|
$ConfigPath = Join-Path $AgentRoot "config\agent99.config.json"
|
|
}
|
|
|
|
$configuredPoll = $null
|
|
if (Test-Path -LiteralPath $ConfigPath -PathType Leaf) {
|
|
$config = Get-Content -LiteralPath $ConfigPath -Raw | ConvertFrom-Json
|
|
$relay = Get-AgentField $config "sreAlertRelay" $null
|
|
$configuredPoll = Get-AgentField $relay "alertChainPoll" $null
|
|
}
|
|
|
|
if (-not $SourceUrl) {
|
|
$SourceUrl = [string](Get-AgentField $configuredPoll "sourceUrl" $CanonicalSourceUrl)
|
|
}
|
|
if (-not $SourceHost) {
|
|
$SourceHost = [string](Get-AgentField $configuredPoll "sourceHost" $CanonicalSourceHost)
|
|
}
|
|
if ($PollIntervalSeconds -le 0) {
|
|
$PollIntervalSeconds = [int](Get-AgentField $configuredPoll "pollIntervalSeconds" 60)
|
|
}
|
|
if (-not $IngressUrl) {
|
|
$IngressUrl = [string](Get-AgentField $configuredPoll "ingressUrl" $CanonicalIngressUrl)
|
|
}
|
|
if (-not $TokenEnv) {
|
|
$TokenEnv = [string](Get-AgentField $configuredPoll "tokenEnv" $DefaultTokenEnv)
|
|
}
|
|
|
|
$sourceUri = [Uri]$SourceUrl
|
|
$ingressUri = [Uri]$IngressUrl
|
|
if (
|
|
$SourceHost -ne $CanonicalSourceHost -or
|
|
$SourceUrl -ne $CanonicalSourceUrl -or
|
|
$sourceUri.Scheme -ne "http" -or
|
|
$sourceUri.Host -ne $CanonicalSourceHost -or
|
|
$sourceUri.Port -ne 9093 -or
|
|
$sourceUri.AbsolutePath -ne "/api/v2/alerts" -or
|
|
$sourceUri.Query -or
|
|
$sourceUri.UserInfo
|
|
) {
|
|
throw "alertmanager_poll_source_identity_not_exact"
|
|
}
|
|
if (
|
|
$IngressUrl -ne $CanonicalIngressUrl -or
|
|
$ingressUri.Scheme -ne "https" -or
|
|
$ingressUri.Host -ne "awoooi.wooo.work" -or
|
|
$ingressUri.AbsolutePath -ne "/api/v1/webhooks/agent99/alertmanager-emergency" -or
|
|
$ingressUri.Query -or
|
|
$ingressUri.UserInfo
|
|
) {
|
|
throw "alertmanager_poll_ingress_identity_not_exact"
|
|
}
|
|
if (
|
|
$PollIntervalSeconds -lt 60 -or
|
|
$PollIntervalSeconds -gt 300 -or
|
|
($PollIntervalSeconds % 60) -ne 0
|
|
) {
|
|
throw "alertmanager_poll_interval_out_of_bounds"
|
|
}
|
|
if ($TokenEnv -ne $DefaultTokenEnv) {
|
|
throw "alertmanager_poll_token_env_not_allowlisted"
|
|
}
|
|
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence\alertmanager-alertchain-poll"
|
|
$ReceiptDir = Join-Path $EvidenceDir "dedupe-receipts"
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $ReceiptDir | Out-Null
|
|
|
|
$pollStartedAt = (Get-Date).ToUniversalTime()
|
|
$summary = [ordered]@{
|
|
schemaVersion = "agent99_alertmanager_alertchain_poll_summary_v1"
|
|
status = "poll_in_progress"
|
|
sourceUrl = $CanonicalSourceUrl
|
|
sourceHost = $CanonicalSourceHost
|
|
pollIntervalSeconds = $PollIntervalSeconds
|
|
alertname = $CanonicalAlertName
|
|
firstHopAuthenticationUsed = $false
|
|
firstHopSecretTransmitted = $false
|
|
observedCount = 0
|
|
exactFiringCount = 0
|
|
relayedCount = 0
|
|
deduplicatedCount = 0
|
|
rejectedExactCount = 0
|
|
failedCount = 0
|
|
rawPayloadStored = $false
|
|
secretValueLogged = $false
|
|
runtimeWritePerformed = $false
|
|
runtimeClosureVerified = $false
|
|
polledAt = $pollStartedAt.ToString("o")
|
|
}
|
|
|
|
try {
|
|
# First hop is deliberately unauthenticated read-only HTTP on the exact LAN
|
|
# host. No credential, bearer header, query parameter or raw payload is saved.
|
|
$alerts = @(Invoke-RestMethod `
|
|
-Method Get `
|
|
-Uri $CanonicalSourceUrl `
|
|
-TimeoutSec 15)
|
|
} catch {
|
|
$summary.status = "alertmanager_poll_source_unavailable"
|
|
$summary.failedCount = 1
|
|
$summary.errorType = $_.Exception.GetType().Name
|
|
$summary.completedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
$summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json"
|
|
Write-AgentJsonAtomically -Path $summaryPath -Value $summary
|
|
$summary | ConvertTo-Json -Compress -Depth 8
|
|
throw "alertmanager_poll_source_unavailable"
|
|
}
|
|
|
|
$summary.observedCount = $alerts.Count
|
|
foreach ($alert in $alerts) {
|
|
$labels = Get-AgentField $alert "labels" $null
|
|
if ([string](Get-AgentField $labels "alertname" "") -ne $CanonicalAlertName) {
|
|
continue
|
|
}
|
|
|
|
$status = Get-AgentField $alert "status" $null
|
|
$severity = [string](Get-AgentField $labels "severity" "")
|
|
$state = [string](Get-AgentField $status "state" "")
|
|
$fingerprint = ([string](Get-AgentField $alert "fingerprint" "")).Trim().ToLowerInvariant()
|
|
$startsAt = ([string](Get-AgentField $alert "startsAt" "")).Trim()
|
|
$parsedStartsAt = [DateTimeOffset]::MinValue
|
|
if (
|
|
$severity -ne "critical" -or
|
|
$state -ne "active" -or
|
|
$fingerprint -notmatch "^[0-9a-f]{8,64}$" -or
|
|
-not [DateTimeOffset]::TryParse($startsAt, [ref]$parsedStartsAt)
|
|
) {
|
|
$summary.rejectedExactCount++
|
|
continue
|
|
}
|
|
|
|
$summary.exactFiringCount++
|
|
$sourceStartedAt = $parsedStartsAt.UtcDateTime.ToString("o")
|
|
$receiptDigest = Get-AgentSha256Text "$fingerprint|$sourceStartedAt"
|
|
$relayReceiptId = "agent99-alertchain-$receiptDigest"
|
|
$receiptPath = Join-Path $ReceiptDir "$relayReceiptId.json"
|
|
if (Test-Path -LiteralPath $receiptPath -PathType Leaf) {
|
|
try {
|
|
$existing = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json
|
|
if (
|
|
[string](Get-AgentField $existing "relayReceiptId" "") -eq $relayReceiptId -and
|
|
[string](Get-AgentField $existing "sourceFingerprint" "") -eq $fingerprint -and
|
|
[bool](Get-AgentField $existing "candidateQueued" $false) -and
|
|
-not [bool](Get-AgentField $existing "rawPayloadStored" $true)
|
|
) {
|
|
$summary.deduplicatedCount++
|
|
continue
|
|
}
|
|
} catch {
|
|
# A malformed public receipt cannot suppress retry.
|
|
}
|
|
}
|
|
|
|
$token = Get-AgentEnvValue $TokenEnv
|
|
if (-not $token) {
|
|
$summary.failedCount++
|
|
continue
|
|
}
|
|
|
|
$envelope = [ordered]@{
|
|
schema_version = "agent99_alertmanager_emergency_relay_v1"
|
|
relay_receipt_id = $relayReceiptId
|
|
source_host = $CanonicalSourceHost
|
|
source_url = $CanonicalSourceUrl
|
|
source_fingerprint = $fingerprint
|
|
source_started_at = $sourceStartedAt
|
|
alertname = $CanonicalAlertName
|
|
status = "firing"
|
|
severity = "critical"
|
|
target_resource = "alertmanager"
|
|
namespace = "monitoring"
|
|
agent99_role = "relay_coordination_only"
|
|
executor = "host_ansible_executor"
|
|
catalog_id = $CanonicalCatalogId
|
|
runtime_write_performed = $false
|
|
raw_payload_stored = $false
|
|
}
|
|
|
|
try {
|
|
$response = Invoke-RestMethod `
|
|
-Method Post `
|
|
-Uri $CanonicalIngressUrl `
|
|
-Headers @{ "X-Agent99-Relay-Token" = $token } `
|
|
-ContentType "application/json; charset=utf-8" `
|
|
-Body ($envelope | ConvertTo-Json -Compress -Depth 6) `
|
|
-TimeoutSec 30
|
|
} catch {
|
|
$summary.failedCount++
|
|
continue
|
|
}
|
|
|
|
$identity = Get-AgentField $response "identity" $null
|
|
$runId = [string](Get-AgentField $identity "run_id" "")
|
|
$traceId = [string](Get-AgentField $identity "trace_id" "")
|
|
$workItemId = [string](Get-AgentField $identity "work_item_id" "")
|
|
$parsedRunId = [guid]::Empty
|
|
$accepted = [bool](
|
|
[string](Get-AgentField $response "schema_version" "") -eq "alert_chain_emergency_ingress_result_v1" -and
|
|
[bool](Get-AgentField $response "candidate_persisted" $false) -and
|
|
[bool](Get-AgentField $response "candidate_queued" $false) -and
|
|
[string](Get-AgentField $response "agent99_role" "") -eq "relay_coordination_only" -and
|
|
[string](Get-AgentField $response "executor" "") -eq "host_ansible_executor" -and
|
|
[string](Get-AgentField $response "catalog_id" "") -eq $CanonicalCatalogId -and
|
|
-not [bool](Get-AgentField $response "runtime_write_performed" $true) -and
|
|
-not [bool](Get-AgentField $response "runtime_closure_verified" $true) -and
|
|
[guid]::TryParse($runId, [ref]$parsedRunId) -and
|
|
$traceId -eq $runId -and
|
|
$workItemId.StartsWith("alertchain:awoooi:")
|
|
)
|
|
if (-not $accepted) {
|
|
$summary.failedCount++
|
|
continue
|
|
}
|
|
|
|
$publicReceipt = [ordered]@{
|
|
schemaVersion = "agent99_alertmanager_alertchain_poll_receipt_v1"
|
|
relayReceiptId = $relayReceiptId
|
|
sourceHost = $CanonicalSourceHost
|
|
sourceFingerprint = $fingerprint
|
|
sourceStartedAt = $sourceStartedAt
|
|
candidatePersisted = $true
|
|
candidateQueued = $true
|
|
runId = $runId
|
|
traceId = $traceId
|
|
workItemId = $workItemId
|
|
agent99Role = "relay_coordination_only"
|
|
executor = "host_ansible_executor"
|
|
catalogId = $CanonicalCatalogId
|
|
firstHopAuthenticationUsed = $false
|
|
firstHopSecretTransmitted = $false
|
|
runtimeWritePerformed = $false
|
|
runtimeClosureVerified = $false
|
|
rawPayloadStored = $false
|
|
secretValueLogged = $false
|
|
receivedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
}
|
|
Write-AgentJsonAtomically -Path $receiptPath -Value $publicReceipt
|
|
$summary.relayedCount++
|
|
}
|
|
|
|
$summary.status = if ($summary.failedCount -eq 0) {
|
|
"poll_completed"
|
|
} else {
|
|
"poll_partial_retry_required"
|
|
}
|
|
$summary.completedAt = (Get-Date).ToUniversalTime().ToString("o")
|
|
$summaryPath = Join-Path $EvidenceDir "poll-$($pollStartedAt.ToString('yyyyMMdd-HHmmss'))-$([guid]::NewGuid().ToString('N')).json"
|
|
Write-AgentJsonAtomically -Path $summaryPath -Value $summary
|
|
$summary | ConvertTo-Json -Compress -Depth 8
|
|
if ($summary.failedCount -gt 0) {
|
|
throw "alertmanager_poll_partial_retry_required"
|
|
}
|