34e32f6ed6
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m27s
CD Pipeline / build-and-deploy (push) Successful in 6m39s
CD Pipeline / post-deploy-checks (push) Successful in 1m55s
857 lines
32 KiB
PowerShell
857 lines
32 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[switch]$RunNow,
|
|
[switch]$SelfTest,
|
|
[int]$MaxAlerts = 10
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$AlertRoot = Join-Path $AgentRoot "alerts"
|
|
$IncomingDir = Join-Path $AlertRoot "incoming"
|
|
$RunningDir = Join-Path $AlertRoot "running"
|
|
$ProcessedDir = Join-Path $AlertRoot "processed"
|
|
$FailedDir = Join-Path $AlertRoot "failed"
|
|
$IgnoredDir = Join-Path $AlertRoot "ignored"
|
|
$QueueDir = Join-Path $AgentRoot "queue"
|
|
$RequestDir = Join-Path $AgentRoot "requests"
|
|
$StateDir = Join-Path $AgentRoot "state"
|
|
$DedupeStatePath = Join-Path $StateDir "sre-alert-single-flight.json"
|
|
$InboxLockPath = Join-Path $StateDir "sre-alert-inbox.lock"
|
|
$DedupeWindowSeconds = 300
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null
|
|
|
|
$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage")
|
|
$remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed")
|
|
|
|
function Get-AgentField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[object]$Default = $null
|
|
)
|
|
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
|
return $Object.PSObject.Properties[$Name].Value
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Convert-AgentBool {
|
|
param([object]$Value)
|
|
if ($null -eq $Value) { return $false }
|
|
if ($Value -is [bool]) { return [bool]$Value }
|
|
$text = ([string]$Value).Trim()
|
|
if (-not $text) { return $false }
|
|
return [bool]($text -match "^(?i:true|1|yes|y|on)$")
|
|
}
|
|
|
|
function Convert-AgentSafeFileToken {
|
|
param([string]$Value)
|
|
$safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
|
|
if (-not $safe) { return "alert" }
|
|
if ($safe.Length -gt 80) { return $safe.Substring(0, 80) }
|
|
return $safe
|
|
}
|
|
|
|
function Get-AgentAlertText {
|
|
param([object]$Alert)
|
|
$labels = @()
|
|
$rawLabels = Get-AgentField $Alert "labels" @()
|
|
foreach ($label in @($rawLabels)) {
|
|
if ($label -ne $null) { $labels += [string]$label }
|
|
}
|
|
@(
|
|
(Get-AgentField $Alert "source" ""),
|
|
(Get-AgentField $Alert "severity" ""),
|
|
(Get-AgentField $Alert "kind" ""),
|
|
(Get-AgentField $Alert "service" ""),
|
|
(Get-AgentField $Alert "host" ""),
|
|
(Get-AgentField $Alert "title" ""),
|
|
(Get-AgentField $Alert "message" ""),
|
|
($labels -join " ")
|
|
) -join " "
|
|
}
|
|
|
|
function Test-AgentAlertActionable {
|
|
param([object]$Alert)
|
|
$force = Convert-AgentBool (Get-AgentField $Alert "force" $false)
|
|
if ($force) { return $true }
|
|
|
|
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
|
|
$severity = ([string](Get-AgentField $Alert "severity" "")).ToLowerInvariant()
|
|
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
|
|
return $true
|
|
}
|
|
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
|
|
return $true
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Resolve-AgentAlertRoute {
|
|
param([object]$Alert)
|
|
|
|
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
|
|
$suggestedMode = ([string](Get-AgentField $Alert "suggestedMode" "")).Trim()
|
|
$kind = ([string](Get-AgentField $Alert "kind" "")).Trim().ToLowerInvariant()
|
|
|
|
if ($suggestedMode -and $suggestedMode -in $allowedModes) {
|
|
return [pscustomobject]@{ mode = $suggestedMode; source = "suggestedMode"; matched = $suggestedMode }
|
|
}
|
|
|
|
$kindModes = @{
|
|
"provider_freshness_signal" = "ProviderFreshness"
|
|
"backup_health" = "BackupCheck"
|
|
"backup_missing" = "BackupCheck"
|
|
"backup_failed" = "BackupCheck"
|
|
"security_health" = "SecurityTriage"
|
|
"wazuh_security_alert" = "SecurityTriage"
|
|
"public_route_502" = "Recover"
|
|
"host_recovery" = "Recover"
|
|
"host_down" = "Recover"
|
|
"host_reboot" = "Recover"
|
|
"vm_not_running" = "Recover"
|
|
"performance_pressure" = "Perf"
|
|
"performance_cpu" = "Perf"
|
|
"performance_memory" = "Perf"
|
|
"performance_disk" = "Perf"
|
|
"harbor_registry" = "HarborRepair"
|
|
"awoooi_k3s" = "AwoooRepair"
|
|
"database_health" = "Status"
|
|
"alert_chain_health" = "Status"
|
|
"application_ai_config_error" = "Status"
|
|
"repository_missing" = "Status"
|
|
"cicd_failure" = "Status"
|
|
}
|
|
if ($kind -and $kindModes.ContainsKey($kind)) {
|
|
return [pscustomobject]@{ mode = [string]$kindModes[$kind]; source = "kind"; matched = $kind }
|
|
}
|
|
|
|
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
|
|
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" }
|
|
}
|
|
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
|
|
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" }
|
|
}
|
|
if ($text -match "backup|snapshot|restore drill|cron") {
|
|
return [pscustomobject]@{ mode = "BackupCheck"; source = "text_fallback"; matched = "backup" }
|
|
}
|
|
if ($text -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|privilege|vulnerability|cve|ioc|suspicious|unauthorized") {
|
|
return [pscustomobject]@{ mode = "SecurityTriage"; source = "text_fallback"; matched = "security" }
|
|
}
|
|
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") {
|
|
return [pscustomobject]@{ mode = "ProviderFreshness"; source = "text_fallback"; matched = "provider_freshness" }
|
|
}
|
|
if ($text -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag|alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") {
|
|
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "diagnostic_only" }
|
|
}
|
|
if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") {
|
|
return [pscustomobject]@{ mode = "Perf"; source = "text_fallback"; matched = "performance" }
|
|
}
|
|
if ($text -match "harbor|registry") {
|
|
return [pscustomobject]@{ mode = "HarborRepair"; source = "text_fallback"; matched = "harbor" }
|
|
}
|
|
if ($text -match "ci/?cd|cicd|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") {
|
|
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "cicd_failure" }
|
|
}
|
|
if ($text -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") {
|
|
return [pscustomobject]@{ mode = "AwoooRepair"; source = "text_fallback"; matched = "k3s" }
|
|
}
|
|
if ($text -match "missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
|
|
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "application_config" }
|
|
}
|
|
if ($text -match "gitea|repo_missing|repository_missing|repo missing|repository missing") {
|
|
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "repository" }
|
|
}
|
|
return [pscustomobject]@{ mode = "Status"; source = "default"; matched = "none" }
|
|
}
|
|
|
|
function Resolve-AgentAlertMode {
|
|
param([object]$Alert)
|
|
$route = Resolve-AgentAlertRoute $Alert
|
|
return [string]$route.mode
|
|
}
|
|
|
|
function Get-AgentAlertCorrelationKey {
|
|
param([object]$Alert)
|
|
|
|
$routing = Get-AgentField $Alert "routing" $null
|
|
$correlationKey = [string](Get-AgentField $routing "correlationKey" "")
|
|
if ($correlationKey) { return "route:$($correlationKey.ToLowerInvariant())" }
|
|
|
|
$awoooi = Get-AgentField $Alert "awoooi" $null
|
|
$fingerprint = [string](Get-AgentField $awoooi "fingerprint" "")
|
|
if (-not $fingerprint) { $fingerprint = [string](Get-AgentField $Alert "fingerprint" "") }
|
|
if ($fingerprint) { return "fingerprint:$($fingerprint.ToLowerInvariant())" }
|
|
|
|
$kind = ([string](Get-AgentField $Alert "kind" "monitoring_alert")).ToLowerInvariant()
|
|
$service = ([string](Get-AgentField $Alert "service" "unknown")).ToLowerInvariant()
|
|
$hostName = ([string](Get-AgentField $Alert "host" "no-host")).ToLowerInvariant()
|
|
return "fallback:$kind`:$service`:$hostName"
|
|
}
|
|
|
|
function Get-AgentAlertSeverityRank {
|
|
param([string]$Severity)
|
|
switch ($Severity.ToLowerInvariant()) {
|
|
"critical" { return 4 }
|
|
"error" { return 4 }
|
|
"failed" { return 4 }
|
|
"warning" { return 3 }
|
|
"degraded" { return 3 }
|
|
"info" { return 2 }
|
|
default { return 1 }
|
|
}
|
|
}
|
|
|
|
function Save-AgentAlertDedupeState {
|
|
param([object[]]$Entries)
|
|
$payload = [pscustomobject]@{
|
|
schemaVersion = "agent99_sre_alert_single_flight_v1"
|
|
generatedAt = (Get-Date -Format o)
|
|
windowSeconds = $DedupeWindowSeconds
|
|
entries = @($Entries)
|
|
}
|
|
$temporaryPath = "$DedupeStatePath.tmp-$PID"
|
|
$payload | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8
|
|
Move-Item -Force $temporaryPath $DedupeStatePath
|
|
}
|
|
|
|
function New-AgentInstructionFromAlert {
|
|
param(
|
|
[object]$Alert,
|
|
[string]$AlertId,
|
|
[string]$ModeName
|
|
)
|
|
|
|
$existing = [string](Get-AgentField $Alert "instruction" "")
|
|
if ($existing) { return $existing }
|
|
|
|
$source = [string](Get-AgentField $Alert "source" "sre-war-room")
|
|
$severity = [string](Get-AgentField $Alert "severity" "unknown")
|
|
$kind = [string](Get-AgentField $Alert "kind" "unknown")
|
|
$service = [string](Get-AgentField $Alert "service" "unknown")
|
|
$hostName = [string](Get-AgentField $Alert "host" "")
|
|
$title = [string](Get-AgentField $Alert "title" "")
|
|
$message = [string](Get-AgentField $Alert "message" "")
|
|
|
|
if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|provider freshness|provider window|last_seen|last seen|ingestion") {
|
|
return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload."
|
|
}
|
|
if ($ModeName -eq "SecurityTriage" -or $kind -match "security|wazuh|siem") {
|
|
return "Security triage required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; classify blast radius, affected host/service, source log reference, repeated count, and recommended allowlisted next action; do not run arbitrary shell, delete data, disable controls, or expose secrets."
|
|
}
|
|
|
|
$parts = @("SRE alert", "id=$AlertId", "source=$source", "severity=$severity", "service=$service", "kind=$kind", "mode=$ModeName")
|
|
if ($hostName) { $parts += "host=$hostName" }
|
|
if ($title) { $parts += "title=$title" }
|
|
if ($message) { $parts += "message=$message" }
|
|
return ($parts -join "; ")
|
|
}
|
|
|
|
function Get-AgentControlledApplyForAlert {
|
|
param(
|
|
[object]$Alert,
|
|
[string]$ModeName
|
|
)
|
|
if ($Alert -and $Alert.PSObject.Properties["controlledApply"]) {
|
|
return [bool]$Alert.controlledApply
|
|
}
|
|
return [bool]($ModeName -in $remediationModes)
|
|
}
|
|
|
|
function Test-AgentAlertSuppressTelegram {
|
|
param([object]$Alert)
|
|
if (Convert-AgentBool (Get-AgentField $Alert "suppressTelegram" $false)) { return $true }
|
|
$text = (Get-AgentAlertText $Alert)
|
|
if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true }
|
|
return $false
|
|
}
|
|
|
|
function Invoke-AgentAlertRoutingSelfTest {
|
|
$cases = @(
|
|
[pscustomobject]@{
|
|
name = "public route 502"
|
|
expectedActionable = $true
|
|
expectedMode = "Recover"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-public-502"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "public_route_502"
|
|
service = "awoooi"
|
|
host = "192.168.0.120"
|
|
message = "HTTP 502 on public route"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "host down or reboot"
|
|
expectedActionable = $true
|
|
expectedMode = "Recover"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-host-down"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "host_down"
|
|
service = "vmware"
|
|
host = "192.168.0.110"
|
|
message = "host_reboot detected vm_not_running"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "performance pressure"
|
|
expectedActionable = $true
|
|
expectedMode = "Perf"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-performance"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "performance_pressure"
|
|
service = "reboot-auto-recovery-slo"
|
|
host = "110"
|
|
message = "load_per_core critical cpu pressure disk_used high"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "backup freshness"
|
|
expectedActionable = $true
|
|
expectedMode = "BackupCheck"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-backup"
|
|
source = "agent99-routing-selftest"
|
|
severity = "warning"
|
|
kind = "backup_missing"
|
|
service = "backup"
|
|
message = "backup_failed snapshot cron freshness missing"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "harbor registry"
|
|
expectedActionable = $true
|
|
expectedMode = "HarborRepair"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-harbor"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "registry_unhealthy"
|
|
service = "harbor"
|
|
host = "192.168.0.110"
|
|
message = "harbor registry unhealthy"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "k3s pod failure"
|
|
expectedActionable = $true
|
|
expectedMode = "AwoooRepair"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-k3s"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "pod_unhealthy"
|
|
service = "awoooi"
|
|
host = "192.168.0.120"
|
|
message = "k3s pod crashloopbackoff imagepullbackoff deployment failure"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "reboot slo wins over cpu contamination"
|
|
expectedActionable = $true
|
|
expectedMode = "Recover"
|
|
expectedRouteSource = "suggestedMode"
|
|
expectedControlledApply = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-reboot-slo"
|
|
source = "awoooi-api-alertmanager"
|
|
severity = "critical"
|
|
kind = "host_recovery"
|
|
suggestedMode = "Recover"
|
|
service = "reboot-auto-recovery-slo"
|
|
host = "192.168.0.110"
|
|
message = "all_required_hosts_not_in_10_minute_reboot_window cpu pressure"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "postgres does not route to k3s"
|
|
expectedActionable = $true
|
|
expectedMode = "Status"
|
|
expectedRouteSource = "kind"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-postgres"
|
|
source = "awoooi-api-alertmanager"
|
|
severity = "critical"
|
|
kind = "database_health"
|
|
service = "awoooi"
|
|
message = "PostgreSQL database unavailable in deployment"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "alert chain does not route to k3s"
|
|
expectedActionable = $true
|
|
expectedMode = "Status"
|
|
expectedRouteSource = "kind"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-alert-chain"
|
|
source = "awoooi-api-alertmanager"
|
|
severity = "critical"
|
|
kind = "alert_chain_health"
|
|
service = "awoooi"
|
|
message = "AlertChainBroken monitoring pipeline down"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "provider freshness"
|
|
expectedActionable = $true
|
|
expectedMode = "ProviderFreshness"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-provider-freshness"
|
|
source = "agent99-routing-selftest"
|
|
severity = "warning"
|
|
kind = "provider_freshness_signal"
|
|
service = "provider_freshness_signal"
|
|
message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "provider freshness do not page"
|
|
expectedActionable = $true
|
|
expectedMode = "ProviderFreshness"
|
|
expectedControlledApply = $false
|
|
expectedSuppressTelegram = $true
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-provider-freshness-do-not-page"
|
|
source = "agent99-routing-selftest"
|
|
severity = "warning"
|
|
kind = "provider_freshness_signal"
|
|
service = "provider_freshness_signal"
|
|
message = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
|
|
suppressTelegram = $true
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "security alert"
|
|
expectedActionable = $true
|
|
expectedMode = "SecurityTriage"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-security"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "wazuh_security_alert"
|
|
service = "iwooos-security"
|
|
host = "192.168.0.112"
|
|
message = "Wazuh SIEM security alert auth_failed brute force suspicious login"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "gitea repository missing"
|
|
expectedActionable = $true
|
|
expectedMode = "Status"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-gitea-repo-missing"
|
|
source = "agent99-routing-selftest"
|
|
severity = "critical"
|
|
kind = "repository_missing"
|
|
service = "gitea"
|
|
host = "192.168.0.110"
|
|
message = "Gitea repo_missing repository missing for production project"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "application ai config error"
|
|
expectedActionable = $true
|
|
expectedMode = "Status"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-app-ai-config"
|
|
source = "agent99-routing-selftest"
|
|
severity = "warning"
|
|
kind = "application_ai_config_error"
|
|
service = "bitan-pharmacy"
|
|
host = ""
|
|
message = "AI support locale guard failed E_AI_MISSING_KEY missing_key"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "unknown warning"
|
|
expectedActionable = $true
|
|
expectedMode = "Status"
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-unknown-warning"
|
|
source = "agent99-routing-selftest"
|
|
severity = "warning"
|
|
kind = "unknown"
|
|
service = "unknown"
|
|
message = "warning with no known remediation pattern"
|
|
}
|
|
},
|
|
[pscustomobject]@{
|
|
name = "info noise"
|
|
expectedActionable = $false
|
|
expectedMode = $null
|
|
expectedControlledApply = $false
|
|
alert = [pscustomobject]@{
|
|
id = "selftest-info-noise"
|
|
source = "agent99-routing-selftest"
|
|
severity = "info"
|
|
kind = "heartbeat"
|
|
service = "agent99"
|
|
message = "normal heartbeat"
|
|
}
|
|
}
|
|
)
|
|
|
|
$results = @()
|
|
foreach ($case in $cases) {
|
|
$actionable = Test-AgentAlertActionable $case.alert
|
|
$route = if ($actionable) { Resolve-AgentAlertRoute $case.alert } else { $null }
|
|
$mode = if ($route) { [string]$route.mode } else { $null }
|
|
$routeSource = if ($route) { [string]$route.source } else { $null }
|
|
$controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false }
|
|
$suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false }
|
|
$expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
|
|
$expectedRouteSource = if ($case.PSObject.Properties["expectedRouteSource"]) { [string]$case.expectedRouteSource } else { $null }
|
|
$instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null }
|
|
$ok = [bool](
|
|
$actionable -eq $case.expectedActionable -and
|
|
[string]$mode -eq [string]$case.expectedMode -and
|
|
(-not $expectedRouteSource -or $routeSource -eq $expectedRouteSource) -and
|
|
$controlled -eq $case.expectedControlledApply -and
|
|
$suppressTelegram -eq $expectedSuppressTelegram
|
|
)
|
|
$results += [pscustomobject]@{
|
|
name = $case.name
|
|
ok = $ok
|
|
alertId = $case.alert.id
|
|
actionable = $actionable
|
|
expectedActionable = $case.expectedActionable
|
|
mode = $mode
|
|
expectedMode = $case.expectedMode
|
|
routeSource = $routeSource
|
|
expectedRouteSource = $expectedRouteSource
|
|
controlledApply = $controlled
|
|
expectedControlledApply = $case.expectedControlledApply
|
|
suppressTelegram = $suppressTelegram
|
|
expectedSuppressTelegram = $expectedSuppressTelegram
|
|
instruction = $instruction
|
|
}
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = [bool](@($results | Where-Object { -not $_.ok }).Count -eq 0)
|
|
caseCount = @($results).Count
|
|
passedCount = @($results | Where-Object { $_.ok }).Count
|
|
failedCount = @($results | Where-Object { -not $_.ok }).Count
|
|
allowedModes = $allowedModes
|
|
remediationModes = $remediationModes
|
|
results = $results
|
|
}
|
|
$path = Join-Path $EvidenceDir ("agent99-SreAlertRoutingSelfTest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
|
|
$result | Add-Member -MemberType NoteProperty -Name evidencePath -Value $path -Force
|
|
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $path -Encoding UTF8
|
|
$result
|
|
}
|
|
|
|
if ($SelfTest) {
|
|
$selfTestResult = Invoke-AgentAlertRoutingSelfTest
|
|
$selfTestResult | ConvertTo-Json -Depth 10
|
|
if (-not $selfTestResult.ok) { exit 1 }
|
|
exit 0
|
|
}
|
|
|
|
$inboxLockStream = $null
|
|
try {
|
|
$inboxLockStream = [System.IO.File]::Open(
|
|
$InboxLockPath,
|
|
[System.IO.FileMode]::OpenOrCreate,
|
|
[System.IO.FileAccess]::ReadWrite,
|
|
[System.IO.FileShare]::None
|
|
)
|
|
} catch {
|
|
$lockResult = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = $true
|
|
skipped = $true
|
|
reason = "single_flight_lock_held"
|
|
lockPath = $InboxLockPath
|
|
}
|
|
$lockEvidencePath = Join-Path $EvidenceDir ("agent99-SreAlertInboxLock-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
|
|
$lockResult | ConvertTo-Json -Depth 6 | Set-Content -Path $lockEvidencePath -Encoding UTF8
|
|
$lockResult | ConvertTo-Json -Depth 6
|
|
exit 0
|
|
}
|
|
|
|
$dedupeEntries = @()
|
|
if (Test-Path $DedupeStatePath) {
|
|
try {
|
|
$dedupePayload = Get-Content $DedupeStatePath -Raw | ConvertFrom-Json
|
|
foreach ($entry in @($dedupePayload.entries)) {
|
|
try {
|
|
$ageSeconds = ((Get-Date).ToUniversalTime() - ([datetime]::Parse([string]$entry.lastSeen)).ToUniversalTime()).TotalSeconds
|
|
if ($ageSeconds -ge 0 -and $ageSeconds -le $DedupeWindowSeconds) {
|
|
$dedupeEntries += $entry
|
|
}
|
|
} catch {
|
|
continue
|
|
}
|
|
}
|
|
} catch {
|
|
$dedupeEntries = @()
|
|
}
|
|
}
|
|
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json"
|
|
$queued = @()
|
|
$ignored = @()
|
|
$failed = @()
|
|
$duplicateSuppressedCount = 0
|
|
$severityEscalationCount = 0
|
|
|
|
$files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime |
|
|
Select-Object -First $MaxAlerts)
|
|
|
|
foreach ($file in $files) {
|
|
$runningPath = Join-Path $RunningDir $file.Name
|
|
Move-Item -Force $file.FullName $runningPath
|
|
|
|
try {
|
|
$alert = Get-Content $runningPath -Raw | ConvertFrom-Json
|
|
} catch {
|
|
$failedPath = Join-Path $FailedDir ("failed-" + $file.Name)
|
|
Move-Item -Force $runningPath $failedPath
|
|
$failed += [pscustomobject]@{
|
|
file = $failedPath
|
|
ok = $false
|
|
reason = "invalid_json"
|
|
error = $_.Exception.Message
|
|
}
|
|
continue
|
|
}
|
|
|
|
$alertId = [string](Get-AgentField $alert "id" $file.BaseName)
|
|
$actionable = Test-AgentAlertActionable $alert
|
|
if (-not $actionable) {
|
|
$ignoredPath = Join-Path $IgnoredDir ("ignored-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
Move-Item -Force $runningPath $ignoredPath
|
|
$ignored += [pscustomobject]@{
|
|
alertId = $alertId
|
|
file = $ignoredPath
|
|
reason = "not_actionable"
|
|
}
|
|
continue
|
|
}
|
|
|
|
$routeDecision = Resolve-AgentAlertRoute $alert
|
|
$modeName = [string]$routeDecision.mode
|
|
if ($modeName -notin $allowedModes) {
|
|
$failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
Move-Item -Force $runningPath $failedPath
|
|
$failed += [pscustomobject]@{
|
|
alertId = $alertId
|
|
file = $failedPath
|
|
ok = $false
|
|
reason = "mode_not_allowlisted"
|
|
mode = $modeName
|
|
}
|
|
continue
|
|
}
|
|
|
|
$correlationKey = Get-AgentAlertCorrelationKey $alert
|
|
$severity = [string](Get-AgentField $alert "severity" "unknown")
|
|
$existingDedupe = @($dedupeEntries | Where-Object { [string]$_.key -eq $correlationKey } | Select-Object -First 1)
|
|
$dedupeDecision = "new_dispatch"
|
|
if (@($existingDedupe).Count -gt 0) {
|
|
$previous = $existingDedupe[0]
|
|
$currentRank = Get-AgentAlertSeverityRank $severity
|
|
$previousRank = Get-AgentAlertSeverityRank ([string]$previous.severity)
|
|
if ($currentRank -le $previousRank) {
|
|
$occurrences = 1
|
|
if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrences = [int]$previous.occurrenceCount + 1 }
|
|
$dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey })
|
|
$dedupeEntries += [pscustomobject]@{
|
|
key = $correlationKey
|
|
alertId = [string]$previous.alertId
|
|
queueId = [string]$previous.queueId
|
|
severity = [string]$previous.severity
|
|
firstSeen = [string]$previous.firstSeen
|
|
lastSeen = (Get-Date -Format o)
|
|
occurrenceCount = $occurrences
|
|
lastDecision = "duplicate_single_flight_suppressed"
|
|
lastSuppressedAlertId = $alertId
|
|
}
|
|
Save-AgentAlertDedupeState $dedupeEntries
|
|
|
|
$ignoredPath = Join-Path $IgnoredDir ("duplicate-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
Move-Item -Force $runningPath $ignoredPath
|
|
$ignored += [pscustomobject]@{
|
|
alertId = $alertId
|
|
file = $ignoredPath
|
|
reason = "duplicate_single_flight"
|
|
correlationKey = $correlationKey
|
|
originalAlertId = [string]$previous.alertId
|
|
originalQueueId = [string]$previous.queueId
|
|
occurrenceCount = $occurrences
|
|
severity = $severity
|
|
originalSeverity = [string]$previous.severity
|
|
}
|
|
$duplicateSuppressedCount += 1
|
|
continue
|
|
}
|
|
$dedupeDecision = "severity_escalation_dispatch"
|
|
$severityEscalationCount += 1
|
|
}
|
|
|
|
$queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8))
|
|
$controlled = Get-AgentControlledApplyForAlert $alert $modeName
|
|
$instruction = New-AgentInstructionFromAlert $alert $alertId $modeName
|
|
$source = [string](Get-AgentField $alert "source" "sre-war-room")
|
|
$kind = [string](Get-AgentField $alert "kind" "unknown")
|
|
$service = [string](Get-AgentField $alert "service" "unknown")
|
|
$hostName = [string](Get-AgentField $alert "host" "")
|
|
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
|
|
$suppressTelegram = Test-AgentAlertSuppressTelegram $alert
|
|
$resolution = Get-AgentField $alert "resolution" $null
|
|
$sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required")
|
|
$sourceEventResolutionRequired = Convert-AgentBool (Get-AgentField $resolution "required" $true)
|
|
|
|
$requestPath = Join-Path $RequestDir "$queueId.json"
|
|
$queuePath = Join-Path $QueueDir "$queueId.json"
|
|
$processedPath = Join-Path $ProcessedDir ("processed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
|
|
[pscustomobject]@{
|
|
id = $queueId
|
|
instruction = $instruction
|
|
resolvedMode = $modeName
|
|
routeSource = [string]$routeDecision.source
|
|
routeMatched = [string]$routeDecision.matched
|
|
correlationKey = $correlationKey
|
|
dedupeDecision = $dedupeDecision
|
|
controlledApply = $controlled
|
|
requestedBy = $env:USERNAME
|
|
source = "agent99-sre-alert-inbox"
|
|
alertId = $alertId
|
|
alertSource = $source
|
|
alertKind = $kind
|
|
alertSeverity = $severity
|
|
alertService = $service
|
|
alertHost = $hostName
|
|
alertPath = $processedPath
|
|
suppressTelegram = $suppressTelegram
|
|
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
|
sourceEventResolutionRequired = $sourceEventResolutionRequired
|
|
createdAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8
|
|
|
|
[pscustomobject]@{
|
|
id = $queueId
|
|
mode = $modeName
|
|
routeSource = [string]$routeDecision.source
|
|
routeMatched = [string]$routeDecision.matched
|
|
correlationKey = $correlationKey
|
|
dedupeDecision = $dedupeDecision
|
|
controlledApply = $controlled
|
|
requestedBy = $env:USERNAME
|
|
source = "agent99-sre-alert-inbox"
|
|
reason = "sre_alert"
|
|
instruction = $instruction
|
|
requestPath = $requestPath
|
|
alertId = $alertId
|
|
alertSource = $source
|
|
alertKind = $kind
|
|
alertSeverity = $severity
|
|
alertService = $service
|
|
alertHost = $hostName
|
|
alertPath = $processedPath
|
|
replyChatId = $replyChatId
|
|
suppressTelegram = $suppressTelegram
|
|
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
|
sourceEventResolutionRequired = $sourceEventResolutionRequired
|
|
createdAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8
|
|
|
|
Move-Item -Force $runningPath $processedPath
|
|
$firstSeen = (Get-Date -Format o)
|
|
$occurrenceCount = 1
|
|
if (@($existingDedupe).Count -gt 0) {
|
|
$previous = $existingDedupe[0]
|
|
if ($previous.PSObject.Properties["firstSeen"] -and $previous.firstSeen) { $firstSeen = [string]$previous.firstSeen }
|
|
if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrenceCount = [int]$previous.occurrenceCount + 1 }
|
|
}
|
|
$dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey })
|
|
$dedupeEntries += [pscustomobject]@{
|
|
key = $correlationKey
|
|
alertId = $alertId
|
|
queueId = $queueId
|
|
severity = $severity
|
|
firstSeen = $firstSeen
|
|
lastSeen = (Get-Date -Format o)
|
|
occurrenceCount = $occurrenceCount
|
|
lastDecision = $dedupeDecision
|
|
mode = $modeName
|
|
}
|
|
Save-AgentAlertDedupeState $dedupeEntries
|
|
|
|
$queued += [pscustomobject]@{
|
|
alertId = $alertId
|
|
source = $source
|
|
severity = $severity
|
|
kind = $kind
|
|
service = $service
|
|
host = $hostName
|
|
mode = $modeName
|
|
routeSource = [string]$routeDecision.source
|
|
routeMatched = [string]$routeDecision.matched
|
|
correlationKey = $correlationKey
|
|
dedupeDecision = $dedupeDecision
|
|
controlledApply = $controlled
|
|
suppressTelegram = $suppressTelegram
|
|
sourceEventResolutionPolicy = $sourceEventResolutionPolicy
|
|
sourceEventResolutionRequired = $sourceEventResolutionRequired
|
|
requestPath = $requestPath
|
|
queuePath = $queuePath
|
|
alertPath = $processedPath
|
|
}
|
|
}
|
|
|
|
$controlExitCode = $null
|
|
if ($RunNow -and @($queued).Count -gt 0) {
|
|
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
|
|
$process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden
|
|
$process.Refresh()
|
|
$controlExitCode = $process.ExitCode
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = [bool](@($failed).Count -eq 0)
|
|
incomingPath = $IncomingDir
|
|
queuedCount = @($queued).Count
|
|
ignoredCount = @($ignored).Count
|
|
duplicateSuppressedCount = $duplicateSuppressedCount
|
|
severityEscalationCount = $severityEscalationCount
|
|
failedCount = @($failed).Count
|
|
queued = $queued
|
|
ignored = $ignored
|
|
failed = $failed
|
|
runNow = [bool]$RunNow
|
|
controlExitCode = $controlExitCode
|
|
}
|
|
|
|
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8
|
|
$result | ConvertTo-Json -Depth 10
|
|
if ($inboxLockStream) { $inboxLockStream.Dispose() }
|