Files
awoooi/agent99-sre-alert-inbox.ps1

581 lines
21 KiB
PowerShell

param(
[string]$AgentRoot = "C:\Wooo\Agent99",
[switch]$RunNow,
[switch]$SelfTest,
[int]$MaxAlerts = 10
)
$ErrorActionPreference = "Stop"
$EvidenceDir = Join-Path $AgentRoot "evidence"
$AlertRoot = Join-Path $AgentRoot "alerts"
$IncomingDir = Join-Path $AlertRoot "incoming"
$RunningDir = Join-Path $AlertRoot "running"
$ProcessedDir = Join-Path $AlertRoot "processed"
$FailedDir = Join-Path $AlertRoot "failed"
$IgnoredDir = Join-Path $AlertRoot "ignored"
$QueueDir = Join-Path $AgentRoot "queue"
$RequestDir = Join-Path $AgentRoot "requests"
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir | Out-Null
$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage")
$remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed")
function Get-AgentField {
param(
[object]$Object,
[string]$Name,
[object]$Default = $null
)
if ($Object -and $Object.PSObject.Properties[$Name]) {
return $Object.PSObject.Properties[$Name].Value
}
return $Default
}
function Convert-AgentBool {
param([object]$Value)
if ($null -eq $Value) { return $false }
if ($Value -is [bool]) { return [bool]$Value }
$text = ([string]$Value).Trim()
if (-not $text) { return $false }
return [bool]($text -match "^(?i:true|1|yes|y|on)$")
}
function Convert-AgentSafeFileToken {
param([string]$Value)
$safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
if (-not $safe) { return "alert" }
if ($safe.Length -gt 80) { return $safe.Substring(0, 80) }
return $safe
}
function Get-AgentAlertText {
param([object]$Alert)
$labels = @()
$rawLabels = Get-AgentField $Alert "labels" @()
foreach ($label in @($rawLabels)) {
if ($label -ne $null) { $labels += [string]$label }
}
@(
(Get-AgentField $Alert "source" ""),
(Get-AgentField $Alert "severity" ""),
(Get-AgentField $Alert "kind" ""),
(Get-AgentField $Alert "service" ""),
(Get-AgentField $Alert "host" ""),
(Get-AgentField $Alert "title" ""),
(Get-AgentField $Alert "message" ""),
($labels -join " ")
) -join " "
}
function Test-AgentAlertActionable {
param([object]$Alert)
$force = Convert-AgentBool (Get-AgentField $Alert "force" $false)
if ($force) { return $true }
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
$severity = ([string](Get-AgentField $Alert "severity" "")).ToLowerInvariant()
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
return $true
}
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
return $true
}
return $false
}
function Resolve-AgentAlertMode {
param([object]$Alert)
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
$suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "")
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
return "Recover"
}
if ($text -match "host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
return "Recover"
}
if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") {
return "Perf"
}
if ($text -match "backup|snapshot|restore drill|cron") {
return "BackupCheck"
}
if ($text -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|privilege|vulnerability|cve|ioc|suspicious|unauthorized") {
return "SecurityTriage"
}
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") {
return "ProviderFreshness"
}
if ($text -match "harbor|registry") {
return "HarborRepair"
}
if ($text -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") {
return "AwoooRepair"
}
if ($text -match "missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
return "Status"
}
if ($text -match "gitea|repo_missing|repository_missing|repo missing|repository missing") {
return "Status"
}
if ($suggestedMode -and $suggestedMode -in $allowedModes) {
return $suggestedMode
}
return "Status"
}
function New-AgentInstructionFromAlert {
param(
[object]$Alert,
[string]$AlertId,
[string]$ModeName
)
$existing = [string](Get-AgentField $Alert "instruction" "")
if ($existing) { return $existing }
$source = [string](Get-AgentField $Alert "source" "sre-war-room")
$severity = [string](Get-AgentField $Alert "severity" "unknown")
$kind = [string](Get-AgentField $Alert "kind" "unknown")
$service = [string](Get-AgentField $Alert "service" "unknown")
$hostName = [string](Get-AgentField $Alert "host" "")
$title = [string](Get-AgentField $Alert "title" "")
$message = [string](Get-AgentField $Alert "message" "")
if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|provider freshness|provider window|last_seen|last seen|ingestion") {
return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload."
}
if ($ModeName -eq "SecurityTriage" -or $kind -match "security|wazuh|siem") {
return "Security triage required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; classify blast radius, affected host/service, source log reference, repeated count, and recommended allowlisted next action; do not run arbitrary shell, delete data, disable controls, or expose secrets."
}
$parts = @("SRE alert", "id=$AlertId", "source=$source", "severity=$severity", "service=$service", "kind=$kind", "mode=$ModeName")
if ($hostName) { $parts += "host=$hostName" }
if ($title) { $parts += "title=$title" }
if ($message) { $parts += "message=$message" }
return ($parts -join "; ")
}
function Get-AgentControlledApplyForAlert {
param(
[object]$Alert,
[string]$ModeName
)
if ($Alert -and $Alert.PSObject.Properties["controlledApply"]) {
return [bool]$Alert.controlledApply
}
return [bool]($ModeName -in $remediationModes)
}
function Test-AgentAlertSuppressTelegram {
param([object]$Alert)
if (Convert-AgentBool (Get-AgentField $Alert "suppressTelegram" $false)) { return $true }
$text = (Get-AgentAlertText $Alert)
if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true }
return $false
}
function Invoke-AgentAlertRoutingSelfTest {
$cases = @(
[pscustomobject]@{
name = "public route 502"
expectedActionable = $true
expectedMode = "Recover"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-public-502"
source = "agent99-routing-selftest"
severity = "critical"
kind = "public_route_502"
service = "awoooi"
host = "192.168.0.120"
message = "HTTP 502 on public route"
}
},
[pscustomobject]@{
name = "host down or reboot"
expectedActionable = $true
expectedMode = "Recover"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-host-down"
source = "agent99-routing-selftest"
severity = "critical"
kind = "host_down"
service = "vmware"
host = "192.168.0.110"
message = "host_reboot detected vm_not_running"
}
},
[pscustomobject]@{
name = "performance pressure"
expectedActionable = $true
expectedMode = "Perf"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-performance"
source = "agent99-routing-selftest"
severity = "critical"
kind = "performance_pressure"
service = "reboot-auto-recovery-slo"
host = "110"
message = "load_per_core critical cpu pressure disk_used high"
}
},
[pscustomobject]@{
name = "backup freshness"
expectedActionable = $true
expectedMode = "BackupCheck"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-backup"
source = "agent99-routing-selftest"
severity = "warning"
kind = "backup_missing"
service = "backup"
message = "backup_failed snapshot cron freshness missing"
}
},
[pscustomobject]@{
name = "harbor registry"
expectedActionable = $true
expectedMode = "HarborRepair"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-harbor"
source = "agent99-routing-selftest"
severity = "critical"
kind = "registry_unhealthy"
service = "harbor"
host = "192.168.0.110"
message = "harbor registry unhealthy"
}
},
[pscustomobject]@{
name = "k3s pod failure"
expectedActionable = $true
expectedMode = "AwoooRepair"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-k3s"
source = "agent99-routing-selftest"
severity = "critical"
kind = "pod_unhealthy"
service = "awoooi"
host = "192.168.0.120"
message = "k3s pod crashloopbackoff imagepullbackoff deployment failure"
}
},
[pscustomobject]@{
name = "provider freshness"
expectedActionable = $true
expectedMode = "ProviderFreshness"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-provider-freshness"
source = "agent99-routing-selftest"
severity = "warning"
kind = "provider_freshness_signal"
service = "provider_freshness_signal"
message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
}
},
[pscustomobject]@{
name = "provider freshness do not page"
expectedActionable = $true
expectedMode = "ProviderFreshness"
expectedControlledApply = $false
expectedSuppressTelegram = $true
alert = [pscustomobject]@{
id = "selftest-provider-freshness-do-not-page"
source = "agent99-routing-selftest"
severity = "warning"
kind = "provider_freshness_signal"
service = "provider_freshness_signal"
message = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
suppressTelegram = $true
}
},
[pscustomobject]@{
name = "security alert"
expectedActionable = $true
expectedMode = "SecurityTriage"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-security"
source = "agent99-routing-selftest"
severity = "critical"
kind = "wazuh_security_alert"
service = "iwooos-security"
host = "192.168.0.112"
message = "Wazuh SIEM security alert auth_failed brute force suspicious login"
}
},
[pscustomobject]@{
name = "gitea repository missing"
expectedActionable = $true
expectedMode = "Status"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-gitea-repo-missing"
source = "agent99-routing-selftest"
severity = "critical"
kind = "repository_missing"
service = "gitea"
host = "192.168.0.110"
message = "Gitea repo_missing repository missing for production project"
}
},
[pscustomobject]@{
name = "application ai config error"
expectedActionable = $true
expectedMode = "Status"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-app-ai-config"
source = "agent99-routing-selftest"
severity = "warning"
kind = "application_ai_config_error"
service = "bitan-pharmacy"
host = ""
message = "AI support locale guard failed E_AI_MISSING_KEY missing_key"
}
},
[pscustomobject]@{
name = "unknown warning"
expectedActionable = $true
expectedMode = "Status"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-unknown-warning"
source = "agent99-routing-selftest"
severity = "warning"
kind = "unknown"
service = "unknown"
message = "warning with no known remediation pattern"
}
},
[pscustomobject]@{
name = "info noise"
expectedActionable = $false
expectedMode = $null
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-info-noise"
source = "agent99-routing-selftest"
severity = "info"
kind = "heartbeat"
service = "agent99"
message = "normal heartbeat"
}
}
)
$results = @()
foreach ($case in $cases) {
$actionable = Test-AgentAlertActionable $case.alert
$mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null }
$controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false }
$suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false }
$expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
$instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null }
$ok = [bool](
$actionable -eq $case.expectedActionable -and
[string]$mode -eq [string]$case.expectedMode -and
$controlled -eq $case.expectedControlledApply -and
$suppressTelegram -eq $expectedSuppressTelegram
)
$results += [pscustomobject]@{
name = $case.name
ok = $ok
alertId = $case.alert.id
actionable = $actionable
expectedActionable = $case.expectedActionable
mode = $mode
expectedMode = $case.expectedMode
controlledApply = $controlled
expectedControlledApply = $case.expectedControlledApply
suppressTelegram = $suppressTelegram
expectedSuppressTelegram = $expectedSuppressTelegram
instruction = $instruction
}
}
$result = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = [bool](@($results | Where-Object { -not $_.ok }).Count -eq 0)
caseCount = @($results).Count
passedCount = @($results | Where-Object { $_.ok }).Count
failedCount = @($results | Where-Object { -not $_.ok }).Count
allowedModes = $allowedModes
remediationModes = $remediationModes
results = $results
}
$path = Join-Path $EvidenceDir ("agent99-SreAlertRoutingSelfTest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
$result | Add-Member -MemberType NoteProperty -Name evidencePath -Value $path -Force
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $path -Encoding UTF8
$result
}
if ($SelfTest) {
$selfTestResult = Invoke-AgentAlertRoutingSelfTest
$selfTestResult | ConvertTo-Json -Depth 10
if (-not $selfTestResult.ok) { exit 1 }
exit 0
}
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json"
$queued = @()
$ignored = @()
$failed = @()
$files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
Sort-Object LastWriteTime |
Select-Object -First $MaxAlerts)
foreach ($file in $files) {
$runningPath = Join-Path $RunningDir $file.Name
Move-Item -Force $file.FullName $runningPath
try {
$alert = Get-Content $runningPath -Raw | ConvertFrom-Json
} catch {
$failedPath = Join-Path $FailedDir ("failed-" + $file.Name)
Move-Item -Force $runningPath $failedPath
$failed += [pscustomobject]@{
file = $failedPath
ok = $false
reason = "invalid_json"
error = $_.Exception.Message
}
continue
}
$alertId = [string](Get-AgentField $alert "id" $file.BaseName)
$actionable = Test-AgentAlertActionable $alert
if (-not $actionable) {
$ignoredPath = Join-Path $IgnoredDir ("ignored-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
Move-Item -Force $runningPath $ignoredPath
$ignored += [pscustomobject]@{
alertId = $alertId
file = $ignoredPath
reason = "not_actionable"
}
continue
}
$modeName = Resolve-AgentAlertMode $alert
if ($modeName -notin $allowedModes) {
$failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
Move-Item -Force $runningPath $failedPath
$failed += [pscustomobject]@{
alertId = $alertId
file = $failedPath
ok = $false
reason = "mode_not_allowlisted"
mode = $modeName
}
continue
}
$queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8))
$controlled = Get-AgentControlledApplyForAlert $alert $modeName
$instruction = New-AgentInstructionFromAlert $alert $alertId $modeName
$source = [string](Get-AgentField $alert "source" "sre-war-room")
$kind = [string](Get-AgentField $alert "kind" "unknown")
$severity = [string](Get-AgentField $alert "severity" "unknown")
$service = [string](Get-AgentField $alert "service" "unknown")
$hostName = [string](Get-AgentField $alert "host" "")
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
$suppressTelegram = Test-AgentAlertSuppressTelegram $alert
$requestPath = Join-Path $RequestDir "$queueId.json"
$queuePath = Join-Path $QueueDir "$queueId.json"
$processedPath = Join-Path $ProcessedDir ("processed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
[pscustomobject]@{
id = $queueId
instruction = $instruction
resolvedMode = $modeName
controlledApply = $controlled
requestedBy = $env:USERNAME
source = "agent99-sre-alert-inbox"
alertId = $alertId
alertSource = $source
alertKind = $kind
alertSeverity = $severity
alertService = $service
alertHost = $hostName
alertPath = $processedPath
suppressTelegram = $suppressTelegram
createdAt = (Get-Date -Format o)
} | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8
[pscustomobject]@{
id = $queueId
mode = $modeName
controlledApply = $controlled
requestedBy = $env:USERNAME
source = "agent99-sre-alert-inbox"
reason = "sre_alert"
instruction = $instruction
requestPath = $requestPath
alertId = $alertId
alertSource = $source
alertKind = $kind
alertSeverity = $severity
alertService = $service
alertHost = $hostName
alertPath = $processedPath
replyChatId = $replyChatId
suppressTelegram = $suppressTelegram
createdAt = (Get-Date -Format o)
} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8
Move-Item -Force $runningPath $processedPath
$queued += [pscustomobject]@{
alertId = $alertId
source = $source
severity = $severity
kind = $kind
service = $service
host = $hostName
mode = $modeName
controlledApply = $controlled
suppressTelegram = $suppressTelegram
requestPath = $requestPath
queuePath = $queuePath
alertPath = $processedPath
}
}
$controlExitCode = $null
if ($RunNow -and @($queued).Count -gt 0) {
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
$process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden
$process.Refresh()
$controlExitCode = $process.ExitCode
}
$result = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = [bool](@($failed).Count -eq 0)
incomingPath = $IncomingDir
queuedCount = @($queued).Count
ignoredCount = @($ignored).Count
failedCount = @($failed).Count
queued = $queued
ignored = $ignored
failed = $failed
runNow = [bool]$RunNow
controlExitCode = $controlExitCode
}
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8
$result | ConvertTo-Json -Depth 10