298 lines
10 KiB
PowerShell
298 lines
10 KiB
PowerShell
param(
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[switch]$RunNow,
|
|
[int]$MaxAlerts = 10
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
$EvidenceDir = Join-Path $AgentRoot "evidence"
|
|
$AlertRoot = Join-Path $AgentRoot "alerts"
|
|
$IncomingDir = Join-Path $AlertRoot "incoming"
|
|
$RunningDir = Join-Path $AlertRoot "running"
|
|
$ProcessedDir = Join-Path $AlertRoot "processed"
|
|
$FailedDir = Join-Path $AlertRoot "failed"
|
|
$IgnoredDir = Join-Path $AlertRoot "ignored"
|
|
$QueueDir = Join-Path $AgentRoot "queue"
|
|
$RequestDir = Join-Path $AgentRoot "requests"
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir | Out-Null
|
|
|
|
$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness")
|
|
$remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed")
|
|
|
|
function Get-AgentField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[object]$Default = $null
|
|
)
|
|
if ($Object -and $Object.PSObject.Properties[$Name]) {
|
|
return $Object.PSObject.Properties[$Name].Value
|
|
}
|
|
return $Default
|
|
}
|
|
|
|
function Convert-AgentSafeFileToken {
|
|
param([string]$Value)
|
|
$safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
|
|
if (-not $safe) { return "alert" }
|
|
if ($safe.Length -gt 80) { return $safe.Substring(0, 80) }
|
|
return $safe
|
|
}
|
|
|
|
function Get-AgentAlertText {
|
|
param([object]$Alert)
|
|
$labels = @()
|
|
$rawLabels = Get-AgentField $Alert "labels" @()
|
|
foreach ($label in @($rawLabels)) {
|
|
if ($label -ne $null) { $labels += [string]$label }
|
|
}
|
|
@(
|
|
(Get-AgentField $Alert "source" ""),
|
|
(Get-AgentField $Alert "severity" ""),
|
|
(Get-AgentField $Alert "kind" ""),
|
|
(Get-AgentField $Alert "service" ""),
|
|
(Get-AgentField $Alert "host" ""),
|
|
(Get-AgentField $Alert "title" ""),
|
|
(Get-AgentField $Alert "message" ""),
|
|
($labels -join " ")
|
|
) -join " "
|
|
}
|
|
|
|
function Test-AgentAlertActionable {
|
|
param([object]$Alert)
|
|
$force = [bool](Get-AgentField $Alert "force" $false)
|
|
if ($force) { return $true }
|
|
|
|
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
|
|
$severity = ([string](Get-AgentField $Alert "severity" "")).ToLowerInvariant()
|
|
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
|
|
return $true
|
|
}
|
|
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s") {
|
|
return $true
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Resolve-AgentAlertMode {
|
|
param([object]$Alert)
|
|
|
|
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
|
|
$suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "")
|
|
|
|
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion") {
|
|
return "ProviderFreshness"
|
|
}
|
|
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
|
|
return "Recover"
|
|
}
|
|
if ($text -match "host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
|
|
return "Recover"
|
|
}
|
|
if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") {
|
|
return "Perf"
|
|
}
|
|
if ($text -match "backup|snapshot|restore drill|cron") {
|
|
return "BackupCheck"
|
|
}
|
|
if ($text -match "harbor|registry") {
|
|
return "HarborRepair"
|
|
}
|
|
if ($text -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") {
|
|
return "AwoooRepair"
|
|
}
|
|
if ($suggestedMode -and $suggestedMode -in $allowedModes) {
|
|
return $suggestedMode
|
|
}
|
|
return "Status"
|
|
}
|
|
|
|
function New-AgentInstructionFromAlert {
|
|
param(
|
|
[object]$Alert,
|
|
[string]$AlertId,
|
|
[string]$ModeName
|
|
)
|
|
|
|
$existing = [string](Get-AgentField $Alert "instruction" "")
|
|
if ($existing) { return $existing }
|
|
|
|
$source = [string](Get-AgentField $Alert "source" "sre-war-room")
|
|
$severity = [string](Get-AgentField $Alert "severity" "unknown")
|
|
$kind = [string](Get-AgentField $Alert "kind" "unknown")
|
|
$service = [string](Get-AgentField $Alert "service" "unknown")
|
|
$hostName = [string](Get-AgentField $Alert "host" "")
|
|
$title = [string](Get-AgentField $Alert "title" "")
|
|
$message = [string](Get-AgentField $Alert "message" "")
|
|
|
|
if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness") {
|
|
return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload."
|
|
}
|
|
|
|
$parts = @("SRE alert", "id=$AlertId", "source=$source", "severity=$severity", "service=$service", "kind=$kind", "mode=$ModeName")
|
|
if ($hostName) { $parts += "host=$hostName" }
|
|
if ($title) { $parts += "title=$title" }
|
|
if ($message) { $parts += "message=$message" }
|
|
return ($parts -join "; ")
|
|
}
|
|
|
|
function Get-AgentControlledApplyForAlert {
|
|
param(
|
|
[object]$Alert,
|
|
[string]$ModeName
|
|
)
|
|
if ($Alert -and $Alert.PSObject.Properties["controlledApply"]) {
|
|
return [bool]$Alert.controlledApply
|
|
}
|
|
return [bool]($ModeName -in $remediationModes)
|
|
}
|
|
|
|
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
|
|
$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json"
|
|
$queued = @()
|
|
$ignored = @()
|
|
$failed = @()
|
|
|
|
$files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime |
|
|
Select-Object -First $MaxAlerts)
|
|
|
|
foreach ($file in $files) {
|
|
$runningPath = Join-Path $RunningDir $file.Name
|
|
Move-Item -Force $file.FullName $runningPath
|
|
|
|
try {
|
|
$alert = Get-Content $runningPath -Raw | ConvertFrom-Json
|
|
} catch {
|
|
$failedPath = Join-Path $FailedDir ("failed-" + $file.Name)
|
|
Move-Item -Force $runningPath $failedPath
|
|
$failed += [pscustomobject]@{
|
|
file = $failedPath
|
|
ok = $false
|
|
reason = "invalid_json"
|
|
error = $_.Exception.Message
|
|
}
|
|
continue
|
|
}
|
|
|
|
$alertId = [string](Get-AgentField $alert "id" $file.BaseName)
|
|
$actionable = Test-AgentAlertActionable $alert
|
|
if (-not $actionable) {
|
|
$ignoredPath = Join-Path $IgnoredDir ("ignored-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
Move-Item -Force $runningPath $ignoredPath
|
|
$ignored += [pscustomobject]@{
|
|
alertId = $alertId
|
|
file = $ignoredPath
|
|
reason = "not_actionable"
|
|
}
|
|
continue
|
|
}
|
|
|
|
$modeName = Resolve-AgentAlertMode $alert
|
|
if ($modeName -notin $allowedModes) {
|
|
$failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
Move-Item -Force $runningPath $failedPath
|
|
$failed += [pscustomobject]@{
|
|
alertId = $alertId
|
|
file = $failedPath
|
|
ok = $false
|
|
reason = "mode_not_allowlisted"
|
|
mode = $modeName
|
|
}
|
|
continue
|
|
}
|
|
|
|
$queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8))
|
|
$controlled = Get-AgentControlledApplyForAlert $alert $modeName
|
|
$instruction = New-AgentInstructionFromAlert $alert $alertId $modeName
|
|
$source = [string](Get-AgentField $alert "source" "sre-war-room")
|
|
$kind = [string](Get-AgentField $alert "kind" "unknown")
|
|
$severity = [string](Get-AgentField $alert "severity" "unknown")
|
|
$service = [string](Get-AgentField $alert "service" "unknown")
|
|
$hostName = [string](Get-AgentField $alert "host" "")
|
|
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
|
|
|
|
$requestPath = Join-Path $RequestDir "$queueId.json"
|
|
$queuePath = Join-Path $QueueDir "$queueId.json"
|
|
$processedPath = Join-Path $ProcessedDir ("processed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
|
|
|
|
[pscustomobject]@{
|
|
id = $queueId
|
|
instruction = $instruction
|
|
resolvedMode = $modeName
|
|
controlledApply = $controlled
|
|
requestedBy = $env:USERNAME
|
|
source = "agent99-sre-alert-inbox"
|
|
alertId = $alertId
|
|
alertSource = $source
|
|
alertKind = $kind
|
|
alertSeverity = $severity
|
|
alertService = $service
|
|
alertHost = $hostName
|
|
alertPath = $processedPath
|
|
createdAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8
|
|
|
|
[pscustomobject]@{
|
|
id = $queueId
|
|
mode = $modeName
|
|
controlledApply = $controlled
|
|
requestedBy = $env:USERNAME
|
|
source = "agent99-sre-alert-inbox"
|
|
reason = "sre_alert"
|
|
instruction = $instruction
|
|
requestPath = $requestPath
|
|
alertId = $alertId
|
|
alertSource = $source
|
|
alertKind = $kind
|
|
alertSeverity = $severity
|
|
alertService = $service
|
|
alertHost = $hostName
|
|
alertPath = $processedPath
|
|
replyChatId = $replyChatId
|
|
createdAt = (Get-Date -Format o)
|
|
} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8
|
|
|
|
Move-Item -Force $runningPath $processedPath
|
|
$queued += [pscustomobject]@{
|
|
alertId = $alertId
|
|
source = $source
|
|
severity = $severity
|
|
kind = $kind
|
|
service = $service
|
|
host = $hostName
|
|
mode = $modeName
|
|
controlledApply = $controlled
|
|
requestPath = $requestPath
|
|
queuePath = $queuePath
|
|
alertPath = $processedPath
|
|
}
|
|
}
|
|
|
|
$controlExitCode = $null
|
|
if ($RunNow -and @($queued).Count -gt 0) {
|
|
$launcher = Join-Path $AgentRoot "agent99-run.ps1"
|
|
$process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden
|
|
$process.Refresh()
|
|
$controlExitCode = $process.ExitCode
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
timestamp = (Get-Date -Format o)
|
|
ok = [bool](@($failed).Count -eq 0)
|
|
incomingPath = $IncomingDir
|
|
queuedCount = @($queued).Count
|
|
ignoredCount = @($ignored).Count
|
|
failedCount = @($failed).Count
|
|
queued = $queued
|
|
ignored = $ignored
|
|
failed = $failed
|
|
runNow = [bool]$RunNow
|
|
controlExitCode = $controlExitCode
|
|
}
|
|
|
|
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8
|
|
$result | ConvertTo-Json -Depth 10
|