param( [string]$AgentRoot = "C:\Wooo\Agent99", [switch]$RunNow, [switch]$SelfTest, [int]$MaxAlerts = 10 ) $ErrorActionPreference = "Stop" $EvidenceDir = Join-Path $AgentRoot "evidence" $AlertRoot = Join-Path $AgentRoot "alerts" $IncomingDir = Join-Path $AlertRoot "incoming" $RunningDir = Join-Path $AlertRoot "running" $ProcessedDir = Join-Path $AlertRoot "processed" $FailedDir = Join-Path $AlertRoot "failed" $IgnoredDir = Join-Path $AlertRoot "ignored" $QueueDir = Join-Path $AgentRoot "queue" $RequestDir = Join-Path $AgentRoot "requests" $StateDir = Join-Path $AgentRoot "state" $DedupeStatePath = Join-Path $StateDir "sre-alert-single-flight.json" $InboxLockPath = Join-Path $StateDir "sre-alert-inbox.lock" $DedupeWindowSeconds = 300 New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage") $remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed") function Get-AgentField { param( [object]$Object, [string]$Name, [object]$Default = $null ) if ($Object -and $Object.PSObject.Properties[$Name]) { return $Object.PSObject.Properties[$Name].Value } return $Default } function Convert-AgentBool { param([object]$Value) if ($null -eq $Value) { return $false } if ($Value -is [bool]) { return [bool]$Value } $text = ([string]$Value).Trim() if (-not $text) { return $false } return [bool]($text -match "^(?i:true|1|yes|y|on)$") } function Convert-AgentSafeFileToken { param([string]$Value) $safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_") if (-not $safe) { return "alert" } if ($safe.Length -gt 80) { return $safe.Substring(0, 80) } return $safe } function Get-AgentAlertText { param([object]$Alert) $labels = @() $rawLabels = Get-AgentField $Alert "labels" @() foreach ($label in @($rawLabels)) { if ($label -ne $null) { $labels += [string]$label } } @( (Get-AgentField $Alert "source" ""), (Get-AgentField $Alert "severity" ""), (Get-AgentField $Alert "kind" ""), (Get-AgentField $Alert "service" ""), (Get-AgentField $Alert "host" ""), (Get-AgentField $Alert "title" ""), (Get-AgentField $Alert "message" ""), ($labels -join " ") ) -join " " } function Test-AgentAlertActionable { param([object]$Alert) $force = Convert-AgentBool (Get-AgentField $Alert "force" $false) if ($force) { return $true } $text = (Get-AgentAlertText $Alert).ToLowerInvariant() $severity = ([string](Get-AgentField $Alert "severity" "")).ToLowerInvariant() if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) { return $true } if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|cold[-_ ]start|reboot[-_ ]auto[-_ ]recovery|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") { return $true } return $false } function Resolve-AgentAlertRoute { param([object]$Alert) $text = (Get-AgentAlertText $Alert).ToLowerInvariant() $suggestedMode = ([string](Get-AgentField $Alert "suggestedMode" "")).Trim() $kind = ([string](Get-AgentField $Alert "kind" "")).Trim().ToLowerInvariant() if ($suggestedMode -and $suggestedMode -in $allowedModes) { return [pscustomobject]@{ mode = $suggestedMode; source = "suggestedMode"; matched = $suggestedMode } } $kindModes = @{ "provider_freshness_signal" = "ProviderFreshness" "backup_health" = "BackupCheck" "backup_missing" = "BackupCheck" "backup_failed" = "BackupCheck" "security_health" = "SecurityTriage" "wazuh_security_alert" = "SecurityTriage" "public_route_502" = "Recover" "host_recovery" = "Recover" "host_down" = "Recover" "host_reboot" = "Recover" "vm_not_running" = "Recover" "host112_guest_recovery" = "Recover" "host112_guest_not_ready" = "Recover" "host112_guest_readback_missing" = "Recover" "performance_pressure" = "Perf" "performance_cpu" = "Perf" "performance_memory" = "Perf" "performance_disk" = "Perf" "harbor_registry" = "HarborRepair" "awoooi_k3s" = "AwoooRepair" "database_health" = "Status" "alert_chain_health" = "Status" "application_ai_config_error" = "Status" "repository_missing" = "Status" "cicd_failure" = "Status" } if ($kind -and $kindModes.ContainsKey($kind)) { return [pscustomobject]@{ mode = [string]$kindModes[$kind]; source = "kind"; matched = $kind } } if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" } } if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup|host112_guest|host 112.*(?:guest|console|lightdm)") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" } } if ($text -match "backup|snapshot|restore drill|cron") { return [pscustomobject]@{ mode = "BackupCheck"; source = "text_fallback"; matched = "backup" } } if ($text -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|privilege|vulnerability|cve|ioc|suspicious|unauthorized") { return [pscustomobject]@{ mode = "SecurityTriage"; source = "text_fallback"; matched = "security" } } if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { return [pscustomobject]@{ mode = "ProviderFreshness"; source = "text_fallback"; matched = "provider_freshness" } } if ($text -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag|alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "diagnostic_only" } } if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") { return [pscustomobject]@{ mode = "Perf"; source = "text_fallback"; matched = "performance" } } if ($text -match "harbor|registry") { return [pscustomobject]@{ mode = "HarborRepair"; source = "text_fallback"; matched = "harbor" } } if ($text -match "ci/?cd|cicd|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "cicd_failure" } } if ($text -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return [pscustomobject]@{ mode = "AwoooRepair"; source = "text_fallback"; matched = "k3s" } } if ($text -match "missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") { return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "application_config" } } if ($text -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "repository" } } return [pscustomobject]@{ mode = "Status"; source = "default"; matched = "none" } } function Resolve-AgentAlertMode { param([object]$Alert) $route = Resolve-AgentAlertRoute $Alert return [string]$route.mode } function Get-AgentAlertCorrelationKey { param([object]$Alert) $awoooi = Get-AgentField $Alert "awoooi" $null $dispatchIdentity = Get-AgentField $awoooi "agent99DispatchIdentity" $null $stableSingleFlight = [string](Get-AgentField $dispatchIdentity "single_flight_key" "") if ($stableSingleFlight) { return "controlled:$($stableSingleFlight.ToLowerInvariant())" } $routing = Get-AgentField $Alert "routing" $null $correlationKey = [string](Get-AgentField $routing "correlationKey" "") if ($correlationKey) { return "route:$($correlationKey.ToLowerInvariant())" } $fingerprint = [string](Get-AgentField $awoooi "fingerprint" "") if (-not $fingerprint) { $fingerprint = [string](Get-AgentField $Alert "fingerprint" "") } if ($fingerprint) { return "fingerprint:$($fingerprint.ToLowerInvariant())" } $kind = ([string](Get-AgentField $Alert "kind" "monitoring_alert")).ToLowerInvariant() $service = ([string](Get-AgentField $Alert "service" "unknown")).ToLowerInvariant() $hostName = ([string](Get-AgentField $Alert "host" "no-host")).ToLowerInvariant() return "fallback:$kind`:$service`:$hostName" } function Get-AgentAlertSeverityRank { param([string]$Severity) switch ($Severity.ToLowerInvariant()) { "critical" { return 4 } "error" { return 4 } "failed" { return 4 } "warning" { return 3 } "degraded" { return 3 } "info" { return 2 } default { return 1 } } } function Save-AgentAlertDedupeState { param([object[]]$Entries) $payload = [pscustomobject]@{ schemaVersion = "agent99_sre_alert_single_flight_v1" generatedAt = (Get-Date -Format o) windowSeconds = $DedupeWindowSeconds entries = @($Entries) } $temporaryPath = "$DedupeStatePath.tmp-$PID" $payload | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8 Move-Item -Force $temporaryPath $DedupeStatePath } function New-AgentInstructionFromAlert { param( [object]$Alert, [string]$AlertId, [string]$ModeName ) $existing = [string](Get-AgentField $Alert "instruction" "") if ($existing) { return $existing } $source = [string](Get-AgentField $Alert "source" "sre-war-room") $severity = [string](Get-AgentField $Alert "severity" "unknown") $kind = [string](Get-AgentField $Alert "kind" "unknown") $service = [string](Get-AgentField $Alert "service" "unknown") $hostName = [string](Get-AgentField $Alert "host" "") $title = [string](Get-AgentField $Alert "title" "") $message = [string](Get-AgentField $Alert "message" "") if ($kind -eq "provider_freshness_signal" -or $message.ToLowerInvariant() -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|provider freshness|provider window|last_seen|last seen|ingestion") { return "Provider freshness candidate required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; require providerWindow,lastSeen,directReference,verifierReadback; mark missing,expired,timeout,no_false_green; do not switch provider, call paid model, edit env, or reload." } if ($ModeName -eq "SecurityTriage" -or $kind -match "security|wazuh|siem") { return "Security triage required; alertId=$AlertId; source=$source; severity=$severity; service=$service; mode=$ModeName; classify blast radius, affected host/service, source log reference, repeated count, and recommended allowlisted next action; do not run arbitrary shell, delete data, disable controls, or expose secrets." } $parts = @("SRE alert", "id=$AlertId", "source=$source", "severity=$severity", "service=$service", "kind=$kind", "mode=$ModeName") if ($hostName) { $parts += "host=$hostName" } if ($title) { $parts += "title=$title" } if ($message) { $parts += "message=$message" } return ($parts -join "; ") } function Get-AgentControlledApplyForAlert { param( [object]$Alert, [string]$ModeName ) if ($Alert -and $Alert.PSObject.Properties["controlledApply"]) { return [bool]$Alert.controlledApply } return [bool]($ModeName -in $remediationModes) } function Test-AgentAlertSuppressTelegram { param([object]$Alert) if (Convert-AgentBool (Get-AgentField $Alert "suppressTelegram" $false)) { return $true } $text = (Get-AgentAlertText $Alert) if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true } return $false } function Invoke-AgentAlertRoutingSelfTest { $cases = @( [pscustomobject]@{ name = "public route 502" expectedActionable = $true expectedMode = "Recover" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-public-502" source = "agent99-routing-selftest" severity = "critical" kind = "public_route_502" service = "awoooi" host = "192.168.0.120" message = "HTTP 502 on public route" } }, [pscustomobject]@{ name = "host down or reboot" expectedActionable = $true expectedMode = "Recover" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-host-down" source = "agent99-routing-selftest" severity = "critical" kind = "host_down" service = "vmware" host = "192.168.0.110" message = "host_reboot detected vm_not_running" } }, [pscustomobject]@{ name = "host 112 guest recovery" expectedActionable = $true expectedMode = "Recover" expectedRouteSource = "kind" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-host112-guest" source = "agent99-routing-selftest" severity = "critical" kind = "host112_guest_recovery" service = "host112-guest-recovery" host = "192.168.0.112" message = "Host 112 Console LightDM VMware Tools and Wazuh not ready" } }, [pscustomobject]@{ name = "performance pressure" expectedActionable = $true expectedMode = "Perf" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-performance" source = "agent99-routing-selftest" severity = "critical" kind = "performance_pressure" service = "reboot-auto-recovery-slo" host = "110" message = "load_per_core critical cpu pressure disk_used high" } }, [pscustomobject]@{ name = "backup freshness" expectedActionable = $true expectedMode = "BackupCheck" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-backup" source = "agent99-routing-selftest" severity = "warning" kind = "backup_missing" service = "backup" message = "backup_failed snapshot cron freshness missing" } }, [pscustomobject]@{ name = "harbor registry" expectedActionable = $true expectedMode = "HarborRepair" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-harbor" source = "agent99-routing-selftest" severity = "critical" kind = "registry_unhealthy" service = "harbor" host = "192.168.0.110" message = "harbor registry unhealthy" } }, [pscustomobject]@{ name = "k3s pod failure" expectedActionable = $true expectedMode = "AwoooRepair" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-k3s" source = "agent99-routing-selftest" severity = "critical" kind = "pod_unhealthy" service = "awoooi" host = "192.168.0.120" message = "k3s pod crashloopbackoff imagepullbackoff deployment failure" } }, [pscustomobject]@{ name = "reboot slo wins over cpu contamination" expectedActionable = $true expectedMode = "Recover" expectedRouteSource = "suggestedMode" expectedControlledApply = $true alert = [pscustomobject]@{ id = "selftest-reboot-slo" source = "awoooi-api-alertmanager" severity = "critical" kind = "host_recovery" suggestedMode = "Recover" service = "reboot-auto-recovery-slo" host = "192.168.0.110" message = "all_required_hosts_not_in_10_minute_reboot_window cpu pressure" } }, [pscustomobject]@{ name = "postgres does not route to k3s" expectedActionable = $true expectedMode = "Status" expectedRouteSource = "kind" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-postgres" source = "awoooi-api-alertmanager" severity = "critical" kind = "database_health" service = "awoooi" message = "PostgreSQL database unavailable in deployment" } }, [pscustomobject]@{ name = "alert chain does not route to k3s" expectedActionable = $true expectedMode = "Status" expectedRouteSource = "kind" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-alert-chain" source = "awoooi-api-alertmanager" severity = "critical" kind = "alert_chain_health" service = "awoooi" message = "AlertChainBroken monitoring pipeline down" } }, [pscustomobject]@{ name = "provider freshness" expectedActionable = $true expectedMode = "ProviderFreshness" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-provider-freshness" source = "agent99-routing-selftest" severity = "warning" kind = "provider_freshness_signal" service = "provider_freshness_signal" message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate" } }, [pscustomobject]@{ name = "provider freshness do not page" expectedActionable = $true expectedMode = "ProviderFreshness" expectedControlledApply = $false expectedSuppressTelegram = $true alert = [pscustomobject]@{ id = "selftest-provider-freshness-do-not-page" source = "agent99-routing-selftest" severity = "warning" kind = "provider_freshness_signal" service = "provider_freshness_signal" message = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate" suppressTelegram = $true } }, [pscustomobject]@{ name = "security alert" expectedActionable = $true expectedMode = "SecurityTriage" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-security" source = "agent99-routing-selftest" severity = "critical" kind = "wazuh_security_alert" service = "iwooos-security" host = "192.168.0.112" message = "Wazuh SIEM security alert auth_failed brute force suspicious login" } }, [pscustomobject]@{ name = "gitea repository missing" expectedActionable = $true expectedMode = "Status" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-gitea-repo-missing" source = "agent99-routing-selftest" severity = "critical" kind = "repository_missing" service = "gitea" host = "192.168.0.110" message = "Gitea repo_missing repository missing for production project" } }, [pscustomobject]@{ name = "application ai config error" expectedActionable = $true expectedMode = "Status" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-app-ai-config" source = "agent99-routing-selftest" severity = "warning" kind = "application_ai_config_error" service = "bitan-pharmacy" host = "" message = "AI support locale guard failed E_AI_MISSING_KEY missing_key" } }, [pscustomobject]@{ name = "unknown warning" expectedActionable = $true expectedMode = "Status" expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-unknown-warning" source = "agent99-routing-selftest" severity = "warning" kind = "unknown" service = "unknown" message = "warning with no known remediation pattern" } }, [pscustomobject]@{ name = "info noise" expectedActionable = $false expectedMode = $null expectedControlledApply = $false alert = [pscustomobject]@{ id = "selftest-info-noise" source = "agent99-routing-selftest" severity = "info" kind = "heartbeat" service = "agent99" message = "normal heartbeat" } } ) $results = @() foreach ($case in $cases) { $actionable = Test-AgentAlertActionable $case.alert $route = if ($actionable) { Resolve-AgentAlertRoute $case.alert } else { $null } $mode = if ($route) { [string]$route.mode } else { $null } $routeSource = if ($route) { [string]$route.source } else { $null } $controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false } $suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false } $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false } $expectedRouteSource = if ($case.PSObject.Properties["expectedRouteSource"]) { [string]$case.expectedRouteSource } else { $null } $instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null } $ok = [bool]( $actionable -eq $case.expectedActionable -and [string]$mode -eq [string]$case.expectedMode -and (-not $expectedRouteSource -or $routeSource -eq $expectedRouteSource) -and $controlled -eq $case.expectedControlledApply -and $suppressTelegram -eq $expectedSuppressTelegram ) $results += [pscustomobject]@{ name = $case.name ok = $ok alertId = $case.alert.id actionable = $actionable expectedActionable = $case.expectedActionable mode = $mode expectedMode = $case.expectedMode routeSource = $routeSource expectedRouteSource = $expectedRouteSource controlledApply = $controlled expectedControlledApply = $case.expectedControlledApply suppressTelegram = $suppressTelegram expectedSuppressTelegram = $expectedSuppressTelegram instruction = $instruction } } $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = [bool](@($results | Where-Object { -not $_.ok }).Count -eq 0) caseCount = @($results).Count passedCount = @($results | Where-Object { $_.ok }).Count failedCount = @($results | Where-Object { -not $_.ok }).Count allowedModes = $allowedModes remediationModes = $remediationModes results = $results } $path = Join-Path $EvidenceDir ("agent99-SreAlertRoutingSelfTest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json") $result | Add-Member -MemberType NoteProperty -Name evidencePath -Value $path -Force $result | ConvertTo-Json -Depth 10 | Set-Content -Path $path -Encoding UTF8 $result } if ($SelfTest) { $selfTestResult = Invoke-AgentAlertRoutingSelfTest $selfTestResult | ConvertTo-Json -Depth 10 if (-not $selfTestResult.ok) { exit 1 } exit 0 } $inboxLockStream = $null try { $inboxLockStream = [System.IO.File]::Open( $InboxLockPath, [System.IO.FileMode]::OpenOrCreate, [System.IO.FileAccess]::ReadWrite, [System.IO.FileShare]::None ) } catch { $lockResult = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = $true skipped = $true reason = "single_flight_lock_held" lockPath = $InboxLockPath } $lockEvidencePath = Join-Path $EvidenceDir ("agent99-SreAlertInboxLock-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json") $lockResult | ConvertTo-Json -Depth 6 | Set-Content -Path $lockEvidencePath -Encoding UTF8 $lockResult | ConvertTo-Json -Depth 6 exit 0 } $dedupeEntries = @() if (Test-Path $DedupeStatePath) { try { $dedupePayload = Get-Content $DedupeStatePath -Raw | ConvertFrom-Json foreach ($entry in @($dedupePayload.entries)) { try { $ageSeconds = ((Get-Date).ToUniversalTime() - ([datetime]::Parse([string]$entry.lastSeen)).ToUniversalTime()).TotalSeconds if ($ageSeconds -ge 0 -and $ageSeconds -le $DedupeWindowSeconds) { $dedupeEntries += $entry } } catch { continue } } } catch { $dedupeEntries = @() } } $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json" $queued = @() $ignored = @() $failed = @() $duplicateSuppressedCount = 0 $severityEscalationCount = 0 $files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime | Select-Object -First $MaxAlerts) foreach ($file in $files) { $runningPath = Join-Path $RunningDir $file.Name Move-Item -Force $file.FullName $runningPath try { $alert = Get-Content $runningPath -Raw | ConvertFrom-Json } catch { $failedPath = Join-Path $FailedDir ("failed-" + $file.Name) Move-Item -Force $runningPath $failedPath $failed += [pscustomobject]@{ file = $failedPath ok = $false reason = "invalid_json" error = $_.Exception.Message } continue } $alertId = [string](Get-AgentField $alert "id" $file.BaseName) $actionable = Test-AgentAlertActionable $alert if (-not $actionable) { $ignoredPath = Join-Path $IgnoredDir ("ignored-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $ignoredPath $ignored += [pscustomobject]@{ alertId = $alertId file = $ignoredPath reason = "not_actionable" } continue } $routeDecision = Resolve-AgentAlertRoute $alert $modeName = [string]$routeDecision.mode if ($modeName -notin $allowedModes) { $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $failedPath $failed += [pscustomobject]@{ alertId = $alertId file = $failedPath ok = $false reason = "mode_not_allowlisted" mode = $modeName } continue } $correlationKey = Get-AgentAlertCorrelationKey $alert $severity = [string](Get-AgentField $alert "severity" "unknown") $existingDedupe = @($dedupeEntries | Where-Object { [string]$_.key -eq $correlationKey } | Select-Object -First 1) $dedupeDecision = "new_dispatch" if (@($existingDedupe).Count -gt 0) { $previous = $existingDedupe[0] $currentRank = Get-AgentAlertSeverityRank $severity $previousRank = Get-AgentAlertSeverityRank ([string]$previous.severity) if ($currentRank -le $previousRank) { $occurrences = 1 if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrences = [int]$previous.occurrenceCount + 1 } $dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey }) $dedupeEntries += [pscustomobject]@{ key = $correlationKey alertId = [string]$previous.alertId queueId = [string]$previous.queueId severity = [string]$previous.severity firstSeen = [string]$previous.firstSeen lastSeen = (Get-Date -Format o) occurrenceCount = $occurrences lastDecision = "duplicate_single_flight_suppressed" lastSuppressedAlertId = $alertId } Save-AgentAlertDedupeState $dedupeEntries $ignoredPath = Join-Path $IgnoredDir ("duplicate-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $ignoredPath $ignored += [pscustomobject]@{ alertId = $alertId file = $ignoredPath reason = "duplicate_single_flight" correlationKey = $correlationKey originalAlertId = [string]$previous.alertId originalQueueId = [string]$previous.queueId occurrenceCount = $occurrences severity = $severity originalSeverity = [string]$previous.severity } $duplicateSuppressedCount += 1 continue } $dedupeDecision = "severity_escalation_dispatch" $severityEscalationCount += 1 } $queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) $controlled = Get-AgentControlledApplyForAlert $alert $modeName $instruction = New-AgentInstructionFromAlert $alert $alertId $modeName $source = [string](Get-AgentField $alert "source" "sre-war-room") $kind = [string](Get-AgentField $alert "kind" "unknown") $service = [string](Get-AgentField $alert "service" "unknown") $hostName = [string](Get-AgentField $alert "host" "") $replyChatId = [string](Get-AgentField $alert "replyChatId" "") $replyMessageId = [string](Get-AgentField $alert "telegramMessageId" "") $suppressTelegram = Test-AgentAlertSuppressTelegram $alert $resolution = Get-AgentField $alert "resolution" $null $sourceEventResolutionPolicy = [string](Get-AgentField $resolution "policy" "external_source_receipt_required") $sourceEventResolutionRequired = Convert-AgentBool (Get-AgentField $resolution "required" $true) $routing = Get-AgentField $alert "routing" $null $awoooiMetadata = Get-AgentField $alert "awoooi" $null $dispatchIdentity = Get-AgentField $awoooiMetadata "agent99DispatchIdentity" $null $automationRunId = [string](Get-AgentField $dispatchIdentity "run_id" (Get-AgentField $routing "runId" "")) $traceId = [string](Get-AgentField $dispatchIdentity "trace_id" (Get-AgentField $routing "traceId" "")) $workItemId = [string](Get-AgentField $dispatchIdentity "work_item_id" (Get-AgentField $routing "workItemId" "")) $idempotencyKey = [string](Get-AgentField $dispatchIdentity "idempotency_key" (Get-AgentField $routing "idempotencyKey" "")) $executionGeneration = [string](Get-AgentField $dispatchIdentity "execution_generation" (Get-AgentField $routing "executionGeneration" "1")) $incidentId = [string](Get-AgentField $dispatchIdentity "incident_id" (Get-AgentField $awoooiMetadata "incidentId" "")) $approvalId = [string](Get-AgentField $dispatchIdentity "approval_id" (Get-AgentField $awoooiMetadata "approvalId" "")) $projectId = [string](Get-AgentField $dispatchIdentity "project_id" "") $sourceFingerprint = [string](Get-AgentField $dispatchIdentity "source_fingerprint" "") $dispatchRouteId = [string](Get-AgentField $dispatchIdentity "route_id" "") $singleFlightKey = [string](Get-AgentField $dispatchIdentity "single_flight_key" "") $lockOwner = [string](Get-AgentField $dispatchIdentity "lock_owner" "") $canonicalDigest = [string](Get-AgentField $dispatchIdentity "canonical_digest" "") $hasDispatchIdentity = [bool]($automationRunId -or $traceId -or $workItemId -or $idempotencyKey -or $incidentId -or $projectId -or $sourceFingerprint -or $dispatchRouteId -or $singleFlightKey) $dispatchIdentityIncomplete = [bool](-not $automationRunId -or -not $traceId -or -not $workItemId -or -not $idempotencyKey -or -not $incidentId -or -not $projectId -or -not $sourceFingerprint -or -not $dispatchRouteId -or -not $singleFlightKey -or -not $lockOwner -or -not $canonicalDigest -or -not $executionGeneration) if (($controlled -and $modeName -eq "Recover" -and -not $hasDispatchIdentity) -or ($hasDispatchIdentity -and $dispatchIdentityIncomplete)) { $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $failedPath $failed += [pscustomobject]@{ alertId = $alertId file = $failedPath ok = $false reason = "controlled_dispatch_identity_missing" mode = $modeName runtimeWritePerformed = $false } continue } $requestPath = Join-Path $RequestDir "$queueId.json" $queuePath = Join-Path $QueueDir "$queueId.json" $processedPath = Join-Path $ProcessedDir ("processed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) [pscustomobject]@{ id = $queueId instruction = $instruction resolvedMode = $modeName routeSource = [string]$routeDecision.source routeMatched = [string]$routeDecision.matched correlationKey = $correlationKey dedupeDecision = $dedupeDecision controlledApply = $controlled requestedBy = $env:USERNAME source = "agent99-sre-alert-inbox" alertId = $alertId alertSource = $source alertKind = $kind alertSeverity = $severity alertService = $service alertHost = $hostName replyMessageId = $replyMessageId alertPath = $processedPath suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired automationRunId = $automationRunId traceId = $traceId workItemId = $workItemId idempotencyKey = $idempotencyKey executionGeneration = $executionGeneration incidentId = $incidentId approvalId = $approvalId projectId = $projectId sourceFingerprint = $sourceFingerprint dispatchRouteId = $dispatchRouteId singleFlightKey = $singleFlightKey lockOwner = $lockOwner canonicalDigest = $canonicalDigest createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 [pscustomobject]@{ id = $queueId mode = $modeName routeSource = [string]$routeDecision.source routeMatched = [string]$routeDecision.matched correlationKey = $correlationKey dedupeDecision = $dedupeDecision controlledApply = $controlled requestedBy = $env:USERNAME source = "agent99-sre-alert-inbox" reason = "sre_alert" instruction = $instruction requestPath = $requestPath alertId = $alertId alertSource = $source alertKind = $kind alertSeverity = $severity alertService = $service alertHost = $hostName alertPath = $processedPath replyChatId = $replyChatId replyMessageId = $replyMessageId suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired automationRunId = $automationRunId traceId = $traceId workItemId = $workItemId idempotencyKey = $idempotencyKey executionGeneration = $executionGeneration incidentId = $incidentId approvalId = $approvalId projectId = $projectId sourceFingerprint = $sourceFingerprint dispatchRouteId = $dispatchRouteId singleFlightKey = $singleFlightKey lockOwner = $lockOwner canonicalDigest = $canonicalDigest createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 Move-Item -Force $runningPath $processedPath $firstSeen = (Get-Date -Format o) $occurrenceCount = 1 if (@($existingDedupe).Count -gt 0) { $previous = $existingDedupe[0] if ($previous.PSObject.Properties["firstSeen"] -and $previous.firstSeen) { $firstSeen = [string]$previous.firstSeen } if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrenceCount = [int]$previous.occurrenceCount + 1 } } $dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey }) $dedupeEntries += [pscustomobject]@{ key = $correlationKey alertId = $alertId queueId = $queueId severity = $severity firstSeen = $firstSeen lastSeen = (Get-Date -Format o) occurrenceCount = $occurrenceCount lastDecision = $dedupeDecision mode = $modeName } Save-AgentAlertDedupeState $dedupeEntries $queued += [pscustomobject]@{ alertId = $alertId source = $source severity = $severity kind = $kind service = $service host = $hostName mode = $modeName routeSource = [string]$routeDecision.source routeMatched = [string]$routeDecision.matched correlationKey = $correlationKey dedupeDecision = $dedupeDecision controlledApply = $controlled suppressTelegram = $suppressTelegram sourceEventResolutionPolicy = $sourceEventResolutionPolicy sourceEventResolutionRequired = $sourceEventResolutionRequired automationRunId = $automationRunId traceId = $traceId workItemId = $workItemId idempotencyKey = $idempotencyKey executionGeneration = $executionGeneration incidentId = $incidentId approvalId = $approvalId requestPath = $requestPath queuePath = $queuePath alertPath = $processedPath } } $controlExitCode = $null if ($RunNow -and @($queued).Count -gt 0) { $launcher = Join-Path $AgentRoot "agent99-run.ps1" $process = Start-Process -FilePath "powershell.exe" -ArgumentList @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", "ControlTick") -Wait -PassThru -WindowStyle Hidden $process.Refresh() $controlExitCode = $process.ExitCode } $result = [pscustomobject]@{ timestamp = (Get-Date -Format o) ok = [bool](@($failed).Count -eq 0) incomingPath = $IncomingDir queuedCount = @($queued).Count ignoredCount = @($ignored).Count duplicateSuppressedCount = $duplicateSuppressedCount severityEscalationCount = $severityEscalationCount failedCount = @($failed).Count queued = $queued ignored = $ignored failed = $failed runNow = [bool]$RunNow controlExitCode = $controlExitCode } $result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8 $result | ConvertTo-Json -Depth 10 if ($inboxLockStream) { $inboxLockStream.Dispose() }