wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(agent99): route and dedupe alerts before dispatch

Browse Source
This commit is contained in:
ogt
2026-07-10 16:44:01 +08:00
parent ea98503a9a
commit 5c620844fe
7 changed files with 937 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

309
agent99-sre-alert-inbox.ps1
View File

@@ -16,8 +16,12 @@ $FailedDir = Join-Path $AlertRoot "failed"
$IgnoredDir = Join-Path $AlertRoot "ignored"
$QueueDir = Join-Path $AgentRoot "queue"
$RequestDir = Join-Path $AgentRoot "requests"
$StateDir = Join-Path $AgentRoot "state"
$DedupeStatePath = Join-Path $StateDir "sre-alert-single-flight.json"
$InboxLockPath = Join-Path $StateDir "sre-alert-inbox.lock"
$DedupeWindowSeconds = 300
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir | Out-Null
New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null
$allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage")
$remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed")
@@ -86,46 +90,128 @@ function Test-AgentAlertActionable {
return $false
}
function Resolve-AgentAlertMode {
function Resolve-AgentAlertRoute {
param([object]$Alert)
$text = (Get-AgentAlertText $Alert).ToLowerInvariant()
$suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "")
$suggestedMode = ([string](Get-AgentField $Alert "suggestedMode" "")).Trim()
$kind = ([string](Get-AgentField $Alert "kind" "")).Trim().ToLowerInvariant()
if ($suggestedMode -and $suggestedMode -in $allowedModes) {
return [pscustomobject]@{ mode = $suggestedMode; source = "suggestedMode"; matched = $suggestedMode }
}
$kindModes = @{
"provider_freshness_signal" = "ProviderFreshness"
"backup_health" = "BackupCheck"
"backup_missing" = "BackupCheck"
"backup_failed" = "BackupCheck"
"security_health" = "SecurityTriage"
"wazuh_security_alert" = "SecurityTriage"
"public_route_502" = "Recover"
"host_recovery" = "Recover"
"host_down" = "Recover"
"host_reboot" = "Recover"
"vm_not_running" = "Recover"
"performance_pressure" = "Perf"
"performance_cpu" = "Perf"
"performance_memory" = "Perf"
"performance_disk" = "Perf"
"harbor_registry" = "HarborRepair"
"awoooi_k3s" = "AwoooRepair"
"database_health" = "Status"
"alert_chain_health" = "Status"
"application_ai_config_error" = "Status"
"repository_missing" = "Status"
}
if ($kind -and $kindModes.ContainsKey($kind)) {
return [pscustomobject]@{ mode = [string]$kindModes[$kind]; source = "kind"; matched = $kind }
}
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
return "Recover"
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" }
}
if ($text -match "host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
return "Recover"
}
if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") {
return "Perf"
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" }
}
if ($text -match "backup|snapshot|restore drill|cron") {
return "BackupCheck"
return [pscustomobject]@{ mode = "BackupCheck"; source = "text_fallback"; matched = "backup" }
}
if ($text -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|privilege|vulnerability|cve|ioc|suspicious|unauthorized") {
return "SecurityTriage"
return [pscustomobject]@{ mode = "SecurityTriage"; source = "text_fallback"; matched = "security" }
}
if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") {
return "ProviderFreshness"
return [pscustomobject]@{ mode = "ProviderFreshness"; source = "text_fallback"; matched = "provider_freshness" }
}
if ($text -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag|alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") {
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "diagnostic_only" }
}
if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") {
return [pscustomobject]@{ mode = "Perf"; source = "text_fallback"; matched = "performance" }
}
if ($text -match "harbor|registry") {
return "HarborRepair"
return [pscustomobject]@{ mode = "HarborRepair"; source = "text_fallback"; matched = "harbor" }
}
if ($text -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") {
return "AwoooRepair"
if ($text -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") {
return [pscustomobject]@{ mode = "AwoooRepair"; source = "text_fallback"; matched = "k3s" }
}
if ($text -match "missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
return "Status"
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "application_config" }
}
if ($text -match "gitea|repo_missing|repository_missing|repo missing|repository missing") {
return "Status"
return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "repository" }
}
if ($suggestedMode -and $suggestedMode -in $allowedModes) {
return $suggestedMode
return [pscustomobject]@{ mode = "Status"; source = "default"; matched = "none" }
}
function Resolve-AgentAlertMode {
param([object]$Alert)
$route = Resolve-AgentAlertRoute $Alert
return [string]$route.mode
}
function Get-AgentAlertCorrelationKey {
param([object]$Alert)
$routing = Get-AgentField $Alert "routing" $null
$correlationKey = [string](Get-AgentField $routing "correlationKey" "")
if ($correlationKey) { return "route:$($correlationKey.ToLowerInvariant())" }
$awoooi = Get-AgentField $Alert "awoooi" $null
$fingerprint = [string](Get-AgentField $awoooi "fingerprint" "")
if (-not $fingerprint) { $fingerprint = [string](Get-AgentField $Alert "fingerprint" "") }
if ($fingerprint) { return "fingerprint:$($fingerprint.ToLowerInvariant())" }
$kind = ([string](Get-AgentField $Alert "kind" "monitoring_alert")).ToLowerInvariant()
$service = ([string](Get-AgentField $Alert "service" "unknown")).ToLowerInvariant()
$hostName = ([string](Get-AgentField $Alert "host" "no-host")).ToLowerInvariant()
return "fallback:$kind`:$service`:$hostName"
}
function Get-AgentAlertSeverityRank {
param([string]$Severity)
switch ($Severity.ToLowerInvariant()) {
"critical" { return 4 }
"error" { return 4 }
"failed" { return 4 }
"warning" { return 3 }
"degraded" { return 3 }
"info" { return 2 }
default { return 1 }
}
return "Status"
}
function Save-AgentAlertDedupeState {
param([object[]]$Entries)
$payload = [pscustomobject]@{
schemaVersion = "agent99_sre_alert_single_flight_v1"
generatedAt = (Get-Date -Format o)
windowSeconds = $DedupeWindowSeconds
entries = @($Entries)
}
$temporaryPath = "$DedupeStatePath.tmp-$PID"
$payload | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8
Move-Item -Force $temporaryPath $DedupeStatePath
}
function New-AgentInstructionFromAlert {
@@ -270,6 +356,53 @@ function Invoke-AgentAlertRoutingSelfTest {
message = "k3s pod crashloopbackoff imagepullbackoff deployment failure"
}
},
[pscustomobject]@{
name = "reboot slo wins over cpu contamination"
expectedActionable = $true
expectedMode = "Recover"
expectedRouteSource = "suggestedMode"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-reboot-slo"
source = "awoooi-api-alertmanager"
severity = "critical"
kind = "host_recovery"
suggestedMode = "Recover"
service = "reboot-auto-recovery-slo"
host = "192.168.0.110"
message = "all_required_hosts_not_in_10_minute_reboot_window cpu pressure"
}
},
[pscustomobject]@{
name = "postgres does not route to k3s"
expectedActionable = $true
expectedMode = "Status"
expectedRouteSource = "kind"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-postgres"
source = "awoooi-api-alertmanager"
severity = "critical"
kind = "database_health"
service = "awoooi"
message = "PostgreSQL database unavailable in deployment"
}
},
[pscustomobject]@{
name = "alert chain does not route to k3s"
expectedActionable = $true
expectedMode = "Status"
expectedRouteSource = "kind"
expectedControlledApply = $false
alert = [pscustomobject]@{
id = "selftest-alert-chain"
source = "awoooi-api-alertmanager"
severity = "critical"
kind = "alert_chain_health"
service = "awoooi"
message = "AlertChainBroken monitoring pipeline down"
}
},
[pscustomobject]@{
name = "provider freshness"
expectedActionable = $true
@@ -378,14 +511,18 @@ function Invoke-AgentAlertRoutingSelfTest {
$results = @()
foreach ($case in $cases) {
$actionable = Test-AgentAlertActionable $case.alert
$mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null }
$route = if ($actionable) { Resolve-AgentAlertRoute $case.alert } else { $null }
$mode = if ($route) { [string]$route.mode } else { $null }
$routeSource = if ($route) { [string]$route.source } else { $null }
$controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false }
$suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false }
$expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
$expectedRouteSource = if ($case.PSObject.Properties["expectedRouteSource"]) { [string]$case.expectedRouteSource } else { $null }
$instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null }
$ok = [bool](
$actionable -eq $case.expectedActionable -and
[string]$mode -eq [string]$case.expectedMode -and
(-not $expectedRouteSource -or $routeSource -eq $expectedRouteSource) -and
$controlled -eq $case.expectedControlledApply -and
$suppressTelegram -eq $expectedSuppressTelegram
)
@@ -397,6 +534,8 @@ function Invoke-AgentAlertRoutingSelfTest {
expectedActionable = $case.expectedActionable
mode = $mode
expectedMode = $case.expectedMode
routeSource = $routeSource
expectedRouteSource = $expectedRouteSource
controlledApply = $controlled
expectedControlledApply = $case.expectedControlledApply
suppressTelegram = $suppressTelegram
@@ -428,11 +567,54 @@ if ($SelfTest) {
exit 0
}
$inboxLockStream = $null
try {
$inboxLockStream = [System.IO.File]::Open(
$InboxLockPath,
[System.IO.FileMode]::OpenOrCreate,
[System.IO.FileAccess]::ReadWrite,
[System.IO.FileShare]::None
)
} catch {
$lockResult = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = $true
skipped = $true
reason = "single_flight_lock_held"
lockPath = $InboxLockPath
}
$lockEvidencePath = Join-Path $EvidenceDir ("agent99-SreAlertInboxLock-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json")
$lockResult | ConvertTo-Json -Depth 6 | Set-Content -Path $lockEvidencePath -Encoding UTF8
$lockResult | ConvertTo-Json -Depth 6
exit 0
}
$dedupeEntries = @()
if (Test-Path $DedupeStatePath) {
try {
$dedupePayload = Get-Content $DedupeStatePath -Raw | ConvertFrom-Json
foreach ($entry in @($dedupePayload.entries)) {
try {
$ageSeconds = ((Get-Date).ToUniversalTime() - ([datetime]::Parse([string]$entry.lastSeen)).ToUniversalTime()).TotalSeconds
if ($ageSeconds -ge 0 -and $ageSeconds -le $DedupeWindowSeconds) {
$dedupeEntries += $entry
}
} catch {
continue
}
}
} catch {
$dedupeEntries = @()
}
}
$stamp = Get-Date -Format "yyyyMMdd-HHmmss"
$evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json"
$queued = @()
$ignored = @()
$failed = @()
$duplicateSuppressedCount = 0
$severityEscalationCount = 0
$files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue |
Sort-Object LastWriteTime |
@@ -469,7 +651,8 @@ foreach ($file in $files) {
continue
}
$modeName = Resolve-AgentAlertMode $alert
$routeDecision = Resolve-AgentAlertRoute $alert
$modeName = [string]$routeDecision.mode
if ($modeName -notin $allowedModes) {
$failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
Move-Item -Force $runningPath $failedPath
@@ -483,12 +666,56 @@ foreach ($file in $files) {
continue
}
$correlationKey = Get-AgentAlertCorrelationKey $alert
$severity = [string](Get-AgentField $alert "severity" "unknown")
$existingDedupe = @($dedupeEntries | Where-Object { [string]$_.key -eq $correlationKey } | Select-Object -First 1)
$dedupeDecision = "new_dispatch"
if (@($existingDedupe).Count -gt 0) {
$previous = $existingDedupe[0]
$currentRank = Get-AgentAlertSeverityRank $severity
$previousRank = Get-AgentAlertSeverityRank ([string]$previous.severity)
if ($currentRank -le $previousRank) {
$occurrences = 1
if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrences = [int]$previous.occurrenceCount + 1 }
$dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey })
$dedupeEntries += [pscustomobject]@{
key = $correlationKey
alertId = [string]$previous.alertId
queueId = [string]$previous.queueId
severity = [string]$previous.severity
firstSeen = [string]$previous.firstSeen
lastSeen = (Get-Date -Format o)
occurrenceCount = $occurrences
lastDecision = "duplicate_single_flight_suppressed"
lastSuppressedAlertId = $alertId
}
Save-AgentAlertDedupeState $dedupeEntries
$ignoredPath = Join-Path $IgnoredDir ("duplicate-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name)
Move-Item -Force $runningPath $ignoredPath
$ignored += [pscustomobject]@{
alertId = $alertId
file = $ignoredPath
reason = "duplicate_single_flight"
correlationKey = $correlationKey
originalAlertId = [string]$previous.alertId
originalQueueId = [string]$previous.queueId
occurrenceCount = $occurrences
severity = $severity
originalSeverity = [string]$previous.severity
}
$duplicateSuppressedCount += 1
continue
}
$dedupeDecision = "severity_escalation_dispatch"
$severityEscalationCount += 1
}
$queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8))
$controlled = Get-AgentControlledApplyForAlert $alert $modeName
$instruction = New-AgentInstructionFromAlert $alert $alertId $modeName
$source = [string](Get-AgentField $alert "source" "sre-war-room")
$kind = [string](Get-AgentField $alert "kind" "unknown")
$severity = [string](Get-AgentField $alert "severity" "unknown")
$service = [string](Get-AgentField $alert "service" "unknown")
$hostName = [string](Get-AgentField $alert "host" "")
$replyChatId = [string](Get-AgentField $alert "replyChatId" "")
@@ -502,6 +729,10 @@ foreach ($file in $files) {
id = $queueId
instruction = $instruction
resolvedMode = $modeName
routeSource = [string]$routeDecision.source
routeMatched = [string]$routeDecision.matched
correlationKey = $correlationKey
dedupeDecision = $dedupeDecision
controlledApply = $controlled
requestedBy = $env:USERNAME
source = "agent99-sre-alert-inbox"
@@ -519,6 +750,10 @@ foreach ($file in $files) {
[pscustomobject]@{
id = $queueId
mode = $modeName
routeSource = [string]$routeDecision.source
routeMatched = [string]$routeDecision.matched
correlationKey = $correlationKey
dedupeDecision = $dedupeDecision
controlledApply = $controlled
requestedBy = $env:USERNAME
source = "agent99-sre-alert-inbox"
@@ -538,6 +773,27 @@ foreach ($file in $files) {
} | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8
Move-Item -Force $runningPath $processedPath
$firstSeen = (Get-Date -Format o)
$occurrenceCount = 1
if (@($existingDedupe).Count -gt 0) {
$previous = $existingDedupe[0]
if ($previous.PSObject.Properties["firstSeen"] -and $previous.firstSeen) { $firstSeen = [string]$previous.firstSeen }
if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrenceCount = [int]$previous.occurrenceCount + 1 }
}
$dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey })
$dedupeEntries += [pscustomobject]@{
key = $correlationKey
alertId = $alertId
queueId = $queueId
severity = $severity
firstSeen = $firstSeen
lastSeen = (Get-Date -Format o)
occurrenceCount = $occurrenceCount
lastDecision = $dedupeDecision
mode = $modeName
}
Save-AgentAlertDedupeState $dedupeEntries
$queued += [pscustomobject]@{
alertId = $alertId
source = $source
@@ -546,6 +802,10 @@ foreach ($file in $files) {
service = $service
host = $hostName
mode = $modeName
routeSource = [string]$routeDecision.source
routeMatched = [string]$routeDecision.matched
correlationKey = $correlationKey
dedupeDecision = $dedupeDecision
controlledApply = $controlled
suppressTelegram = $suppressTelegram
requestPath = $requestPath
@@ -568,6 +828,8 @@ $result = [pscustomobject]@{
incomingPath = $IncomingDir
queuedCount = @($queued).Count
ignoredCount = @($ignored).Count
duplicateSuppressedCount = $duplicateSuppressedCount
severityEscalationCount = $severityEscalationCount
failedCount = @($failed).Count
queued = $queued
ignored = $ignored
@@ -578,3 +840,4 @@ $result = [pscustomobject]@{
$result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8
$result | ConvertTo-Json -Depth 10
if ($inboxLockStream) { $inboxLockStream.Dispose() }

114
agent99-telegram-inbox.ps1
View File

@@ -257,7 +257,7 @@ function Test-AgentMonitoringAlertText {
return $false
}
if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") {
if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot-auto-recovery|postgres|postgresql|database|alertchain|alert_chain|monitoring pipeline|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") {
return $true
}
@@ -296,14 +296,16 @@ function Get-AgentTelegramAlertKind {
if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" }
if ($lower -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { return "repository_missing" }
if ($lower -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { return "provider_freshness_signal" }
if ($lower -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag") { return "database_health" }
if ($lower -match "alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { return "alert_chain_health" }
if ($lower -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|powered_off" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" }
if ($lower -match "502|site_502|public_route|website|route") { return "public_route_502" }
if ($lower -match "missing_key|e_ai_missing_key|ai support|locale guard" -or (Test-AgentTextContainsCode $Text @(0x5BA2, 0x670D))) { return "application_ai_config_error" }
if ($lower -match "cpu|load" -or (Test-AgentTextContainsCode $Text @(0x8CA0, 0x8F09))) { return "performance_cpu" }
if ($lower -match "disk" -or (Test-AgentTextContainsCode $Text @(0x78C1, 0x789F))) { return "performance_disk" }
if ($lower -match "memory|mem_|ram" -or (Test-AgentTextContainsCode $Text @(0x8A18, 0x61B6, 0x9AD4))) { return "performance_memory" }
if ($lower -match "harbor|registry") { return "harbor_registry" }
if ($lower -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" }
if ($lower -match "hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|down" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" }
if ($lower -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" }
return "monitoring_alert"
}
@@ -333,6 +335,52 @@ function Get-AgentTelegramAlertHost {
return ""
}
function Get-AgentTelegramAlertTarget {
param(
[string]$Text,
[string]$Kind,
[string]$Service,
[string]$HostName
)
$targetMatch = [regex]::Match($Text, "(?im)\bTarget\s*[:\uFF1A]\s*([^\r\n]+)")
if ($targetMatch.Success) {
return $targetMatch.Groups[1].Value.Trim()
}
if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" }
if ($Kind -eq "host_recovery" -and $HostName) { return $HostName }
if ($Service -and $Service -ne "unknown") { return $Service }
return $Kind
}
function Get-AgentTelegramSuggestedMode {
param([string]$Kind)
$modeByKind = @{
"provider_freshness_signal" = "ProviderFreshness"
"backup_health" = "BackupCheck"
"wazuh_security_alert" = "SecurityTriage"
"public_route_502" = "Recover"
"host_recovery" = "Recover"
"performance_cpu" = "Perf"
"performance_memory" = "Perf"
"performance_disk" = "Perf"
"harbor_registry" = "HarborRepair"
"awoooi_k3s" = "AwoooRepair"
"database_health" = "Status"
"alert_chain_health" = "Status"
"application_ai_config_error" = "Status"
"repository_missing" = "Status"
}
if ($modeByKind.ContainsKey($Kind)) { return [string]$modeByKind[$Kind] }
return "Status"
}
function Convert-AgentTelegramRouteToken {
param([string]$Value)
$safe = [regex]::Replace($Value.ToLowerInvariant(), "[^a-z0-9_.-]+", "_").Trim([char[]]".-_")
if ($safe) { return $safe }
return "unknown"
}
function Test-AgentTelegramSuppressPage {
param([object]$Update)
$text = if ($Update -and $Update.PSObject.Properties["text"]) { [string]$Update.text } else { "" }
@@ -359,6 +407,15 @@ function New-AgentMonitoringAlertFromTelegram {
$alertPath = Join-Path $AlertIncomingDir ($alertId + ".json")
$message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text }
$kind = Get-AgentTelegramAlertKind $text
$service = Get-AgentTelegramAlertService $text
$hostName = Get-AgentTelegramAlertHost $text
$target = Get-AgentTelegramAlertTarget $text $kind $service $hostName
$suggestedMode = Get-AgentTelegramSuggestedMode $kind
$correlationKey = @(
(Convert-AgentTelegramRouteToken $kind),
(Convert-AgentTelegramRouteToken $target),
$(if ($hostName) { Convert-AgentTelegramRouteToken $hostName } else { "no-host" })
) -join ":"
$suppressTelegram = Test-AgentTelegramSuppressPage $Update
$labels = if ($kind -eq "provider_freshness_signal") {
@("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green")
@@ -373,11 +430,22 @@ function New-AgentMonitoringAlertFromTelegram {
source = "telegram-sre-war-room-monitoring"
severity = $severity
kind = $kind
service = Get-AgentTelegramAlertService $text
host = Get-AgentTelegramAlertHost $text
suggestedMode = $suggestedMode
controlledApply = [bool]($suggestedMode -in @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed"))
service = $service
host = $hostName
title = "AwoooI SRE Telegram monitoring alert"
message = $message
labels = $labels
routing = [pscustomobject]@{
schemaVersion = "agent99_alert_route_v1"
kind = $kind
suggestedMode = $suggestedMode
routeSource = "telegram_structured_kind"
groupKey = $correlationKey
correlationKey = $correlationKey
singleFlightWindowSeconds = 300
}
telegramUpdateId = $Update.update_id
telegramMessageId = $Update.message_id
telegramChatTitle = $Update.chat_title
@@ -490,6 +558,42 @@ function Invoke-AgentTelegramInboxSelfTest {
expectedService = "gitea"
expectedHost = ""
},
[pscustomobject]@{
name = "reboot slo wins over cpu text"
text = "critical Target: reboot-auto-recovery-slo host 192.168.0.110 all_required_hosts_not_in_10_minute_reboot_window cpu pressure"
expectedMonitoring = $true
expectedSeverity = "critical"
expectedKind = "host_recovery"
expectedService = "unknown"
expectedHost = "192.168.0.110"
},
[pscustomobject]@{
name = "postgres does not become k3s"
text = "critical PostgreSQL database unavailable service awoooi deployment"
expectedMonitoring = $true
expectedSeverity = "critical"
expectedKind = "database_health"
expectedService = "awoooi"
expectedHost = ""
},
[pscustomobject]@{
name = "alert chain does not become k3s"
text = "critical AlertChainBroken monitoring pipeline down service awoooi"
expectedMonitoring = $true
expectedSeverity = "critical"
expectedKind = "alert_chain_health"
expectedService = "awoooi"
expectedHost = ""
},
[pscustomobject]@{
name = "k3s stays awooo repair domain"
text = "critical Target: awoooi-api k3s pod crashloopbackoff deployment"
expectedMonitoring = $true
expectedSeverity = "critical"
expectedKind = "awoooi_k3s"
expectedService = "awoooi"
expectedHost = ""
},
[pscustomobject]@{
name = "public 502"
text = "critical public route 502 website https://awoooi.wooo.work"

35
apps/api/src/api/v1/webhooks.py
View File

@@ -2850,22 +2850,6 @@ async def alertmanager_webhook(
labels=dict(alert.labels) if alert.labels else {},
annotations=dict(alert.annotations) if alert.annotations else {},
)
background_tasks.add_task(
bridge_alertmanager_to_agent99,
alert_id=alert_id,
alertname=alertname,
severity=severity,
namespace=namespace,
target_resource=target_resource,
message=message,
labels=dict(alert.labels) if alert.labels else {},
annotations=dict(alert.annotations) if alert.annotations else {},
fingerprint=fingerprint,
alert_category=alert_category,
notification_type=notification_type,
source_url=alert.generatorURL,
)
# ==========================================================================
# ADR-076: 告警聚合引擎 — 5 分鐘滑動視窗，防止告警風暴
# 2026-04-14 Claude Haiku 4.5 Asia/Taipei
@@ -2920,6 +2904,25 @@ async def alertmanager_webhook(
converged=True,
)
# Agent99 only receives the parent route after grouping. The bridge adds a
# Redis NX fingerprint lock so concurrent duplicate deliveries cannot race
# past this point and launch duplicate remediation.
background_tasks.add_task(
bridge_alertmanager_to_agent99,
alert_id=alert_id,
alertname=alertname,
severity=severity,
namespace=namespace,
target_resource=target_resource,
message=message,
labels=dict(alert.labels) if alert.labels else {},
annotations=dict(alert.annotations) if alert.annotations else {},
fingerprint=fingerprint,
alert_category=alert_category,
notification_type=notification_type,
source_url=alert.generatorURL,
)
try:
service = get_approval_service()

228
apps/api/src/services/agent99_sre_bridge.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import asyncio
import hashlib
import json
import re
import urllib.error
@@ -16,6 +17,8 @@ from src.utils.timezone import now_taipei
logger = get_logger("awoooi.agent99_sre_bridge")
AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600
_SENSITIVE_KEY_RE = re.compile(
r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)",
re.IGNORECASE,
@@ -99,18 +102,49 @@ def resolve_agent99_alert_kind(
text,
):
return "provider_freshness_signal"
if re.search(
r"wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|"
r"privilege|vulnerability|\bcve\b|\bioc\b|suspicious|unauthorized",
text,
):
return "security_health"
if re.search(r"backup|snapshot|restore drill|restore_drill|backup cron", text):
return "backup_health"
if re.search(
r"postgres|postgresql|mysql|mariadb|database|db_connection|"
r"pg_isready|connection pool|replication lag",
text,
):
return "database_health"
if re.search(
r"alertchain|alert_chain|alert chain|alertmanager.*(broken|down|failed)|"
r"monitoring pipeline|notification pipeline",
text,
):
return "alert_chain_health"
if re.search(
r"reboot-auto-recovery|reboot_auto_recovery|reboot recovery|"
r"host_down|hostdown|host_reboot|vm_not_running|powered_off|"
r"all_required_hosts_not_in_10_minute_reboot_window",
text,
):
return "host_recovery"
if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text):
return "public_route_502"
if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text):
return "performance_pressure"
if re.search(r"backup|snapshot|restore drill|cron", text):
return "backup_health"
if re.search(r"harbor|registry", text):
return "harbor_registry"
if re.search(r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text):
if re.search(
r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text
):
return "awoooi_k3s"
if re.search(r"host_down|hostdown|vm_not_running|reboot|restart|powered_off", text):
return "host_recovery"
if re.search(
r"missing_key|e_ai_missing_key|application_ai_config_error|"
r"locale guard|ai support",
text,
):
return "application_ai_config_error"
return "monitoring_alert"
@@ -120,12 +154,95 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
"public_route_502": "Recover",
"performance_pressure": "Perf",
"backup_health": "BackupCheck",
"security_health": "SecurityTriage",
"database_health": "Status",
"alert_chain_health": "Status",
"harbor_registry": "HarborRepair",
"awoooi_k3s": "AwoooRepair",
"host_recovery": "Recover",
"application_ai_config_error": "Status",
}.get(kind)
def _agent99_route_group_key(
*,
kind: str,
target_resource: str,
host: str,
) -> str:
parts = (
kind.strip().lower() or "monitoring_alert",
target_resource.strip().lower() or "unknown",
host.strip().lower() or "no-host",
)
return ":".join(_safe_file_token(part) for part in parts)
def agent99_sre_single_flight_key(fingerprint: str) -> str:
digest = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest()
return f"agent99:sre_dispatch:single_flight:{digest}"
async def try_acquire_agent99_sre_single_flight(
fingerprint: str,
alert_id: str,
*,
ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
) -> bool:
"""Grant one Agent99 dispatch per fingerprint inside the controlled window."""
if not fingerprint:
return True
try:
from src.core.redis_client import get_redis
redis = get_redis()
acquired = await redis.set(
agent99_sre_single_flight_key(fingerprint),
alert_id,
ex=ttl_seconds,
nx=True,
)
return bool(acquired)
except Exception as exc:
logger.warning(
"agent99_sre_single_flight_failed_fail_open",
fingerprint=fingerprint,
alert_id=alert_id,
error=str(exc),
)
return True
async def release_agent99_sre_single_flight(fingerprint: str, alert_id: str) -> None:
"""Release only the caller-owned lock when transport did not accept the alert."""
if not fingerprint:
return
try:
from src.core.redis_client import get_redis
redis = get_redis()
await redis.eval(
"""
if redis.call('get', KEYS[1]) == ARGV[1] then
return redis.call('del', KEYS[1])
end
return 0
""",
1,
agent99_sre_single_flight_key(fingerprint),
alert_id,
)
except Exception as exc:
logger.warning(
"agent99_sre_single_flight_release_failed",
fingerprint=fingerprint,
alert_id=alert_id,
error=str(exc),
)
def build_agent99_sre_alert(
*,
alert_id: str,
@@ -164,6 +281,11 @@ def build_agent99_sre_alert(
or target_resource
or "awoooi"
)
route_group_key = _agent99_route_group_key(
kind=kind,
target_resource=target_resource,
host=host,
)
safe_labels = [
"alertmanager",
"awoooi-api",
@@ -172,9 +294,13 @@ def build_agent99_sre_alert(
f"target={_safe_label_value('target', target_resource)}",
]
if fingerprint:
safe_labels.append(f"fingerprint={_safe_label_value('fingerprint', fingerprint)}")
safe_labels.append(
f"fingerprint={_safe_label_value('fingerprint', fingerprint)}"
)
for key, value in sorted(labels.items()):
safe_labels.append(f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}")
safe_labels.append(
f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}"
)
if kind == "provider_freshness_signal":
safe_labels.extend(["provider-freshness", "no-false-green"])
@@ -197,8 +323,26 @@ def build_agent99_sre_alert(
"message": _safe_text(message, max_length=2000),
"labels": safe_labels,
"force": True,
"controlledApply": True,
"controlledApply": suggested_mode
in {
"Recover",
"StartVMs",
"HarborRepair",
"AwoooRepair",
"Perf",
"LoadShed",
},
"createdAt": now_taipei().isoformat(),
"routing": {
"schemaVersion": "agent99_alert_route_v1",
"kind": kind,
"suggestedMode": suggested_mode or "Status",
"routeSource": "structured_kind",
"groupKey": route_group_key,
"correlationKey": route_group_key,
"sourceFingerprint": _safe_text(fingerprint, max_length=160),
"singleFlightWindowSeconds": AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS,
},
"awoooi": {
"alertId": _safe_text(alert_id, max_length=160),
"alertname": _safe_text(alertname, max_length=160),
@@ -222,7 +366,9 @@ def write_agent99_sre_alert(
*,
inbox_path: str | None = None,
) -> Path | None:
target_dir_text = inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH
target_dir_text = (
inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH
)
if not target_dir_text:
logger.info(
"agent99_sre_bridge_disabled",
@@ -258,11 +404,17 @@ def post_agent99_sre_alert(
relay_token: str | None = None,
timeout_seconds: float | None = None,
) -> dict[str, Any] | None:
target_url = relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
target_url = (
relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL
)
if not target_url:
return None
token = relay_token if relay_token is not None else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
token = (
relay_token
if relay_token is not None
else settings.AGENT99_SRE_ALERT_RELAY_TOKEN
)
timeout = (
timeout_seconds
if timeout_seconds is not None
@@ -276,7 +428,9 @@ def post_agent99_sre_alert(
if token:
headers["X-Agent99-Relay-Token"] = token
request = urllib.request.Request(target_url, data=data, headers=headers, method="POST")
request = urllib.request.Request(
target_url, data=data, headers=headers, method="POST"
)
try:
with urllib.request.urlopen(request, timeout=timeout) as response:
status = int(response.getcode())
@@ -331,7 +485,9 @@ async def bridge_alertmanager_to_agent99(
alert_category: str = "",
notification_type: str = "",
source_url: str | None = None,
) -> None:
) -> dict[str, Any]:
single_flight_fingerprint = ""
single_flight_acquired = False
try:
payload = build_agent99_sre_alert(
alert_id=alert_id,
@@ -347,11 +503,55 @@ async def bridge_alertmanager_to_agent99(
notification_type=notification_type,
source_url=source_url,
)
await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
routing = (
payload.get("routing") if isinstance(payload.get("routing"), dict) else {}
)
single_flight_fingerprint = fingerprint or str(
routing.get("correlationKey") or ""
)
single_flight_acquired = await try_acquire_agent99_sre_single_flight(
single_flight_fingerprint,
alert_id,
)
if not single_flight_acquired:
logger.info(
"agent99_sre_dispatch_single_flight_suppressed",
alert_id=alert_id,
alertname=alertname,
fingerprint=single_flight_fingerprint,
group_key=routing.get("groupKey"),
)
return {
"status": "suppressed",
"reason": "single_flight_duplicate",
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
}
dispatch_result = await asyncio.to_thread(dispatch_agent99_sre_alert, payload)
if dispatch_result == "disabled":
await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id)
single_flight_acquired = False
return {
"status": "failed",
"reason": "bridge_disabled",
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
}
return {
"status": "dispatched",
"dispatch": dispatch_result,
"fingerprint": single_flight_fingerprint,
"groupKey": routing.get("groupKey"),
"kind": payload.get("kind"),
"suggestedMode": payload.get("suggestedMode"),
}
except Exception as exc:
if single_flight_acquired:
await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id)
logger.warning(
"agent99_sre_bridge_failed_fail_open",
alert_id=alert_id,
alertname=alertname,
error=str(exc),
)
return {"status": "failed", "reason": type(exc).__name__}

61
apps/api/tests/test_agent99_alert_dispatch_order_contract.py Normal file
View File

@@ -0,0 +1,61 @@
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1"
TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1"
def test_alertmanager_groups_before_agent99_dispatch() -> None:
source = (ROOT / "apps/api/src/api/v1/webhooks.py").read_text(encoding="utf-8")
normalized_at = source.index('"alertmanager_normalized"')
grouping_at = source.index("get_alert_grouping_service().evaluate", normalized_at)
grouped_return_at = source.index(
"if grouping_result and grouping_result.is_grouped", grouping_at
)
dispatch_at = source.index("bridge_alertmanager_to_agent99", grouped_return_at)
assert grouping_at < grouped_return_at < dispatch_at
def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None:
source = (ROOT / "apps/api/src/services/agent99_sre_bridge.py").read_text(
encoding="utf-8"
)
bridge_at = source.index("async def bridge_alertmanager_to_agent99")
lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at)
dispatch_at = source.index("dispatch_agent99_sre_alert, payload", lock_at)
assert lock_at < dispatch_at
def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
source = SRE_INBOX.read_text(encoding="utf-8")
route_at = source.index("function Resolve-AgentAlertRoute")
suggested_at = source.index("if ($suggestedMode", route_at)
kind_at = source.index("if ($kind -and $kindModes.ContainsKey", suggested_at)
fallback_at = source.index('if ($text -match "502', kind_at)
assert suggested_at < kind_at < fallback_at
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source
assert '"host_recovery" = "Recover"' in source
def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None:
source = SRE_INBOX.read_text(encoding="utf-8")
assert '"sre-alert-single-flight.json"' in source
assert "[System.IO.FileShare]::None" in source
assert 'reason = "duplicate_single_flight"' in source
assert "severity_escalation_dispatch" in source
assert "correlationKey = $correlationKey" in source
def test_telegram_ingress_writes_same_structured_route_contract() -> None:
source = TELEGRAM_INBOX.read_text(encoding="utf-8")
assert "function Get-AgentTelegramSuggestedMode" in source
assert 'schemaVersion = "agent99_alert_route_v1"' in source
assert 'routeSource = "telegram_structured_kind"' in source
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source

247
apps/api/tests/test_agent99_sre_bridge.py
View File

@@ -2,9 +2,14 @@ from __future__ import annotations
import json
import pytest
from src.services.agent99_sre_bridge import (
agent99_sre_single_flight_key,
bridge_alertmanager_to_agent99,
build_agent99_sre_alert,
post_agent99_sre_alert,
release_agent99_sre_single_flight,
resolve_agent99_alert_kind,
write_agent99_sre_alert,
)
@@ -37,13 +42,99 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
assert payload["source"] == "awoooi-api-alertmanager"
assert payload["kind"] == "provider_freshness_signal"
assert payload["suggestedMode"] == "ProviderFreshness"
assert payload["controlledApply"] is True
assert payload["controlledApply"] is False
assert payload["force"] is True
assert "provider-freshness" in payload["labels"]
assert "no-false-green" in payload["labels"]
assert "token=[redacted]" in payload["labels"]
assert "must-not-leak" not in json.dumps(payload)
assert "do not switch provider" in payload["instruction"]
assert payload["routing"]["routeSource"] == "structured_kind"
assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"]
@pytest.mark.parametrize(
("alertname", "target", "message", "labels", "expected_kind", "expected_mode"),
[
(
"HostBackupFailed",
"backup",
"snapshot stale",
{},
"backup_health",
"BackupCheck",
),
(
"provider_freshness_signal",
"provider_freshness_signal",
"last_seen missing",
{},
"provider_freshness_signal",
"ProviderFreshness",
),
(
"RebootAutoRecoveryActiveBlocker",
"reboot-auto-recovery-slo",
"all_required_hosts_not_in_10_minute_reboot_window cpu pressure",
{"service": "awoooi"},
"host_recovery",
"Recover",
),
(
"PostgresqlDown",
"postgresql",
"database unavailable deployment awoooi",
{"service": "awoooi"},
"database_health",
"Status",
),
(
"AlertChainBroken",
"alertmanager",
"monitoring pipeline down awoooi",
{"service": "awoooi"},
"alert_chain_health",
"Status",
),
(
"HostDown",
"192.168.0.120",
"host_down vm_not_running",
{"host": "192.168.0.120"},
"host_recovery",
"Recover",
),
(
"KubePodCrashLooping",
"awoooi-api",
"k3s pod crashloopbackoff",
{"service": "awoooi"},
"awoooi_k3s",
"AwoooRepair",
),
],
)
def test_agent99_routing_replay_corpus(
alertname: str,
target: str,
message: str,
labels: dict[str, str],
expected_kind: str,
expected_mode: str,
) -> None:
payload = build_agent99_sre_alert(
alert_id=f"replay-{alertname}",
alertname=alertname,
severity="critical",
namespace="awoooi-prod",
target_resource=target,
message=message,
labels=labels,
fingerprint=f"fp-{alertname}",
)
assert payload["kind"] == expected_kind
assert payload["suggestedMode"] == expected_mode
def test_alert_kind_mapping_for_route_and_performance() -> None:
@@ -108,7 +199,7 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
def read(self, _limit: int) -> bytes:
return b'{"ok":true}'
def __enter__(self) -> "FakeResponse":
def __enter__(self) -> FakeResponse:
return self
def __exit__(self, *_args: object) -> None:
@@ -141,3 +232,155 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None:
assert "authorization=[redacted]" in seen["payload"]["labels"]
assert "must-not-leak" not in json.dumps(seen["payload"])
assert seen["headers"]["X-agent99-relay-token"] == "relay-token"
def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None:
key = agent99_sre_single_flight_key("private/source/fingerprint")
assert key.startswith("agent99:sre_dispatch:single_flight:")
assert "private" not in key
@pytest.mark.asyncio
async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None:
async def duplicate(*_args, **_kwargs) -> bool:
return False
dispatched: list[dict[str, object]] = []
monkeypatch.setattr(
"src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
duplicate,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
)
result = await bridge_alertmanager_to_agent99(
alert_id="duplicate-alert",
alertname="HostDown",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="host_down",
fingerprint="same-fingerprint",
)
assert result["status"] == "suppressed"
assert result["reason"] == "single_flight_duplicate"
assert dispatched == []
@pytest.mark.asyncio
async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None:
async def winner(*_args, **_kwargs) -> bool:
return True
dispatched: list[dict[str, object]] = []
monkeypatch.setattr(
"src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
lambda payload: dispatched.append(payload) or "relay",
)
result = await bridge_alertmanager_to_agent99(
alert_id="winner-alert",
alertname="HostDown",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="host_down",
fingerprint="winner-fingerprint",
)
assert result["status"] == "dispatched"
assert result["dispatch"] == "relay"
assert result["suggestedMode"] == "Recover"
assert len(dispatched) == 1
@pytest.mark.asyncio
async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None:
async def winner(*_args, **_kwargs) -> bool:
return True
released: list[tuple[str, str]] = []
async def release(fingerprint: str, alert_id: str) -> None:
released.append((fingerprint, alert_id))
def fail_transport(_payload: dict[str, object]) -> str:
raise RuntimeError("relay unavailable")
monkeypatch.setattr(
"src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight",
winner,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.release_agent99_sre_single_flight",
release,
)
monkeypatch.setattr(
"src.services.agent99_sre_bridge.dispatch_agent99_sre_alert",
fail_transport,
)
result = await bridge_alertmanager_to_agent99(
alert_id="transport-failure",
alertname="HostDown",
severity="critical",
namespace="node",
target_resource="192.168.0.120",
message="host_down",
fingerprint="transport-fingerprint",
)
assert result == {"status": "failed", "reason": "RuntimeError"}
assert released == [("transport-fingerprint", "transport-failure")]
@pytest.mark.asyncio
async def test_agent99_single_flight_uses_redis_nx(monkeypatch) -> None:
seen: dict[str, object] = {}
class FakeRedis:
async def set(self, key, value, *, ex, nx): # type: ignore[no-untyped-def]
seen.update({"key": key, "value": value, "ex": ex, "nx": nx})
return True
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis())
from src.services.agent99_sre_bridge import try_acquire_agent99_sre_single_flight
acquired = await try_acquire_agent99_sre_single_flight(
"source-fingerprint", "alert-1"
)
assert acquired is True
assert str(seen["key"]).startswith("agent99:sre_dispatch:single_flight:")
assert seen["value"] == "alert-1"
assert seen["ex"] == 600
assert seen["nx"] is True
@pytest.mark.asyncio
async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> None:
seen: dict[str, object] = {}
class FakeRedis:
async def eval(self, script, key_count, key, owner): # type: ignore[no-untyped-def]
seen.update(
{"script": script, "key_count": key_count, "key": key, "owner": owner}
)
return 1
monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis())
await release_agent99_sre_single_flight("source-fingerprint", "alert-1")
assert "redis.call('get'" in str(seen["script"])
assert seen["key_count"] == 1
assert seen["owner"] == "alert-1"

5
apps/api/tests/test_telegram_gateway_error_sanitizer.py
View File

@@ -2,9 +2,9 @@ from __future__ import annotations
from src.services.telegram_gateway import (
_agent99_alert_from_ai_automation_card,
format_aiops_signal_alert_card,
_outbound_source_envelope,
_sanitize_telegram_error,
format_aiops_signal_alert_card,
)
@@ -204,7 +204,8 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz
)
assert payload["kind"] == "provider_freshness_signal"
assert payload["suggestedMode"] == "ProviderFreshness"
assert payload["controlledApply"] is True
assert payload["controlledApply"] is False
assert payload["routing"]["routeSource"] == "structured_kind"
assert payload["awoooi"]["alertCategory"] == "source_provider_freshness_triage"
assert payload["awoooi"]["notificationType"] == "ai_automation_alert_card_v1"
assert "component=telegram_gateway" in payload["labels"]