From 5c620844fe9919ce858e2a3da1a823c6aeb65322 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 16:44:01 +0800 Subject: [PATCH] fix(agent99): route and dedupe alerts before dispatch --- agent99-sre-alert-inbox.ps1 | 309 ++++++++++++++++-- agent99-telegram-inbox.ps1 | 114 ++++++- apps/api/src/api/v1/webhooks.py | 35 +- apps/api/src/services/agent99_sre_bridge.py | 228 ++++++++++++- ...t_agent99_alert_dispatch_order_contract.py | 61 ++++ apps/api/tests/test_agent99_sre_bridge.py | 247 +++++++++++++- .../test_telegram_gateway_error_sanitizer.py | 5 +- 7 files changed, 937 insertions(+), 62 deletions(-) create mode 100644 apps/api/tests/test_agent99_alert_dispatch_order_contract.py diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index d4a420c39..951d2c678 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -16,8 +16,12 @@ $FailedDir = Join-Path $AlertRoot "failed" $IgnoredDir = Join-Path $AlertRoot "ignored" $QueueDir = Join-Path $AgentRoot "queue" $RequestDir = Join-Path $AgentRoot "requests" +$StateDir = Join-Path $AgentRoot "state" +$DedupeStatePath = Join-Path $StateDir "sre-alert-single-flight.json" +$InboxLockPath = Join-Path $StateDir "sre-alert-inbox.lock" +$DedupeWindowSeconds = 300 -New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir | Out-Null +New-Item -ItemType Directory -Force -Path $EvidenceDir, $IncomingDir, $RunningDir, $ProcessedDir, $FailedDir, $IgnoredDir, $QueueDir, $RequestDir, $StateDir | Out-Null $allowedModes = @("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage") $remediationModes = @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed") @@ -86,46 +90,128 @@ function Test-AgentAlertActionable { return $false } -function Resolve-AgentAlertMode { +function Resolve-AgentAlertRoute { param([object]$Alert) $text = (Get-AgentAlertText $Alert).ToLowerInvariant() - $suggestedMode = [string](Get-AgentField $Alert "suggestedMode" "") + $suggestedMode = ([string](Get-AgentField $Alert "suggestedMode" "")).Trim() + $kind = ([string](Get-AgentField $Alert "kind" "")).Trim().ToLowerInvariant() + + if ($suggestedMode -and $suggestedMode -in $allowedModes) { + return [pscustomobject]@{ mode = $suggestedMode; source = "suggestedMode"; matched = $suggestedMode } + } + + $kindModes = @{ + "provider_freshness_signal" = "ProviderFreshness" + "backup_health" = "BackupCheck" + "backup_missing" = "BackupCheck" + "backup_failed" = "BackupCheck" + "security_health" = "SecurityTriage" + "wazuh_security_alert" = "SecurityTriage" + "public_route_502" = "Recover" + "host_recovery" = "Recover" + "host_down" = "Recover" + "host_reboot" = "Recover" + "vm_not_running" = "Recover" + "performance_pressure" = "Perf" + "performance_cpu" = "Perf" + "performance_memory" = "Perf" + "performance_disk" = "Perf" + "harbor_registry" = "HarborRepair" + "awoooi_k3s" = "AwoooRepair" + "database_health" = "Status" + "alert_chain_health" = "Status" + "application_ai_config_error" = "Status" + "repository_missing" = "Status" + } + if ($kind -and $kindModes.ContainsKey($kind)) { + return [pscustomobject]@{ mode = [string]$kindModes[$kind]; source = "kind"; matched = $kind } + } if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { - return "Recover" + return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" } } - if ($text -match "host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { - return "Recover" - } - if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") { - return "Perf" + if ($text -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { + return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" } } if ($text -match "backup|snapshot|restore drill|cron") { - return "BackupCheck" + return [pscustomobject]@{ mode = "BackupCheck"; source = "text_fallback"; matched = "backup" } } if ($text -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|privilege|vulnerability|cve|ioc|suspicious|unauthorized") { - return "SecurityTriage" + return [pscustomobject]@{ mode = "SecurityTriage"; source = "text_fallback"; matched = "security" } } if ($text -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { - return "ProviderFreshness" + return [pscustomobject]@{ mode = "ProviderFreshness"; source = "text_fallback"; matched = "provider_freshness" } + } + if ($text -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag|alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { + return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "diagnostic_only" } + } + if ($text -match "load_per_core|cpu|memory|mem_|disk_used|performance|pressure") { + return [pscustomobject]@{ mode = "Perf"; source = "text_fallback"; matched = "performance" } } if ($text -match "harbor|registry") { - return "HarborRepair" + return [pscustomobject]@{ mode = "HarborRepair"; source = "text_fallback"; matched = "harbor" } } - if ($text -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { - return "AwoooRepair" + if ($text -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { + return [pscustomobject]@{ mode = "AwoooRepair"; source = "text_fallback"; matched = "k3s" } } if ($text -match "missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") { - return "Status" + return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "application_config" } } if ($text -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { - return "Status" + return [pscustomobject]@{ mode = "Status"; source = "text_fallback"; matched = "repository" } } - if ($suggestedMode -and $suggestedMode -in $allowedModes) { - return $suggestedMode + return [pscustomobject]@{ mode = "Status"; source = "default"; matched = "none" } +} + +function Resolve-AgentAlertMode { + param([object]$Alert) + $route = Resolve-AgentAlertRoute $Alert + return [string]$route.mode +} + +function Get-AgentAlertCorrelationKey { + param([object]$Alert) + + $routing = Get-AgentField $Alert "routing" $null + $correlationKey = [string](Get-AgentField $routing "correlationKey" "") + if ($correlationKey) { return "route:$($correlationKey.ToLowerInvariant())" } + + $awoooi = Get-AgentField $Alert "awoooi" $null + $fingerprint = [string](Get-AgentField $awoooi "fingerprint" "") + if (-not $fingerprint) { $fingerprint = [string](Get-AgentField $Alert "fingerprint" "") } + if ($fingerprint) { return "fingerprint:$($fingerprint.ToLowerInvariant())" } + + $kind = ([string](Get-AgentField $Alert "kind" "monitoring_alert")).ToLowerInvariant() + $service = ([string](Get-AgentField $Alert "service" "unknown")).ToLowerInvariant() + $hostName = ([string](Get-AgentField $Alert "host" "no-host")).ToLowerInvariant() + return "fallback:$kind`:$service`:$hostName" +} + +function Get-AgentAlertSeverityRank { + param([string]$Severity) + switch ($Severity.ToLowerInvariant()) { + "critical" { return 4 } + "error" { return 4 } + "failed" { return 4 } + "warning" { return 3 } + "degraded" { return 3 } + "info" { return 2 } + default { return 1 } } - return "Status" +} + +function Save-AgentAlertDedupeState { + param([object[]]$Entries) + $payload = [pscustomobject]@{ + schemaVersion = "agent99_sre_alert_single_flight_v1" + generatedAt = (Get-Date -Format o) + windowSeconds = $DedupeWindowSeconds + entries = @($Entries) + } + $temporaryPath = "$DedupeStatePath.tmp-$PID" + $payload | ConvertTo-Json -Depth 8 | Set-Content -Path $temporaryPath -Encoding UTF8 + Move-Item -Force $temporaryPath $DedupeStatePath } function New-AgentInstructionFromAlert { @@ -270,6 +356,53 @@ function Invoke-AgentAlertRoutingSelfTest { message = "k3s pod crashloopbackoff imagepullbackoff deployment failure" } }, + [pscustomobject]@{ + name = "reboot slo wins over cpu contamination" + expectedActionable = $true + expectedMode = "Recover" + expectedRouteSource = "suggestedMode" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-reboot-slo" + source = "awoooi-api-alertmanager" + severity = "critical" + kind = "host_recovery" + suggestedMode = "Recover" + service = "reboot-auto-recovery-slo" + host = "192.168.0.110" + message = "all_required_hosts_not_in_10_minute_reboot_window cpu pressure" + } + }, + [pscustomobject]@{ + name = "postgres does not route to k3s" + expectedActionable = $true + expectedMode = "Status" + expectedRouteSource = "kind" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-postgres" + source = "awoooi-api-alertmanager" + severity = "critical" + kind = "database_health" + service = "awoooi" + message = "PostgreSQL database unavailable in deployment" + } + }, + [pscustomobject]@{ + name = "alert chain does not route to k3s" + expectedActionable = $true + expectedMode = "Status" + expectedRouteSource = "kind" + expectedControlledApply = $false + alert = [pscustomobject]@{ + id = "selftest-alert-chain" + source = "awoooi-api-alertmanager" + severity = "critical" + kind = "alert_chain_health" + service = "awoooi" + message = "AlertChainBroken monitoring pipeline down" + } + }, [pscustomobject]@{ name = "provider freshness" expectedActionable = $true @@ -378,14 +511,18 @@ function Invoke-AgentAlertRoutingSelfTest { $results = @() foreach ($case in $cases) { $actionable = Test-AgentAlertActionable $case.alert - $mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null } + $route = if ($actionable) { Resolve-AgentAlertRoute $case.alert } else { $null } + $mode = if ($route) { [string]$route.mode } else { $null } + $routeSource = if ($route) { [string]$route.source } else { $null } $controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false } $suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false } $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false } + $expectedRouteSource = if ($case.PSObject.Properties["expectedRouteSource"]) { [string]$case.expectedRouteSource } else { $null } $instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null } $ok = [bool]( $actionable -eq $case.expectedActionable -and [string]$mode -eq [string]$case.expectedMode -and + (-not $expectedRouteSource -or $routeSource -eq $expectedRouteSource) -and $controlled -eq $case.expectedControlledApply -and $suppressTelegram -eq $expectedSuppressTelegram ) @@ -397,6 +534,8 @@ function Invoke-AgentAlertRoutingSelfTest { expectedActionable = $case.expectedActionable mode = $mode expectedMode = $case.expectedMode + routeSource = $routeSource + expectedRouteSource = $expectedRouteSource controlledApply = $controlled expectedControlledApply = $case.expectedControlledApply suppressTelegram = $suppressTelegram @@ -428,11 +567,54 @@ if ($SelfTest) { exit 0 } +$inboxLockStream = $null +try { + $inboxLockStream = [System.IO.File]::Open( + $InboxLockPath, + [System.IO.FileMode]::OpenOrCreate, + [System.IO.FileAccess]::ReadWrite, + [System.IO.FileShare]::None + ) +} catch { + $lockResult = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = $true + skipped = $true + reason = "single_flight_lock_held" + lockPath = $InboxLockPath + } + $lockEvidencePath = Join-Path $EvidenceDir ("agent99-SreAlertInboxLock-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json") + $lockResult | ConvertTo-Json -Depth 6 | Set-Content -Path $lockEvidencePath -Encoding UTF8 + $lockResult | ConvertTo-Json -Depth 6 + exit 0 +} + +$dedupeEntries = @() +if (Test-Path $DedupeStatePath) { + try { + $dedupePayload = Get-Content $DedupeStatePath -Raw | ConvertFrom-Json + foreach ($entry in @($dedupePayload.entries)) { + try { + $ageSeconds = ((Get-Date).ToUniversalTime() - ([datetime]::Parse([string]$entry.lastSeen)).ToUniversalTime()).TotalSeconds + if ($ageSeconds -ge 0 -and $ageSeconds -le $DedupeWindowSeconds) { + $dedupeEntries += $entry + } + } catch { + continue + } + } + } catch { + $dedupeEntries = @() + } +} + $stamp = Get-Date -Format "yyyyMMdd-HHmmss" $evidencePath = Join-Path $EvidenceDir "agent99-SreAlertInbox-$stamp.json" $queued = @() $ignored = @() $failed = @() +$duplicateSuppressedCount = 0 +$severityEscalationCount = 0 $files = @(Get-ChildItem -Path $IncomingDir -Filter "*.json" -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime | @@ -469,7 +651,8 @@ foreach ($file in $files) { continue } - $modeName = Resolve-AgentAlertMode $alert + $routeDecision = Resolve-AgentAlertRoute $alert + $modeName = [string]$routeDecision.mode if ($modeName -notin $allowedModes) { $failedPath = Join-Path $FailedDir ("failed-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) Move-Item -Force $runningPath $failedPath @@ -483,12 +666,56 @@ foreach ($file in $files) { continue } + $correlationKey = Get-AgentAlertCorrelationKey $alert + $severity = [string](Get-AgentField $alert "severity" "unknown") + $existingDedupe = @($dedupeEntries | Where-Object { [string]$_.key -eq $correlationKey } | Select-Object -First 1) + $dedupeDecision = "new_dispatch" + if (@($existingDedupe).Count -gt 0) { + $previous = $existingDedupe[0] + $currentRank = Get-AgentAlertSeverityRank $severity + $previousRank = Get-AgentAlertSeverityRank ([string]$previous.severity) + if ($currentRank -le $previousRank) { + $occurrences = 1 + if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrences = [int]$previous.occurrenceCount + 1 } + $dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey }) + $dedupeEntries += [pscustomobject]@{ + key = $correlationKey + alertId = [string]$previous.alertId + queueId = [string]$previous.queueId + severity = [string]$previous.severity + firstSeen = [string]$previous.firstSeen + lastSeen = (Get-Date -Format o) + occurrenceCount = $occurrences + lastDecision = "duplicate_single_flight_suppressed" + lastSuppressedAlertId = $alertId + } + Save-AgentAlertDedupeState $dedupeEntries + + $ignoredPath = Join-Path $IgnoredDir ("duplicate-" + (Convert-AgentSafeFileToken $alertId) + "-" + $file.Name) + Move-Item -Force $runningPath $ignoredPath + $ignored += [pscustomobject]@{ + alertId = $alertId + file = $ignoredPath + reason = "duplicate_single_flight" + correlationKey = $correlationKey + originalAlertId = [string]$previous.alertId + originalQueueId = [string]$previous.queueId + occurrenceCount = $occurrences + severity = $severity + originalSeverity = [string]$previous.severity + } + $duplicateSuppressedCount += 1 + continue + } + $dedupeDecision = "severity_escalation_dispatch" + $severityEscalationCount += 1 + } + $queueId = "sre-alert-" + (Get-Date -Format "yyyyMMdd-HHmmss") + "-" + ([guid]::NewGuid().ToString("N").Substring(0, 8)) $controlled = Get-AgentControlledApplyForAlert $alert $modeName $instruction = New-AgentInstructionFromAlert $alert $alertId $modeName $source = [string](Get-AgentField $alert "source" "sre-war-room") $kind = [string](Get-AgentField $alert "kind" "unknown") - $severity = [string](Get-AgentField $alert "severity" "unknown") $service = [string](Get-AgentField $alert "service" "unknown") $hostName = [string](Get-AgentField $alert "host" "") $replyChatId = [string](Get-AgentField $alert "replyChatId" "") @@ -502,6 +729,10 @@ foreach ($file in $files) { id = $queueId instruction = $instruction resolvedMode = $modeName + routeSource = [string]$routeDecision.source + routeMatched = [string]$routeDecision.matched + correlationKey = $correlationKey + dedupeDecision = $dedupeDecision controlledApply = $controlled requestedBy = $env:USERNAME source = "agent99-sre-alert-inbox" @@ -519,6 +750,10 @@ foreach ($file in $files) { [pscustomobject]@{ id = $queueId mode = $modeName + routeSource = [string]$routeDecision.source + routeMatched = [string]$routeDecision.matched + correlationKey = $correlationKey + dedupeDecision = $dedupeDecision controlledApply = $controlled requestedBy = $env:USERNAME source = "agent99-sre-alert-inbox" @@ -538,6 +773,27 @@ foreach ($file in $files) { } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 Move-Item -Force $runningPath $processedPath + $firstSeen = (Get-Date -Format o) + $occurrenceCount = 1 + if (@($existingDedupe).Count -gt 0) { + $previous = $existingDedupe[0] + if ($previous.PSObject.Properties["firstSeen"] -and $previous.firstSeen) { $firstSeen = [string]$previous.firstSeen } + if ($previous.PSObject.Properties["occurrenceCount"]) { $occurrenceCount = [int]$previous.occurrenceCount + 1 } + } + $dedupeEntries = @($dedupeEntries | Where-Object { [string]$_.key -ne $correlationKey }) + $dedupeEntries += [pscustomobject]@{ + key = $correlationKey + alertId = $alertId + queueId = $queueId + severity = $severity + firstSeen = $firstSeen + lastSeen = (Get-Date -Format o) + occurrenceCount = $occurrenceCount + lastDecision = $dedupeDecision + mode = $modeName + } + Save-AgentAlertDedupeState $dedupeEntries + $queued += [pscustomobject]@{ alertId = $alertId source = $source @@ -546,6 +802,10 @@ foreach ($file in $files) { service = $service host = $hostName mode = $modeName + routeSource = [string]$routeDecision.source + routeMatched = [string]$routeDecision.matched + correlationKey = $correlationKey + dedupeDecision = $dedupeDecision controlledApply = $controlled suppressTelegram = $suppressTelegram requestPath = $requestPath @@ -568,6 +828,8 @@ $result = [pscustomobject]@{ incomingPath = $IncomingDir queuedCount = @($queued).Count ignoredCount = @($ignored).Count + duplicateSuppressedCount = $duplicateSuppressedCount + severityEscalationCount = $severityEscalationCount failedCount = @($failed).Count queued = $queued ignored = $ignored @@ -578,3 +840,4 @@ $result = [pscustomobject]@{ $result | ConvertTo-Json -Depth 10 | Set-Content -Path $evidencePath -Encoding UTF8 $result | ConvertTo-Json -Depth 10 +if ($inboxLockStream) { $inboxLockStream.Dispose() } diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index 405b51692..db3bbcecc 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -257,7 +257,7 @@ function Test-AgentMonitoringAlertText { return $false } - if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") { + if ($lower -match "alertmanager|\bfiring\b|\bcritical\b|\bwarning\b|\bdegraded\b|\bdown\b|\b502\b|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|harbor|registry|k3s|backup|snapshot|cpu|load|memory|disk|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot-auto-recovery|postgres|postgresql|database|alertchain|alert_chain|monitoring pipeline|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|ai support|locale guard|missing_key|e_ai_missing_key") { return $true } @@ -296,14 +296,16 @@ function Get-AgentTelegramAlertKind { if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" } if ($lower -match "gitea|repo_missing|repository_missing|repo missing|repository missing") { return "repository_missing" } if ($lower -match "provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|last_seen|last seen|provider freshness|provider window|ingestion") { return "provider_freshness_signal" } + if ($lower -match "postgres|postgresql|mysql|mariadb|database|db_connection|pg_isready|replication lag") { return "database_health" } + if ($lower -match "alertchain|alert_chain|alert chain|monitoring pipeline|notification pipeline") { return "alert_chain_health" } + if ($lower -match "reboot-auto-recovery|reboot_auto_recovery|all_required_hosts_not_in_10_minute_reboot_window|hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|powered_off" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" } if ($lower -match "502|site_502|public_route|website|route") { return "public_route_502" } if ($lower -match "missing_key|e_ai_missing_key|ai support|locale guard" -or (Test-AgentTextContainsCode $Text @(0x5BA2, 0x670D))) { return "application_ai_config_error" } if ($lower -match "cpu|load" -or (Test-AgentTextContainsCode $Text @(0x8CA0, 0x8F09))) { return "performance_cpu" } if ($lower -match "disk" -or (Test-AgentTextContainsCode $Text @(0x78C1, 0x789F))) { return "performance_disk" } if ($lower -match "memory|mem_|ram" -or (Test-AgentTextContainsCode $Text @(0x8A18, 0x61B6, 0x9AD4))) { return "performance_memory" } if ($lower -match "harbor|registry") { return "harbor_registry" } - if ($lower -match "awoooi|k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" } - if ($lower -match "hostdown|host_down|vm_not_running|vm_stuck|vmware|reboot|restart|down" -or (Test-AgentTextContainsCode $Text @(0x91CD, 0x555F, 0x95DC, 0x6A5F))) { return "host_recovery" } + if ($lower -match "k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment") { return "awoooi_k3s" } return "monitoring_alert" } @@ -333,6 +335,52 @@ function Get-AgentTelegramAlertHost { return "" } +function Get-AgentTelegramAlertTarget { + param( + [string]$Text, + [string]$Kind, + [string]$Service, + [string]$HostName + ) + $targetMatch = [regex]::Match($Text, "(?im)\bTarget\s*[:\uFF1A]\s*([^\r\n]+)") + if ($targetMatch.Success) { + return $targetMatch.Groups[1].Value.Trim() + } + if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" } + if ($Kind -eq "host_recovery" -and $HostName) { return $HostName } + if ($Service -and $Service -ne "unknown") { return $Service } + return $Kind +} + +function Get-AgentTelegramSuggestedMode { + param([string]$Kind) + $modeByKind = @{ + "provider_freshness_signal" = "ProviderFreshness" + "backup_health" = "BackupCheck" + "wazuh_security_alert" = "SecurityTriage" + "public_route_502" = "Recover" + "host_recovery" = "Recover" + "performance_cpu" = "Perf" + "performance_memory" = "Perf" + "performance_disk" = "Perf" + "harbor_registry" = "HarborRepair" + "awoooi_k3s" = "AwoooRepair" + "database_health" = "Status" + "alert_chain_health" = "Status" + "application_ai_config_error" = "Status" + "repository_missing" = "Status" + } + if ($modeByKind.ContainsKey($Kind)) { return [string]$modeByKind[$Kind] } + return "Status" +} + +function Convert-AgentTelegramRouteToken { + param([string]$Value) + $safe = [regex]::Replace($Value.ToLowerInvariant(), "[^a-z0-9_.-]+", "_").Trim([char[]]".-_") + if ($safe) { return $safe } + return "unknown" +} + function Test-AgentTelegramSuppressPage { param([object]$Update) $text = if ($Update -and $Update.PSObject.Properties["text"]) { [string]$Update.text } else { "" } @@ -359,6 +407,15 @@ function New-AgentMonitoringAlertFromTelegram { $alertPath = Join-Path $AlertIncomingDir ($alertId + ".json") $message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text } $kind = Get-AgentTelegramAlertKind $text + $service = Get-AgentTelegramAlertService $text + $hostName = Get-AgentTelegramAlertHost $text + $target = Get-AgentTelegramAlertTarget $text $kind $service $hostName + $suggestedMode = Get-AgentTelegramSuggestedMode $kind + $correlationKey = @( + (Convert-AgentTelegramRouteToken $kind), + (Convert-AgentTelegramRouteToken $target), + $(if ($hostName) { Convert-AgentTelegramRouteToken $hostName } else { "no-host" }) + ) -join ":" $suppressTelegram = Test-AgentTelegramSuppressPage $Update $labels = if ($kind -eq "provider_freshness_signal") { @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green") @@ -373,11 +430,22 @@ function New-AgentMonitoringAlertFromTelegram { source = "telegram-sre-war-room-monitoring" severity = $severity kind = $kind - service = Get-AgentTelegramAlertService $text - host = Get-AgentTelegramAlertHost $text + suggestedMode = $suggestedMode + controlledApply = [bool]($suggestedMode -in @("Recover", "StartVMs", "HarborRepair", "AwoooRepair", "Perf", "LoadShed")) + service = $service + host = $hostName title = "AwoooI SRE Telegram monitoring alert" message = $message labels = $labels + routing = [pscustomobject]@{ + schemaVersion = "agent99_alert_route_v1" + kind = $kind + suggestedMode = $suggestedMode + routeSource = "telegram_structured_kind" + groupKey = $correlationKey + correlationKey = $correlationKey + singleFlightWindowSeconds = 300 + } telegramUpdateId = $Update.update_id telegramMessageId = $Update.message_id telegramChatTitle = $Update.chat_title @@ -490,6 +558,42 @@ function Invoke-AgentTelegramInboxSelfTest { expectedService = "gitea" expectedHost = "" }, + [pscustomobject]@{ + name = "reboot slo wins over cpu text" + text = "critical Target: reboot-auto-recovery-slo host 192.168.0.110 all_required_hosts_not_in_10_minute_reboot_window cpu pressure" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "host_recovery" + expectedService = "unknown" + expectedHost = "192.168.0.110" + }, + [pscustomobject]@{ + name = "postgres does not become k3s" + text = "critical PostgreSQL database unavailable service awoooi deployment" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "database_health" + expectedService = "awoooi" + expectedHost = "" + }, + [pscustomobject]@{ + name = "alert chain does not become k3s" + text = "critical AlertChainBroken monitoring pipeline down service awoooi" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "alert_chain_health" + expectedService = "awoooi" + expectedHost = "" + }, + [pscustomobject]@{ + name = "k3s stays awooo repair domain" + text = "critical Target: awoooi-api k3s pod crashloopbackoff deployment" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "awoooi_k3s" + expectedService = "awoooi" + expectedHost = "" + }, [pscustomobject]@{ name = "public 502" text = "critical public route 502 website https://awoooi.wooo.work" diff --git a/apps/api/src/api/v1/webhooks.py b/apps/api/src/api/v1/webhooks.py index e20a2b7d1..8ea6a7ca6 100644 --- a/apps/api/src/api/v1/webhooks.py +++ b/apps/api/src/api/v1/webhooks.py @@ -2850,22 +2850,6 @@ async def alertmanager_webhook( labels=dict(alert.labels) if alert.labels else {}, annotations=dict(alert.annotations) if alert.annotations else {}, ) - background_tasks.add_task( - bridge_alertmanager_to_agent99, - alert_id=alert_id, - alertname=alertname, - severity=severity, - namespace=namespace, - target_resource=target_resource, - message=message, - labels=dict(alert.labels) if alert.labels else {}, - annotations=dict(alert.annotations) if alert.annotations else {}, - fingerprint=fingerprint, - alert_category=alert_category, - notification_type=notification_type, - source_url=alert.generatorURL, - ) - # ========================================================================== # ADR-076: 告警聚合引擎 — 5 分鐘滑動視窗,防止告警風暴 # 2026-04-14 Claude Haiku 4.5 Asia/Taipei @@ -2920,6 +2904,25 @@ async def alertmanager_webhook( converged=True, ) + # Agent99 only receives the parent route after grouping. The bridge adds a + # Redis NX fingerprint lock so concurrent duplicate deliveries cannot race + # past this point and launch duplicate remediation. + background_tasks.add_task( + bridge_alertmanager_to_agent99, + alert_id=alert_id, + alertname=alertname, + severity=severity, + namespace=namespace, + target_resource=target_resource, + message=message, + labels=dict(alert.labels) if alert.labels else {}, + annotations=dict(alert.annotations) if alert.annotations else {}, + fingerprint=fingerprint, + alert_category=alert_category, + notification_type=notification_type, + source_url=alert.generatorURL, + ) + try: service = get_approval_service() diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 2f176506b..c6ccaf327 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import hashlib import json import re import urllib.error @@ -16,6 +17,8 @@ from src.utils.timezone import now_taipei logger = get_logger("awoooi.agent99_sre_bridge") +AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS = 600 + _SENSITIVE_KEY_RE = re.compile( r"(token|secret|password|passwd|authorization|cookie|session|private[_-]?key)", re.IGNORECASE, @@ -99,18 +102,49 @@ def resolve_agent99_alert_kind( text, ): return "provider_freshness_signal" + if re.search( + r"wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|" + r"privilege|vulnerability|\bcve\b|\bioc\b|suspicious|unauthorized", + text, + ): + return "security_health" + if re.search(r"backup|snapshot|restore drill|restore_drill|backup cron", text): + return "backup_health" + if re.search( + r"postgres|postgresql|mysql|mariadb|database|db_connection|" + r"pg_isready|connection pool|replication lag", + text, + ): + return "database_health" + if re.search( + r"alertchain|alert_chain|alert chain|alertmanager.*(broken|down|failed)|" + r"monitoring pipeline|notification pipeline", + text, + ): + return "alert_chain_health" + if re.search( + r"reboot-auto-recovery|reboot_auto_recovery|reboot recovery|" + r"host_down|hostdown|host_reboot|vm_not_running|powered_off|" + r"all_required_hosts_not_in_10_minute_reboot_window", + text, + ): + return "host_recovery" if re.search(r"502|site_502|public_route|website|route_down|nginx upstream", text): return "public_route_502" if re.search(r"cpu|load|memory|mem_|disk|pressure|performance", text): return "performance_pressure" - if re.search(r"backup|snapshot|restore drill|cron", text): - return "backup_health" if re.search(r"harbor|registry", text): return "harbor_registry" - if re.search(r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text): + if re.search( + r"k3s|imagepullbackoff|errimagepull|crashloopbackoff|pod|deployment", text + ): return "awoooi_k3s" - if re.search(r"host_down|hostdown|vm_not_running|reboot|restart|powered_off", text): - return "host_recovery" + if re.search( + r"missing_key|e_ai_missing_key|application_ai_config_error|" + r"locale guard|ai support", + text, + ): + return "application_ai_config_error" return "monitoring_alert" @@ -120,12 +154,95 @@ def _suggested_mode_for_kind(kind: str) -> str | None: "public_route_502": "Recover", "performance_pressure": "Perf", "backup_health": "BackupCheck", + "security_health": "SecurityTriage", + "database_health": "Status", + "alert_chain_health": "Status", "harbor_registry": "HarborRepair", "awoooi_k3s": "AwoooRepair", "host_recovery": "Recover", + "application_ai_config_error": "Status", }.get(kind) +def _agent99_route_group_key( + *, + kind: str, + target_resource: str, + host: str, +) -> str: + parts = ( + kind.strip().lower() or "monitoring_alert", + target_resource.strip().lower() or "unknown", + host.strip().lower() or "no-host", + ) + return ":".join(_safe_file_token(part) for part in parts) + + +def agent99_sre_single_flight_key(fingerprint: str) -> str: + digest = hashlib.sha256(fingerprint.encode("utf-8")).hexdigest() + return f"agent99:sre_dispatch:single_flight:{digest}" + + +async def try_acquire_agent99_sre_single_flight( + fingerprint: str, + alert_id: str, + *, + ttl_seconds: int = AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS, +) -> bool: + """Grant one Agent99 dispatch per fingerprint inside the controlled window.""" + + if not fingerprint: + return True + try: + from src.core.redis_client import get_redis + + redis = get_redis() + acquired = await redis.set( + agent99_sre_single_flight_key(fingerprint), + alert_id, + ex=ttl_seconds, + nx=True, + ) + return bool(acquired) + except Exception as exc: + logger.warning( + "agent99_sre_single_flight_failed_fail_open", + fingerprint=fingerprint, + alert_id=alert_id, + error=str(exc), + ) + return True + + +async def release_agent99_sre_single_flight(fingerprint: str, alert_id: str) -> None: + """Release only the caller-owned lock when transport did not accept the alert.""" + + if not fingerprint: + return + try: + from src.core.redis_client import get_redis + + redis = get_redis() + await redis.eval( + """ + if redis.call('get', KEYS[1]) == ARGV[1] then + return redis.call('del', KEYS[1]) + end + return 0 + """, + 1, + agent99_sre_single_flight_key(fingerprint), + alert_id, + ) + except Exception as exc: + logger.warning( + "agent99_sre_single_flight_release_failed", + fingerprint=fingerprint, + alert_id=alert_id, + error=str(exc), + ) + + def build_agent99_sre_alert( *, alert_id: str, @@ -164,6 +281,11 @@ def build_agent99_sre_alert( or target_resource or "awoooi" ) + route_group_key = _agent99_route_group_key( + kind=kind, + target_resource=target_resource, + host=host, + ) safe_labels = [ "alertmanager", "awoooi-api", @@ -172,9 +294,13 @@ def build_agent99_sre_alert( f"target={_safe_label_value('target', target_resource)}", ] if fingerprint: - safe_labels.append(f"fingerprint={_safe_label_value('fingerprint', fingerprint)}") + safe_labels.append( + f"fingerprint={_safe_label_value('fingerprint', fingerprint)}" + ) for key, value in sorted(labels.items()): - safe_labels.append(f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}") + safe_labels.append( + f"{_safe_text(key, max_length=80)}={_safe_label_value(key, value)}" + ) if kind == "provider_freshness_signal": safe_labels.extend(["provider-freshness", "no-false-green"]) @@ -197,8 +323,26 @@ def build_agent99_sre_alert( "message": _safe_text(message, max_length=2000), "labels": safe_labels, "force": True, - "controlledApply": True, + "controlledApply": suggested_mode + in { + "Recover", + "StartVMs", + "HarborRepair", + "AwoooRepair", + "Perf", + "LoadShed", + }, "createdAt": now_taipei().isoformat(), + "routing": { + "schemaVersion": "agent99_alert_route_v1", + "kind": kind, + "suggestedMode": suggested_mode or "Status", + "routeSource": "structured_kind", + "groupKey": route_group_key, + "correlationKey": route_group_key, + "sourceFingerprint": _safe_text(fingerprint, max_length=160), + "singleFlightWindowSeconds": AGENT99_SRE_SINGLE_FLIGHT_TTL_SECONDS, + }, "awoooi": { "alertId": _safe_text(alert_id, max_length=160), "alertname": _safe_text(alertname, max_length=160), @@ -222,7 +366,9 @@ def write_agent99_sre_alert( *, inbox_path: str | None = None, ) -> Path | None: - target_dir_text = inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH + target_dir_text = ( + inbox_path if inbox_path is not None else settings.AGENT99_SRE_ALERT_INBOX_PATH + ) if not target_dir_text: logger.info( "agent99_sre_bridge_disabled", @@ -258,11 +404,17 @@ def post_agent99_sre_alert( relay_token: str | None = None, timeout_seconds: float | None = None, ) -> dict[str, Any] | None: - target_url = relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL + target_url = ( + relay_url if relay_url is not None else settings.AGENT99_SRE_ALERT_RELAY_URL + ) if not target_url: return None - token = relay_token if relay_token is not None else settings.AGENT99_SRE_ALERT_RELAY_TOKEN + token = ( + relay_token + if relay_token is not None + else settings.AGENT99_SRE_ALERT_RELAY_TOKEN + ) timeout = ( timeout_seconds if timeout_seconds is not None @@ -276,7 +428,9 @@ def post_agent99_sre_alert( if token: headers["X-Agent99-Relay-Token"] = token - request = urllib.request.Request(target_url, data=data, headers=headers, method="POST") + request = urllib.request.Request( + target_url, data=data, headers=headers, method="POST" + ) try: with urllib.request.urlopen(request, timeout=timeout) as response: status = int(response.getcode()) @@ -331,7 +485,9 @@ async def bridge_alertmanager_to_agent99( alert_category: str = "", notification_type: str = "", source_url: str | None = None, -) -> None: +) -> dict[str, Any]: + single_flight_fingerprint = "" + single_flight_acquired = False try: payload = build_agent99_sre_alert( alert_id=alert_id, @@ -347,11 +503,55 @@ async def bridge_alertmanager_to_agent99( notification_type=notification_type, source_url=source_url, ) - await asyncio.to_thread(dispatch_agent99_sre_alert, payload) + routing = ( + payload.get("routing") if isinstance(payload.get("routing"), dict) else {} + ) + single_flight_fingerprint = fingerprint or str( + routing.get("correlationKey") or "" + ) + single_flight_acquired = await try_acquire_agent99_sre_single_flight( + single_flight_fingerprint, + alert_id, + ) + if not single_flight_acquired: + logger.info( + "agent99_sre_dispatch_single_flight_suppressed", + alert_id=alert_id, + alertname=alertname, + fingerprint=single_flight_fingerprint, + group_key=routing.get("groupKey"), + ) + return { + "status": "suppressed", + "reason": "single_flight_duplicate", + "fingerprint": single_flight_fingerprint, + "groupKey": routing.get("groupKey"), + } + dispatch_result = await asyncio.to_thread(dispatch_agent99_sre_alert, payload) + if dispatch_result == "disabled": + await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id) + single_flight_acquired = False + return { + "status": "failed", + "reason": "bridge_disabled", + "fingerprint": single_flight_fingerprint, + "groupKey": routing.get("groupKey"), + } + return { + "status": "dispatched", + "dispatch": dispatch_result, + "fingerprint": single_flight_fingerprint, + "groupKey": routing.get("groupKey"), + "kind": payload.get("kind"), + "suggestedMode": payload.get("suggestedMode"), + } except Exception as exc: + if single_flight_acquired: + await release_agent99_sre_single_flight(single_flight_fingerprint, alert_id) logger.warning( "agent99_sre_bridge_failed_fail_open", alert_id=alert_id, alertname=alertname, error=str(exc), ) + return {"status": "failed", "reason": type(exc).__name__} diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py new file mode 100644 index 000000000..f905a8705 --- /dev/null +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -0,0 +1,61 @@ +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +SRE_INBOX = ROOT / "agent99-sre-alert-inbox.ps1" +TELEGRAM_INBOX = ROOT / "agent99-telegram-inbox.ps1" + + +def test_alertmanager_groups_before_agent99_dispatch() -> None: + source = (ROOT / "apps/api/src/api/v1/webhooks.py").read_text(encoding="utf-8") + normalized_at = source.index('"alertmanager_normalized"') + grouping_at = source.index("get_alert_grouping_service().evaluate", normalized_at) + grouped_return_at = source.index( + "if grouping_result and grouping_result.is_grouped", grouping_at + ) + dispatch_at = source.index("bridge_alertmanager_to_agent99", grouped_return_at) + + assert grouping_at < grouped_return_at < dispatch_at + + +def test_agent99_bridge_has_fingerprint_single_flight_before_transport() -> None: + source = (ROOT / "apps/api/src/services/agent99_sre_bridge.py").read_text( + encoding="utf-8" + ) + bridge_at = source.index("async def bridge_alertmanager_to_agent99") + lock_at = source.index("try_acquire_agent99_sre_single_flight", bridge_at) + dispatch_at = source.index("dispatch_agent99_sre_alert, payload", lock_at) + + assert lock_at < dispatch_at + + +def test_agent99_runtime_prioritizes_structured_route_over_text() -> None: + source = SRE_INBOX.read_text(encoding="utf-8") + route_at = source.index("function Resolve-AgentAlertRoute") + suggested_at = source.index("if ($suggestedMode", route_at) + kind_at = source.index("if ($kind -and $kindModes.ContainsKey", suggested_at) + fallback_at = source.index('if ($text -match "502', kind_at) + + assert suggested_at < kind_at < fallback_at + assert '"database_health" = "Status"' in source + assert '"alert_chain_health" = "Status"' in source + assert '"host_recovery" = "Recover"' in source + + +def test_agent99_runtime_has_local_correlation_dedupe_and_lock() -> None: + source = SRE_INBOX.read_text(encoding="utf-8") + + assert '"sre-alert-single-flight.json"' in source + assert "[System.IO.FileShare]::None" in source + assert 'reason = "duplicate_single_flight"' in source + assert "severity_escalation_dispatch" in source + assert "correlationKey = $correlationKey" in source + + +def test_telegram_ingress_writes_same_structured_route_contract() -> None: + source = TELEGRAM_INBOX.read_text(encoding="utf-8") + + assert "function Get-AgentTelegramSuggestedMode" in source + assert 'schemaVersion = "agent99_alert_route_v1"' in source + assert 'routeSource = "telegram_structured_kind"' in source + assert '"database_health" = "Status"' in source + assert '"alert_chain_health" = "Status"' in source diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index e62023656..336bb6997 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -2,9 +2,14 @@ from __future__ import annotations import json +import pytest + from src.services.agent99_sre_bridge import ( + agent99_sre_single_flight_key, + bridge_alertmanager_to_agent99, build_agent99_sre_alert, post_agent99_sre_alert, + release_agent99_sre_single_flight, resolve_agent99_alert_kind, write_agent99_sre_alert, ) @@ -37,13 +42,99 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None: assert payload["source"] == "awoooi-api-alertmanager" assert payload["kind"] == "provider_freshness_signal" assert payload["suggestedMode"] == "ProviderFreshness" - assert payload["controlledApply"] is True + assert payload["controlledApply"] is False assert payload["force"] is True assert "provider-freshness" in payload["labels"] assert "no-false-green" in payload["labels"] assert "token=[redacted]" in payload["labels"] assert "must-not-leak" not in json.dumps(payload) assert "do not switch provider" in payload["instruction"] + assert payload["routing"]["routeSource"] == "structured_kind" + assert payload["routing"]["correlationKey"] == payload["routing"]["groupKey"] + + +@pytest.mark.parametrize( + ("alertname", "target", "message", "labels", "expected_kind", "expected_mode"), + [ + ( + "HostBackupFailed", + "backup", + "snapshot stale", + {}, + "backup_health", + "BackupCheck", + ), + ( + "provider_freshness_signal", + "provider_freshness_signal", + "last_seen missing", + {}, + "provider_freshness_signal", + "ProviderFreshness", + ), + ( + "RebootAutoRecoveryActiveBlocker", + "reboot-auto-recovery-slo", + "all_required_hosts_not_in_10_minute_reboot_window cpu pressure", + {"service": "awoooi"}, + "host_recovery", + "Recover", + ), + ( + "PostgresqlDown", + "postgresql", + "database unavailable deployment awoooi", + {"service": "awoooi"}, + "database_health", + "Status", + ), + ( + "AlertChainBroken", + "alertmanager", + "monitoring pipeline down awoooi", + {"service": "awoooi"}, + "alert_chain_health", + "Status", + ), + ( + "HostDown", + "192.168.0.120", + "host_down vm_not_running", + {"host": "192.168.0.120"}, + "host_recovery", + "Recover", + ), + ( + "KubePodCrashLooping", + "awoooi-api", + "k3s pod crashloopbackoff", + {"service": "awoooi"}, + "awoooi_k3s", + "AwoooRepair", + ), + ], +) +def test_agent99_routing_replay_corpus( + alertname: str, + target: str, + message: str, + labels: dict[str, str], + expected_kind: str, + expected_mode: str, +) -> None: + payload = build_agent99_sre_alert( + alert_id=f"replay-{alertname}", + alertname=alertname, + severity="critical", + namespace="awoooi-prod", + target_resource=target, + message=message, + labels=labels, + fingerprint=f"fp-{alertname}", + ) + + assert payload["kind"] == expected_kind + assert payload["suggestedMode"] == expected_mode def test_alert_kind_mapping_for_route_and_performance() -> None: @@ -108,7 +199,7 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: def read(self, _limit: int) -> bytes: return b'{"ok":true}' - def __enter__(self) -> "FakeResponse": + def __enter__(self) -> FakeResponse: return self def __exit__(self, *_args: object) -> None: @@ -141,3 +232,155 @@ def test_post_agent99_sre_alert_to_relay(monkeypatch) -> None: assert "authorization=[redacted]" in seen["payload"]["labels"] assert "must-not-leak" not in json.dumps(seen["payload"]) assert seen["headers"]["X-agent99-relay-token"] == "relay-token" + + +def test_agent99_single_flight_key_does_not_expose_fingerprint() -> None: + key = agent99_sre_single_flight_key("private/source/fingerprint") + + assert key.startswith("agent99:sre_dispatch:single_flight:") + assert "private" not in key + + +@pytest.mark.asyncio +async def test_agent99_bridge_suppresses_duplicate_single_flight(monkeypatch) -> None: + async def duplicate(*_args, **_kwargs) -> bool: + return False + + dispatched: list[dict[str, object]] = [] + monkeypatch.setattr( + "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", + duplicate, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert", + lambda payload: dispatched.append(payload) or "relay", + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="duplicate-alert", + alertname="HostDown", + severity="critical", + namespace="node", + target_resource="192.168.0.120", + message="host_down", + fingerprint="same-fingerprint", + ) + + assert result["status"] == "suppressed" + assert result["reason"] == "single_flight_duplicate" + assert dispatched == [] + + +@pytest.mark.asyncio +async def test_agent99_bridge_dispatches_single_flight_winner(monkeypatch) -> None: + async def winner(*_args, **_kwargs) -> bool: + return True + + dispatched: list[dict[str, object]] = [] + monkeypatch.setattr( + "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", + winner, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert", + lambda payload: dispatched.append(payload) or "relay", + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="winner-alert", + alertname="HostDown", + severity="critical", + namespace="node", + target_resource="192.168.0.120", + message="host_down", + fingerprint="winner-fingerprint", + ) + + assert result["status"] == "dispatched" + assert result["dispatch"] == "relay" + assert result["suggestedMode"] == "Recover" + assert len(dispatched) == 1 + + +@pytest.mark.asyncio +async def test_agent99_bridge_releases_lock_when_transport_fails(monkeypatch) -> None: + async def winner(*_args, **_kwargs) -> bool: + return True + + released: list[tuple[str, str]] = [] + + async def release(fingerprint: str, alert_id: str) -> None: + released.append((fingerprint, alert_id)) + + def fail_transport(_payload: dict[str, object]) -> str: + raise RuntimeError("relay unavailable") + + monkeypatch.setattr( + "src.services.agent99_sre_bridge.try_acquire_agent99_sre_single_flight", + winner, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.release_agent99_sre_single_flight", + release, + ) + monkeypatch.setattr( + "src.services.agent99_sre_bridge.dispatch_agent99_sre_alert", + fail_transport, + ) + + result = await bridge_alertmanager_to_agent99( + alert_id="transport-failure", + alertname="HostDown", + severity="critical", + namespace="node", + target_resource="192.168.0.120", + message="host_down", + fingerprint="transport-fingerprint", + ) + + assert result == {"status": "failed", "reason": "RuntimeError"} + assert released == [("transport-fingerprint", "transport-failure")] + + +@pytest.mark.asyncio +async def test_agent99_single_flight_uses_redis_nx(monkeypatch) -> None: + seen: dict[str, object] = {} + + class FakeRedis: + async def set(self, key, value, *, ex, nx): # type: ignore[no-untyped-def] + seen.update({"key": key, "value": value, "ex": ex, "nx": nx}) + return True + + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) + + from src.services.agent99_sre_bridge import try_acquire_agent99_sre_single_flight + + acquired = await try_acquire_agent99_sre_single_flight( + "source-fingerprint", "alert-1" + ) + + assert acquired is True + assert str(seen["key"]).startswith("agent99:sre_dispatch:single_flight:") + assert seen["value"] == "alert-1" + assert seen["ex"] == 600 + assert seen["nx"] is True + + +@pytest.mark.asyncio +async def test_agent99_single_flight_release_is_owner_checked(monkeypatch) -> None: + seen: dict[str, object] = {} + + class FakeRedis: + async def eval(self, script, key_count, key, owner): # type: ignore[no-untyped-def] + seen.update( + {"script": script, "key_count": key_count, "key": key, "owner": owner} + ) + return 1 + + monkeypatch.setattr("src.core.redis_client.get_redis", lambda: FakeRedis()) + + await release_agent99_sre_single_flight("source-fingerprint", "alert-1") + + assert "redis.call('get'" in str(seen["script"]) + assert seen["key_count"] == 1 + assert seen["owner"] == "alert-1" diff --git a/apps/api/tests/test_telegram_gateway_error_sanitizer.py b/apps/api/tests/test_telegram_gateway_error_sanitizer.py index b99f09674..5d9f27dfb 100644 --- a/apps/api/tests/test_telegram_gateway_error_sanitizer.py +++ b/apps/api/tests/test_telegram_gateway_error_sanitizer.py @@ -2,9 +2,9 @@ from __future__ import annotations from src.services.telegram_gateway import ( _agent99_alert_from_ai_automation_card, - format_aiops_signal_alert_card, _outbound_source_envelope, _sanitize_telegram_error, + format_aiops_signal_alert_card, ) @@ -204,7 +204,8 @@ Authorization: Bearer abcdefghijklmnopqrstuvwxyz ) assert payload["kind"] == "provider_freshness_signal" assert payload["suggestedMode"] == "ProviderFreshness" - assert payload["controlledApply"] is True + assert payload["controlledApply"] is False + assert payload["routing"]["routeSource"] == "structured_kind" assert payload["awoooi"]["alertCategory"] == "source_provider_freshness_triage" assert payload["awoooi"]["notificationType"] == "ai_automation_alert_card_v1" assert "component=telegram_gateway" in payload["labels"]