fix(agent99): render telegram incident cards

This commit is contained in:
ogt
2026-07-10 10:55:50 +08:00
parent f3c5d5a9f9
commit 23c1cda688
2 changed files with 304 additions and 54 deletions

View File

@@ -456,6 +456,14 @@ Recovery is complete only when all of the following are captured:
- P0-5: route IWOOOS/Wazuh/SIEM alerts to `SecurityTriage`; collect source log reference, affected host/service, blast radius, repeated count, and allowlisted next action without arbitrary shell or secret exposure.
- P0-6: route Gitea repository missing alerts to evidence-first `Status` until a dedicated Gitea recovery playbook is verified.
- P0-7: run live TG-to-Agent99 end-to-end tests for each alert family before re-enabling the paused sensor schedules.
- 2026-07-10 10:54 visual card verifier:
- Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`.
- Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`.
- Parser readback: `parse_ok`.
- Self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramCardSelfTest-20260710-105440.json`.
- Visual card: `C:\Wooo\Agent99\evidence\telegram-cards\agent99-card-performance_warning-20260710-105440.png`.
- Result: `ok=True`, `visualExists=True`, `sentTelegram=False`, `messageFormat=incident_card_v2`.
- Local visual inspection: PNG is `1040x620`, non-blank, formatted as an incident card.
## Agent99 Monitoring Alert Routing Self-Test Receipt

View File

@@ -2,6 +2,7 @@ param(
[ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
[string]$Mode = "Status",
[switch]$ControlledApply,
[switch]$SelfTestTelegramCard,
[string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json",
[string]$EvidenceDir = "C:\Wooo\Agent99\evidence"
)
@@ -85,6 +86,43 @@ print("sent_ok")
}
}
function Get-AgentObjectValue {
param(
[object]$Data,
[string]$Name,
[object]$Default = $null
)
if ($Data -and $Data.PSObject.Properties[$Name]) {
$value = $Data.PSObject.Properties[$Name].Value
if ($null -ne $value -and [string]$value -ne "") {
return $value
}
}
$Default
}
function Limit-AgentTextLine {
param(
[string]$Text,
[int]$MaxLength = 96
)
if (-not $Text) { return "" }
$clean = (($Text -replace "\s+", " ").Trim())
if ($clean.Length -le $MaxLength) { return $clean }
$clean.Substring(0, $MaxLength - 14) + "...(truncated)"
}
function Format-AgentMetricForCard {
param(
[object]$Value,
[string]$Suffix = ""
)
if ($null -eq $Value -or [string]$Value -eq "") {
return "readback_failed"
}
"$Value$Suffix"
}
function Format-AgentTelegramText {
param(
[string]$Severity,
@@ -100,33 +138,46 @@ function Format-AgentTelegramText {
default { $Severity }
}
$lines = @()
$lines += "Agent99 Alert: $severityLabel"
$lines += "Agent99 Incident Card"
$lines += "Severity: $severityLabel"
if ($EventType -match "^performance_") {
$hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" }
$hostName = Get-AgentObjectValue $Data "host" "unknown"
$reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() }
$disk = if ($Data -and $Data.PSObject.Properties["diskUsedPercent"]) { $Data.diskUsedPercent } else { $null }
$load = if ($Data -and $Data.PSObject.Properties["loadPerCore"]) { $Data.loadPerCore } else { $null }
$mem = if ($Data -and $Data.PSObject.Properties["memAvailablePercent"]) { $Data.memAvailablePercent } else { $null }
$disk = Get-AgentObjectValue $Data "diskUsedPercent" $null
$load = Get-AgentObjectValue $Data "loadPerCore" $null
$mem = Get-AgentObjectValue $Data "memAvailablePercent" $null
$lines += "Event: host performance"
$lines += "Host: $hostName"
if ($reasons -contains "disk_used_warning") {
if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) {
$lines += "Reason: performance metric readback failed."
$lines += "Impact: load, memory, and disk values are not trustworthy yet."
$lines += "Agent99 action: keep noisy sensor paused; require verifier readback before remediation."
} elseif ($reasons -contains "disk_used_warning") {
$lines += "Reason: disk usage is $disk percent. Warning threshold reached; critical threshold is 95 percent."
$lines += "Action: Agent99 will run allowlisted disk remediation and suppress duplicates for 30 minutes."
$lines += "Agent99 action: run allowlisted disk remediation and suppress duplicates for 30 minutes."
} elseif ($reasons -contains "disk_used_high") {
$lines += "Reason: disk usage is $disk percent. Critical threshold reached."
$lines += "Action: Agent99 will run allowlisted cleanup first; reboot is not the first response."
$lines += "Agent99 action: run allowlisted cleanup first; reboot is not the first response."
} elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") {
$lines += "Reason: CPU load/core=$load."
$lines += "Action: Agent99 will use allowlisted load reduction only; no arbitrary service stop."
$lines += "Agent99 action: use allowlisted load reduction only; no arbitrary service stop."
} elseif ($reasons -contains "memory_available_low") {
$lines += "Reason: available memory is $mem percent."
$lines += "Action: Agent99 will inspect memory pressure before allowlisted remediation."
$lines += "Agent99 action: inspect memory pressure before allowlisted remediation."
} else {
$lines += "Reason: $($reasons -join ', ')"
$lines += "Action: check Agent99 evidence for full readback."
$lines += "Agent99 action: check Agent99 evidence for full readback."
}
$metricParts = @()
if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" }
if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" }
if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" }
if ($metricParts.Count -gt 0) {
$lines += "Current: $($metricParts -join ', ')."
} else {
$lines += "Current: metrics unavailable; no blank metric alert is trusted."
}
$lines += "Current: load/core=$load, memAvail=$mem percent, disk=$disk percent."
} elseif ($EventType -match "^backup_") {
$targetName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
$reason = if ($Data -and $Data.PSObject.Properties["reason"]) { $Data.reason } else { $EventType }
@@ -138,7 +189,7 @@ function Format-AgentTelegramText {
if ($age -ne $null -or $maxAge -ne $null) {
$lines += "Freshness: age=$age minutes / max=$maxAge minutes"
}
$lines += "Action: inspect /backup/status, /backup/logs, and snapshot metadata only; do not read backup contents or secrets."
$lines += "Agent99 action: inspect /backup/status, /backup/logs, and snapshot metadata only; do not read backup contents or secrets."
} elseif ($EventType -eq "provider_freshness_triage") {
$candidateId = if ($Data -and $Data.PSObject.Properties["candidateId"]) { $Data.candidateId } else { "unknown" }
$targetName = if ($Data -and $Data.PSObject.Properties["target"]) { $Data.target } else { "provider_freshness_signal" }
@@ -150,16 +201,17 @@ function Format-AgentTelegramText {
$lines += "Event: provider freshness auto triage"
$lines += "Target: $targetName"
$lines += "Status: $status"
$lines += "Impact: dashboard green is not trusted until source freshness is proven."
$lines += "Evidence hits: $hitCount"
if ($readback) {
$lines += "Direct readback: ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)"
$lines += "Verifier: directReadback ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)"
}
if (@($missing).Count -gt 0) {
$lines += "Still missing: $($missing -join ', ')"
} else {
$lines += "Missing: direct readback fields completed"
}
$lines += "Action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed."
$lines += "Agent99 action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed."
$lines += "Guard: no provider switch, paid model call, env change, or reload."
if ($candidatePath) {
$lines += "Evidence: $candidatePath"
@@ -174,7 +226,7 @@ function Format-AgentTelegramText {
$rollback = if ($Data -and $Data.PSObject.Properties["rollback"]) { $Data.rollback } else { $null }
$lines += "Event: controlled remediation"
$lines += "Host: $hostName"
$lines += "Action: $actionName"
$lines += "Agent99 action: $actionName"
if ($ok -ne $null) {
$lines += "Result: ok=$ok exitCode=$exitCode"
if ($executionOk -ne $null -or $postVerifyOk -ne $null) {
@@ -220,7 +272,7 @@ function Format-AgentTelegramText {
$serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" }
$lines += "Event: AI service health"
$lines += "Service: $serviceName"
$lines += "Action: inspect health endpoint and service/container state before controlled repair."
$lines += "Agent99 action: inspect health endpoint and service/container state before controlled repair."
} elseif ($EventType -eq "security_triage") {
$status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" }
$hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 }
@@ -228,7 +280,7 @@ function Format-AgentTelegramText {
$lines += "Event: security triage"
$lines += "Status: $status"
$lines += "Evidence hits: $hitCount"
$lines += "Action: classify blast radius, affected host/service, source log reference, repeated count, and allowlisted next action."
$lines += "Agent99 action: classify blast radius, affected host/service, source log reference, repeated count, and allowlisted next action."
$lines += "Guard: no arbitrary shell, no data deletion, no disabling controls, no secret exposure."
if ($evidencePath) {
$lines += "Evidence: $evidencePath"
@@ -236,7 +288,7 @@ function Format-AgentTelegramText {
} elseif ($EventType -eq "agent_selfcheck") {
$lines += "Event: Agent99 self health"
$lines += "Summary: $Message"
$lines += "Action: inspect task state, evidence freshness, and Telegram delivery; reboot is not the first response."
$lines += "Agent99 action: inspect task state, evidence freshness, and Telegram delivery; reboot is not the first response."
} else {
$lines += "Event: $EventType"
$lines += "Summary: $Message"
@@ -250,6 +302,162 @@ function Format-AgentTelegramText {
$text
}
function Get-AgentTelegramVisualPath {
param([object]$Data)
if (-not $Data) { return $null }
foreach ($propName in @("visualPath", "cardImagePath", "imagePath", "screenshotPath")) {
if ($Data.PSObject.Properties[$propName]) {
$path = [string]$Data.PSObject.Properties[$propName].Value
if ($path -and (Test-Path -LiteralPath $path)) {
return $path
}
}
}
$null
}
function New-AgentTelegramCardImage {
param(
[string]$Severity,
[string]$EventType,
[string]$Text
)
$visualCardsEnabled = $true
if ($Config.telegram -and $Config.telegram.PSObject.Properties["visualCardsEnabled"]) {
$visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled
}
if (-not $visualCardsEnabled) { return $null }
if ($Severity -notin @("warning", "critical")) { return $null }
$cardDir = Join-Path $EvidenceDir "telegram-cards"
New-Item -ItemType Directory -Force -Path $cardDir | Out-Null
$safeEvent = ($EventType -replace "[^A-Za-z0-9_.-]", "-")
$imagePath = Join-Path $cardDir "agent99-card-$safeEvent-$stamp.png"
try {
Add-Type -AssemblyName System.Drawing -ErrorAction Stop
$width = 1040
$height = 620
$bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height
$graphics = [System.Drawing.Graphics]::FromImage($bitmap)
$graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias
$graphics.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::ClearTypeGridFit
$bgBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(246, 248, 250))
$panelBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(255, 255, 255))
$textBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(28, 35, 43))
$mutedBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(91, 103, 112))
$accentColor = if ($Severity -eq "critical") {
[System.Drawing.Color]::FromArgb(190, 18, 60)
} elseif ($Severity -eq "warning") {
[System.Drawing.Color]::FromArgb(217, 119, 6)
} else {
[System.Drawing.Color]::FromArgb(37, 99, 235)
}
$accentBrush = New-Object System.Drawing.SolidBrush -ArgumentList $accentColor
$titleFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 25, ([System.Drawing.FontStyle]::Bold)
$labelFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 13, ([System.Drawing.FontStyle]::Bold)
$bodyFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 16, ([System.Drawing.FontStyle]::Regular)
$smallFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 11, ([System.Drawing.FontStyle]::Regular)
$graphics.FillRectangle($bgBrush, 0, 0, $width, $height)
$graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68)
$graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68)
$graphics.DrawString("Agent99 Incident Card", $titleFont, $textBrush, 72, 58)
$graphics.DrawString(("Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106)
$graphics.DrawString(("Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134)
$bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 Incident Card$" } | Select-Object -First 12)
$y = 184
foreach ($line in $bodyLines) {
$graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y)
$y += 32
if ($y -gt 545) { break }
}
$graphics.DrawString(("Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570)
$bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png)
$graphics.Dispose()
$bitmap.Dispose()
$bgBrush.Dispose()
$panelBrush.Dispose()
$textBrush.Dispose()
$mutedBrush.Dispose()
$accentBrush.Dispose()
$titleFont.Dispose()
$labelFont.Dispose()
$bodyFont.Dispose()
$smallFont.Dispose()
return $imagePath
} catch {
Write-AgentLog "telegram_card_image_failed event=$EventType error=$($_.Exception.Message)"
return $null
}
}
function Send-AgentTelegramPhotoDirect {
param(
[string]$Token,
[string]$ChatId,
[string]$Caption,
[string]$PhotoPath
)
Add-Type -AssemblyName System.Net.Http -ErrorAction Stop
$client = New-Object System.Net.Http.HttpClient
$content = New-Object System.Net.Http.MultipartFormDataContent
$fileContent = $null
try {
$content.Add([System.Net.Http.StringContent]::new($ChatId), "chat_id")
$captionToSend = $Caption
if ($captionToSend.Length -gt 900) {
$captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)"
}
$content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption")
$fileBytes = [IO.File]::ReadAllBytes($PhotoPath)
$fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes)
$fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png")
$content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath))
$response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult()
$body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult()
if (-not $response.IsSuccessStatusCode) {
throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body"
}
} finally {
if ($fileContent) { $fileContent.Dispose() }
if ($content) { $content.Dispose() }
if ($client) { $client.Dispose() }
}
}
function Invoke-AgentTelegramCardSelfTest {
$sample = [pscustomobject]@{
host = "192.168.0.110"
reasons = @("perf_readback_failed")
evidencePath = $jsonPath
}
$text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample
$imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text
$ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath))
$result = [pscustomobject]@{
timestamp = (Get-Date -Format o)
ok = $ok
textPreview = @($text -split "`n" | Select-Object -First 12)
visualPath = $imagePath
visualExists = $ok
sentTelegram = $false
messageFormat = "incident_card_v2"
}
$selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json"
$result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8
Write-AgentLog "telegram_card_selftest ok=$ok path=$imagePath evidence=$selfTestPath"
$result | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath
$result | ConvertTo-Json -Depth 6
if (-not $ok) { exit 1 }
exit 0
}
function Send-AgentTelegram {
param(
[string]$Severity,
@@ -286,49 +494,79 @@ function Send-AgentTelegram {
$chatId = if ($chatSecret) { $chatSecret.value } else { $null }
$chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" }
$attempt = [pscustomobject]@{
timestamp = (Get-Date -Format o)
enabled = [bool]$enabled
eventType = $EventType
severity = $Severity
configured = [bool]($token -and $chatId)
tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null }
chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null }
chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null }
relay = $null
sent = $false
error = $null
}
$attempt = [pscustomobject]@{
timestamp = (Get-Date -Format o)
enabled = [bool]$enabled
eventType = $EventType
severity = $Severity
messageFormat = "incident_card_v2"
configured = [bool]($token -and $chatId)
tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null }
chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null }
chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null }
visualPath = $null
visualSent = $false
visualError = $null
relay = $null
sent = $false
error = $null
}
if (-not $enabled) {
$attempt.error = "telegram_disabled"
$script:TelegramAttempts += $attempt
return
}
$text = Format-AgentTelegramText $Severity $EventType $Message $Data
if (-not ($token -and ($chatId -or $chatIdOverride))) {
$relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride
$attempt.relay = $relayResult
if ($relayResult.ok) {
$attempt.sent = $true
$attempt.error = $null
} else {
$attempt.error = "telegram_not_configured"
}
}
$text = Format-AgentTelegramText $Severity $EventType $Message $Data
$visualPath = Get-AgentTelegramVisualPath $Data
if (-not $visualPath) {
$visualPath = New-AgentTelegramCardImage $Severity $EventType $text
}
$attempt.visualPath = $visualPath
if (-not ($token -and ($chatId -or $chatIdOverride))) {
$relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride
$attempt.relay = $relayResult
if ($relayResult.ok) {
$attempt.sent = $true
$attempt.error = $null
if ($visualPath) {
$attempt.visualError = "visual_card_generated_but_relay_sendPhoto_not_available"
}
} else {
$attempt.error = "telegram_not_configured"
}
$script:TelegramAttempts += $attempt
return
}
}
try {
Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{
chat_id = if ($chatIdOverride) { $chatIdOverride } else { $chatId }
text = $text
disable_web_page_preview = "true"
} | Out-Null
$attempt.sent = $true
} catch {
$attempt.error = $_.Exception.Message
}
try {
$targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId }
if ($visualPath -and (Test-Path -LiteralPath $visualPath)) {
try {
Send-AgentTelegramPhotoDirect $token $targetChat $text $visualPath
$attempt.sent = $true
$attempt.visualSent = $true
} catch {
$attempt.visualError = $_.Exception.Message
Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)"
Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{
chat_id = $targetChat
text = $text
disable_web_page_preview = "true"
} | Out-Null
$attempt.sent = $true
}
} else {
Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{
chat_id = $targetChat
text = $text
disable_web_page_preview = "true"
} | Out-Null
$attempt.sent = $true
}
} catch {
$attempt.error = $_.Exception.Message
}
$script:TelegramAttempts += $attempt
}
@@ -2526,6 +2764,10 @@ function Repair-Awooo {
return $true
}
if ($SelfTestTelegramCard) {
Invoke-AgentTelegramCardSelfTest
}
Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath"
Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert