diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index 3afd27070..7ffa9190a 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -456,6 +456,14 @@ Recovery is complete only when all of the following are captured: - P0-5: route IWOOOS/Wazuh/SIEM alerts to `SecurityTriage`; collect source log reference, affected host/service, blast radius, repeated count, and allowlisted next action without arbitrary shell or secret exposure. - P0-6: route Gitea repository missing alerts to evidence-first `Status` until a dedicated Gitea recovery playbook is verified. - P0-7: run live TG-to-Agent99 end-to-end tests for each alert family before re-enabling the paused sensor schedules. +- 2026-07-10 10:54 visual card verifier: + - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. + - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`. + - Parser readback: `parse_ok`. + - Self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramCardSelfTest-20260710-105440.json`. + - Visual card: `C:\Wooo\Agent99\evidence\telegram-cards\agent99-card-performance_warning-20260710-105440.png`. + - Result: `ok=True`, `visualExists=True`, `sentTelegram=False`, `messageFormat=incident_card_v2`. + - Local visual inspection: PNG is `1040x620`, non-blank, formatted as an incident card. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 5e34b2bb2..62391b8a0 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2,6 +2,7 @@ param( [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]$Mode = "Status", [switch]$ControlledApply, + [switch]$SelfTestTelegramCard, [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" ) @@ -85,6 +86,43 @@ print("sent_ok") } } +function Get-AgentObjectValue { + param( + [object]$Data, + [string]$Name, + [object]$Default = $null + ) + if ($Data -and $Data.PSObject.Properties[$Name]) { + $value = $Data.PSObject.Properties[$Name].Value + if ($null -ne $value -and [string]$value -ne "") { + return $value + } + } + $Default +} + +function Limit-AgentTextLine { + param( + [string]$Text, + [int]$MaxLength = 96 + ) + if (-not $Text) { return "" } + $clean = (($Text -replace "\s+", " ").Trim()) + if ($clean.Length -le $MaxLength) { return $clean } + $clean.Substring(0, $MaxLength - 14) + "...(truncated)" +} + +function Format-AgentMetricForCard { + param( + [object]$Value, + [string]$Suffix = "" + ) + if ($null -eq $Value -or [string]$Value -eq "") { + return "readback_failed" + } + "$Value$Suffix" +} + function Format-AgentTelegramText { param( [string]$Severity, @@ -100,33 +138,46 @@ function Format-AgentTelegramText { default { $Severity } } $lines = @() - $lines += "Agent99 Alert: $severityLabel" + $lines += "Agent99 Incident Card" + $lines += "Severity: $severityLabel" if ($EventType -match "^performance_") { - $hostName = if ($Data -and $Data.PSObject.Properties["host"]) { $Data.host } else { "unknown" } + $hostName = Get-AgentObjectValue $Data "host" "unknown" $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } - $disk = if ($Data -and $Data.PSObject.Properties["diskUsedPercent"]) { $Data.diskUsedPercent } else { $null } - $load = if ($Data -and $Data.PSObject.Properties["loadPerCore"]) { $Data.loadPerCore } else { $null } - $mem = if ($Data -and $Data.PSObject.Properties["memAvailablePercent"]) { $Data.memAvailablePercent } else { $null } + $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null + $load = Get-AgentObjectValue $Data "loadPerCore" $null + $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null $lines += "Event: host performance" $lines += "Host: $hostName" - if ($reasons -contains "disk_used_warning") { + if ($reasons -contains "perf_readback_failed" -or ($null -eq $disk -and $null -eq $load -and $null -eq $mem)) { + $lines += "Reason: performance metric readback failed." + $lines += "Impact: load, memory, and disk values are not trustworthy yet." + $lines += "Agent99 action: keep noisy sensor paused; require verifier readback before remediation." + } elseif ($reasons -contains "disk_used_warning") { $lines += "Reason: disk usage is $disk percent. Warning threshold reached; critical threshold is 95 percent." - $lines += "Action: Agent99 will run allowlisted disk remediation and suppress duplicates for 30 minutes." + $lines += "Agent99 action: run allowlisted disk remediation and suppress duplicates for 30 minutes." } elseif ($reasons -contains "disk_used_high") { $lines += "Reason: disk usage is $disk percent. Critical threshold reached." - $lines += "Action: Agent99 will run allowlisted cleanup first; reboot is not the first response." + $lines += "Agent99 action: run allowlisted cleanup first; reboot is not the first response." } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "Reason: CPU load/core=$load." - $lines += "Action: Agent99 will use allowlisted load reduction only; no arbitrary service stop." + $lines += "Agent99 action: use allowlisted load reduction only; no arbitrary service stop." } elseif ($reasons -contains "memory_available_low") { $lines += "Reason: available memory is $mem percent." - $lines += "Action: Agent99 will inspect memory pressure before allowlisted remediation." + $lines += "Agent99 action: inspect memory pressure before allowlisted remediation." } else { $lines += "Reason: $($reasons -join ', ')" - $lines += "Action: check Agent99 evidence for full readback." + $lines += "Agent99 action: check Agent99 evidence for full readback." + } + $metricParts = @() + if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } + if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } + if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } + if ($metricParts.Count -gt 0) { + $lines += "Current: $($metricParts -join ', ')." + } else { + $lines += "Current: metrics unavailable; no blank metric alert is trusted." } - $lines += "Current: load/core=$load, memAvail=$mem percent, disk=$disk percent." } elseif ($EventType -match "^backup_") { $targetName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $reason = if ($Data -and $Data.PSObject.Properties["reason"]) { $Data.reason } else { $EventType } @@ -138,7 +189,7 @@ function Format-AgentTelegramText { if ($age -ne $null -or $maxAge -ne $null) { $lines += "Freshness: age=$age minutes / max=$maxAge minutes" } - $lines += "Action: inspect /backup/status, /backup/logs, and snapshot metadata only; do not read backup contents or secrets." + $lines += "Agent99 action: inspect /backup/status, /backup/logs, and snapshot metadata only; do not read backup contents or secrets." } elseif ($EventType -eq "provider_freshness_triage") { $candidateId = if ($Data -and $Data.PSObject.Properties["candidateId"]) { $Data.candidateId } else { "unknown" } $targetName = if ($Data -and $Data.PSObject.Properties["target"]) { $Data.target } else { "provider_freshness_signal" } @@ -150,16 +201,17 @@ function Format-AgentTelegramText { $lines += "Event: provider freshness auto triage" $lines += "Target: $targetName" $lines += "Status: $status" + $lines += "Impact: dashboard green is not trusted until source freshness is proven." $lines += "Evidence hits: $hitCount" if ($readback) { - $lines += "Direct readback: ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)" + $lines += "Verifier: directReadback ok=$($readback.ok), sourceEvents=$($readback.sourceEventTotal), latest=$($readback.latestReceivedAt)" } if (@($missing).Count -gt 0) { $lines += "Still missing: $($missing -join ', ')" } else { $lines += "Missing: direct readback fields completed" } - $lines += "Action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed." + $lines += "Agent99 action: candidate and KM/RAG were written; duplicate same-state alerts are suppressed." $lines += "Guard: no provider switch, paid model call, env change, or reload." if ($candidatePath) { $lines += "Evidence: $candidatePath" @@ -174,7 +226,7 @@ function Format-AgentTelegramText { $rollback = if ($Data -and $Data.PSObject.Properties["rollback"]) { $Data.rollback } else { $null } $lines += "Event: controlled remediation" $lines += "Host: $hostName" - $lines += "Action: $actionName" + $lines += "Agent99 action: $actionName" if ($ok -ne $null) { $lines += "Result: ok=$ok exitCode=$exitCode" if ($executionOk -ne $null -or $postVerifyOk -ne $null) { @@ -220,7 +272,7 @@ function Format-AgentTelegramText { $serviceName = if ($Data -and $Data.PSObject.Properties["name"]) { $Data.name } else { "unknown" } $lines += "Event: AI service health" $lines += "Service: $serviceName" - $lines += "Action: inspect health endpoint and service/container state before controlled repair." + $lines += "Agent99 action: inspect health endpoint and service/container state before controlled repair." } elseif ($EventType -eq "security_triage") { $status = if ($Data -and $Data.PSObject.Properties["status"]) { $Data.status } else { "unknown" } $hitCount = if ($Data -and $Data.PSObject.Properties["hitCount"]) { $Data.hitCount } else { 0 } @@ -228,7 +280,7 @@ function Format-AgentTelegramText { $lines += "Event: security triage" $lines += "Status: $status" $lines += "Evidence hits: $hitCount" - $lines += "Action: classify blast radius, affected host/service, source log reference, repeated count, and allowlisted next action." + $lines += "Agent99 action: classify blast radius, affected host/service, source log reference, repeated count, and allowlisted next action." $lines += "Guard: no arbitrary shell, no data deletion, no disabling controls, no secret exposure." if ($evidencePath) { $lines += "Evidence: $evidencePath" @@ -236,7 +288,7 @@ function Format-AgentTelegramText { } elseif ($EventType -eq "agent_selfcheck") { $lines += "Event: Agent99 self health" $lines += "Summary: $Message" - $lines += "Action: inspect task state, evidence freshness, and Telegram delivery; reboot is not the first response." + $lines += "Agent99 action: inspect task state, evidence freshness, and Telegram delivery; reboot is not the first response." } else { $lines += "Event: $EventType" $lines += "Summary: $Message" @@ -250,6 +302,162 @@ function Format-AgentTelegramText { $text } +function Get-AgentTelegramVisualPath { + param([object]$Data) + if (-not $Data) { return $null } + foreach ($propName in @("visualPath", "cardImagePath", "imagePath", "screenshotPath")) { + if ($Data.PSObject.Properties[$propName]) { + $path = [string]$Data.PSObject.Properties[$propName].Value + if ($path -and (Test-Path -LiteralPath $path)) { + return $path + } + } + } + $null +} + +function New-AgentTelegramCardImage { + param( + [string]$Severity, + [string]$EventType, + [string]$Text + ) + + $visualCardsEnabled = $true + if ($Config.telegram -and $Config.telegram.PSObject.Properties["visualCardsEnabled"]) { + $visualCardsEnabled = [bool]$Config.telegram.visualCardsEnabled + } + if (-not $visualCardsEnabled) { return $null } + if ($Severity -notin @("warning", "critical")) { return $null } + + $cardDir = Join-Path $EvidenceDir "telegram-cards" + New-Item -ItemType Directory -Force -Path $cardDir | Out-Null + $safeEvent = ($EventType -replace "[^A-Za-z0-9_.-]", "-") + $imagePath = Join-Path $cardDir "agent99-card-$safeEvent-$stamp.png" + + try { + Add-Type -AssemblyName System.Drawing -ErrorAction Stop + $width = 1040 + $height = 620 + $bitmap = New-Object System.Drawing.Bitmap -ArgumentList $width, $height + $graphics = [System.Drawing.Graphics]::FromImage($bitmap) + $graphics.SmoothingMode = [System.Drawing.Drawing2D.SmoothingMode]::AntiAlias + $graphics.TextRenderingHint = [System.Drawing.Text.TextRenderingHint]::ClearTypeGridFit + + $bgBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(246, 248, 250)) + $panelBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(255, 255, 255)) + $textBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(28, 35, 43)) + $mutedBrush = New-Object System.Drawing.SolidBrush -ArgumentList ([System.Drawing.Color]::FromArgb(91, 103, 112)) + $accentColor = if ($Severity -eq "critical") { + [System.Drawing.Color]::FromArgb(190, 18, 60) + } elseif ($Severity -eq "warning") { + [System.Drawing.Color]::FromArgb(217, 119, 6) + } else { + [System.Drawing.Color]::FromArgb(37, 99, 235) + } + $accentBrush = New-Object System.Drawing.SolidBrush -ArgumentList $accentColor + $titleFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 25, ([System.Drawing.FontStyle]::Bold) + $labelFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 13, ([System.Drawing.FontStyle]::Bold) + $bodyFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 16, ([System.Drawing.FontStyle]::Regular) + $smallFont = New-Object System.Drawing.Font -ArgumentList "Segoe UI", 11, ([System.Drawing.FontStyle]::Regular) + + $graphics.FillRectangle($bgBrush, 0, 0, $width, $height) + $graphics.FillRectangle($panelBrush, 36, 34, $width - 72, $height - 68) + $graphics.FillRectangle($accentBrush, 36, 34, 14, $height - 68) + $graphics.DrawString("Agent99 Incident Card", $titleFont, $textBrush, 72, 58) + $graphics.DrawString(("Severity: " + $Severity.ToUpperInvariant()), $labelFont, $accentBrush, 72, 106) + $graphics.DrawString(("Event: " + (Limit-AgentTextLine $EventType 76)), $labelFont, $mutedBrush, 72, 134) + + $bodyLines = @($Text -split "`n" | Where-Object { $_ -and $_ -notmatch "^Agent99 Incident Card$" } | Select-Object -First 12) + $y = 184 + foreach ($line in $bodyLines) { + $graphics.DrawString((Limit-AgentTextLine $line 96), $bodyFont, $textBrush, 72, $y) + $y += 32 + if ($y -gt 545) { break } + } + $graphics.DrawString(("Generated: " + (Get-Date -Format "yyyy-MM-dd HH:mm:ss zzz")), $smallFont, $mutedBrush, 72, 570) + $bitmap.Save($imagePath, [System.Drawing.Imaging.ImageFormat]::Png) + + $graphics.Dispose() + $bitmap.Dispose() + $bgBrush.Dispose() + $panelBrush.Dispose() + $textBrush.Dispose() + $mutedBrush.Dispose() + $accentBrush.Dispose() + $titleFont.Dispose() + $labelFont.Dispose() + $bodyFont.Dispose() + $smallFont.Dispose() + return $imagePath + } catch { + Write-AgentLog "telegram_card_image_failed event=$EventType error=$($_.Exception.Message)" + return $null + } +} + +function Send-AgentTelegramPhotoDirect { + param( + [string]$Token, + [string]$ChatId, + [string]$Caption, + [string]$PhotoPath + ) + + Add-Type -AssemblyName System.Net.Http -ErrorAction Stop + $client = New-Object System.Net.Http.HttpClient + $content = New-Object System.Net.Http.MultipartFormDataContent + $fileContent = $null + try { + $content.Add([System.Net.Http.StringContent]::new($ChatId), "chat_id") + $captionToSend = $Caption + if ($captionToSend.Length -gt 900) { + $captionToSend = $captionToSend.Substring(0, 900) + "`n...(truncated; see evidence)" + } + $content.Add([System.Net.Http.StringContent]::new($captionToSend), "caption") + $fileBytes = [IO.File]::ReadAllBytes($PhotoPath) + $fileContent = [System.Net.Http.ByteArrayContent]::new($fileBytes) + $fileContent.Headers.ContentType = [System.Net.Http.Headers.MediaTypeHeaderValue]::Parse("image/png") + $content.Add($fileContent, "photo", [IO.Path]::GetFileName($PhotoPath)) + $response = $client.PostAsync("https://api.telegram.org/bot$Token/sendPhoto", $content).GetAwaiter().GetResult() + $body = $response.Content.ReadAsStringAsync().GetAwaiter().GetResult() + if (-not $response.IsSuccessStatusCode) { + throw "telegram_sendPhoto_failed status=$([int]$response.StatusCode) body=$body" + } + } finally { + if ($fileContent) { $fileContent.Dispose() } + if ($content) { $content.Dispose() } + if ($client) { $client.Dispose() } + } +} + +function Invoke-AgentTelegramCardSelfTest { + $sample = [pscustomobject]@{ + host = "192.168.0.110" + reasons = @("perf_readback_failed") + evidencePath = $jsonPath + } + $text = Format-AgentTelegramText "warning" "performance_warning" "selftest visual card" $sample + $imagePath = New-AgentTelegramCardImage "warning" "performance_warning" $text + $ok = [bool]($imagePath -and (Test-Path -LiteralPath $imagePath)) + $result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = $ok + textPreview = @($text -split "`n" | Select-Object -First 12) + visualPath = $imagePath + visualExists = $ok + sentTelegram = $false + messageFormat = "incident_card_v2" + } + $selfTestPath = Join-Path $EvidenceDir "agent99-TelegramCardSelfTest-$stamp.json" + $result | ConvertTo-Json -Depth 6 | Set-Content -Path $selfTestPath -Encoding UTF8 + Write-AgentLog "telegram_card_selftest ok=$ok path=$imagePath evidence=$selfTestPath" + $result | ConvertTo-Json -Depth 6 | Set-Content -Encoding UTF8 $jsonPath + $result | ConvertTo-Json -Depth 6 + if (-not $ok) { exit 1 } + exit 0 +} + function Send-AgentTelegram { param( [string]$Severity, @@ -286,49 +494,79 @@ function Send-AgentTelegram { $chatId = if ($chatSecret) { $chatSecret.value } else { $null } $chatIdOverride = if ($Data -and $Data.PSObject.Properties["replyChatId"]) { [string]$Data.replyChatId } else { "" } - $attempt = [pscustomobject]@{ - timestamp = (Get-Date -Format o) - enabled = [bool]$enabled - eventType = $EventType - severity = $Severity - configured = [bool]($token -and $chatId) - tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } - chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } - chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } - relay = $null - sent = $false - error = $null - } + $attempt = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + enabled = [bool]$enabled + eventType = $EventType + severity = $Severity + messageFormat = "incident_card_v2" + configured = [bool]($token -and $chatId) + tokenEnv = if ($tokenSecret) { $tokenSecret.name } else { $null } + chatIdEnv = if ($chatSecret) { $chatSecret.name } else { $null } + chatOverride = if ($chatIdOverride) { "telegram-origin" } else { $null } + visualPath = $null + visualSent = $false + visualError = $null + relay = $null + sent = $false + error = $null + } if (-not $enabled) { $attempt.error = "telegram_disabled" $script:TelegramAttempts += $attempt return - } - $text = Format-AgentTelegramText $Severity $EventType $Message $Data - if (-not ($token -and ($chatId -or $chatIdOverride))) { - $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride - $attempt.relay = $relayResult - if ($relayResult.ok) { - $attempt.sent = $true - $attempt.error = $null - } else { - $attempt.error = "telegram_not_configured" - } + } + $text = Format-AgentTelegramText $Severity $EventType $Message $Data + $visualPath = Get-AgentTelegramVisualPath $Data + if (-not $visualPath) { + $visualPath = New-AgentTelegramCardImage $Severity $EventType $text + } + $attempt.visualPath = $visualPath + if (-not ($token -and ($chatId -or $chatIdOverride))) { + $relayResult = Send-AgentTelegramRelay $text $telegram.relay $chatIdOverride + $attempt.relay = $relayResult + if ($relayResult.ok) { + $attempt.sent = $true + $attempt.error = $null + if ($visualPath) { + $attempt.visualError = "visual_card_generated_but_relay_sendPhoto_not_available" + } + } else { + $attempt.error = "telegram_not_configured" + } $script:TelegramAttempts += $attempt return - } + } - try { - Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ - chat_id = if ($chatIdOverride) { $chatIdOverride } else { $chatId } - text = $text - disable_web_page_preview = "true" - } | Out-Null - $attempt.sent = $true - } catch { - $attempt.error = $_.Exception.Message - } + try { + $targetChat = if ($chatIdOverride) { $chatIdOverride } else { $chatId } + if ($visualPath -and (Test-Path -LiteralPath $visualPath)) { + try { + Send-AgentTelegramPhotoDirect $token $targetChat $text $visualPath + $attempt.sent = $true + $attempt.visualSent = $true + } catch { + $attempt.visualError = $_.Exception.Message + Write-AgentLog "telegram_sendPhoto_failed event=$EventType fallback=sendMessage error=$($_.Exception.Message)" + Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ + chat_id = $targetChat + text = $text + disable_web_page_preview = "true" + } | Out-Null + $attempt.sent = $true + } + } else { + Invoke-RestMethod -Method Post -Uri "https://api.telegram.org/bot$token/sendMessage" -Body @{ + chat_id = $targetChat + text = $text + disable_web_page_preview = "true" + } | Out-Null + $attempt.sent = $true + } + } catch { + $attempt.error = $_.Exception.Message + } $script:TelegramAttempts += $attempt } @@ -2526,6 +2764,10 @@ function Repair-Awooo { return $true } +if ($SelfTestTelegramCard) { + Invoke-AgentTelegramCardSelfTest +} + Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert