From cf5370b49ef9013a23611c3ee71ee61164211709 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:33:11 +0800 Subject: [PATCH] fix(agent99): suppress synthetic monitoring pages --- HOST-REBOOT-AI-AUTOMATION-SOP.md | 19 ++++++++++++ agent99-bootstrap.ps1 | 8 +++-- agent99-control-plane.ps1 | 31 +++++++++++++++++-- agent99-sre-alert-inbox.ps1 | 46 ++++++++++++++++++++++++++-- agent99-telegram-inbox.ps1 | 51 +++++++++++++++++++++++++++----- 5 files changed, 140 insertions(+), 15 deletions(-) diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index d05b822ef..9cd70852d 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -462,6 +462,7 @@ Recovery is complete only when all of the following are captured: - P0-11: paused Agent99 sensors must pass a no-TG/no-remediation sensor restore gate before schedules are re-enabled. The gate must prove Performance, BackupHealth, and SelfHealth are eligible and must write readback evidence with `sentTelegram=false`, `controlledApply=false`, and `ranRemediation=false`. - P0-12: slow SSH readback after reboot must not be collapsed into "host down". Performance and backup metadata checks must record `elapsedMs`, use configurable timeouts, and run a slow retry on timeout before raising `perf_readback_failed` or backup readback failure. - P0-13: first-run evidence refresh after a noise freeze must use controlled `-SuppressAlerts` for Perf/Backup/SelfCheck so stale evidence is refreshed without Telegram noise; schedules are then re-enabled without suppression. + - P0-14: synthetic or `TEST_DO_NOT_PAGE` monitoring tests must still run the full Telegram -> SRE inbox -> ControlTick -> mode evidence path, but `suppressTelegram=true` and `SuppressReason=do_not_page` must propagate end to end so validation never pages the SRE Telegram group. - 2026-07-10 10:54 visual card verifier: - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`. @@ -586,6 +587,24 @@ Recovery is complete only when all of the following are captured: - Scheduled inbox readback: `Wooo-Agent99-Telegram-Inbox` last run `2026-07-10T12:14:17+08:00`, last task result `0`, latest evidence `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-121418.json`. - Scheduled inbox result: `ok=True`, `reason=no_updates`, `updatesSeen=0`, `processedCount=0`, `alertedCount=0`, `sentTelegram=0`. - Scheduled relay readback: `ok=True`, `exitCode=0`, `elapsedMs=35062`, `timeoutSeconds=60`, `slowRetry=False`. + - 2026-07-10 12:31 synthetic do-not-page E2E verifier: + - Runtime scripts deployed: + - `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`, backup `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1.bak-20260710-122314`. + - `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1`, backups `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1.bak-20260710-122315` and `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1.bak-20260710-123108`. + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`, backups `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-122315`, `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-122800`, and `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-123108`. + - `C:\Wooo\Agent99\agent99-run.ps1`, backups `C:\Wooo\Agent99\agent99-run.ps1.bak-20260710-122315` and `C:\Wooo\Agent99\agent99-run.ps1.bak-20260710-122800`. + - Parser readback: all deployed PowerShell scripts and launcher parsed OK before write. + - Safety fix: `suppressTelegram` now uses an explicit bool parser so string `"false"` does not become truthy in Windows PowerShell. + - Self-tests: + - Telegram inbox classifier: `ok=True`, `caseCount=9`, `passedCount=9`, `failedCount=0`; added `synthetic provider freshness do not page`. + - SRE alert routing: `C:\Wooo\Agent99\evidence\agent99-SreAlertRoutingSelfTest-20260710-123126.json`, `ok=True`, `caseCount=13`, `passedCount=13`, `failedCount=0`; added `provider freshness do not page`. + - E2E alert id: `telegram-monitoring-1783686686`. + - Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-123126.json`, `synthetic=True`, `alertedCount=1`, `suppressTelegram=True`. + - SRE inbox evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260710-123126.json`, `queuedCount=1`, `controlExitCode=0`, queue `suppressTelegram=True`. + - ControlTick evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260710-123127.json`, `processedCount=1`, `pendingCount=0`, command `ok=True`, `exitCode=0`, operator result `sent=False`, `suppressed=True`, `reason=do_not_page`. + - ProviderFreshness evidence: `C:\Wooo\Agent99\evidence\agent99-ProviderFreshness-20260710-123128.json`, `status=candidate_created_waiting_direct_readback`, candidate `provider-freshness-20260710-123129`. + - ProviderFreshness Telegram evidence: both `agent_start` and `provider_freshness_triage` have `sent=False`, `suppressed=True`, `reason=do_not_page`. + - Problem/KM writeback: `problemKey=agent99-sre-alert-inbox_unknown_provider_freshness_signal`, `occurrences=6`, `resolved=5`, `failed=1`, `lastStatus=resolved`, KM path `C:\Wooo\Agent99\playbooks\agent99-incident-km.md`. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 801ea1f57..15bfd323d 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -114,12 +114,14 @@ if (-not (Test-Path $configPath)) { $launcher = Join-Path $AgentRoot "agent99-run.ps1" @" param( - [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "ControlTick")] + [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")] [string]`$Mode = "Status", - [switch]`$ControlledApply + [switch]`$ControlledApply, + [switch]`$SuppressAlerts, + [string]`$SuppressReason = "" ) -& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" +& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir" "@ | Set-Content -Encoding UTF8 $launcher $submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 1848c3f4f..46719ffce 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -6,6 +6,7 @@ param( [switch]$SelfTestTelegramDelivery, [switch]$SelfTestSensorGate, [switch]$SuppressAlerts, + [string]$SuppressReason = "", [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" ) @@ -35,6 +36,16 @@ $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } $script:Events = @() $script:TelegramAttempts = @() $script:SuppressAgentAlerts = [bool]$SuppressAlerts +$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" } + +function Convert-AgentBool { + param([object]$Value) + if ($null -eq $Value) { return $false } + if ($Value -is [bool]) { return [bool]$Value } + $text = ([string]$Value).Trim() + if (-not $text) { return $false } + return [bool]($text -match "^(?i:true|1|yes|y|on)$") +} function Get-HostSshUser { param([string]$TargetHost) @@ -813,15 +824,26 @@ function Record-AgentEvent { return } - if ($script:SuppressAgentAlerts) { - Write-AgentLog "telegram_suppressed_by_sensor_gate type=$EventType severity=$Severity" + $eventSuppressTelegram = $false + if ($Data -and $Data.PSObject.Properties["suppressTelegram"]) { + $eventSuppressTelegram = Convert-AgentBool $Data.suppressTelegram + } + if ($script:SuppressAgentAlerts -or $eventSuppressTelegram) { + $suppressReason = if ($eventSuppressTelegram) { + "do_not_page" + } elseif ($script:SuppressAgentAlertReason) { + $script:SuppressAgentAlertReason + } else { + "sensor_gate_no_telegram" + } + Write-AgentLog "telegram_suppressed reason=$suppressReason type=$EventType severity=$Severity" $script:TelegramAttempts += [pscustomobject]@{ timestamp = (Get-Date -Format o) eventType = $EventType severity = $Severity sent = $false suppressed = $true - reason = "sensor_gate_no_telegram" + reason = $suppressReason } return } @@ -2459,6 +2481,7 @@ function Invoke-AgentQueuedCommands { $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null } $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null } $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } + $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false } if ($modeName -notin $allowedModes) { $target = Join-Path $failedDir ("failed-" + $file.Name) Move-Item -Force $file.FullName $target @@ -2476,6 +2499,7 @@ function Invoke-AgentQueuedCommands { $stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err" $args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName) if ($controlled) { $args += "-ControlledApply" } + if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") } $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru $completed = $process.WaitForExit(600000) if (-not $completed) { @@ -2507,6 +2531,7 @@ function Invoke-AgentQueuedCommands { alertSeverity = $alertSeverity alertService = $alertService alertHost = $alertHost + suppressTelegram = $suppressTelegram ok = [bool]($exitCode -eq 0) exitCode = $exitCode durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index f69f5abe6..d4a420c39 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -34,6 +34,15 @@ function Get-AgentField { return $Default } +function Convert-AgentBool { + param([object]$Value) + if ($null -eq $Value) { return $false } + if ($Value -is [bool]) { return [bool]$Value } + $text = ([string]$Value).Trim() + if (-not $text) { return $false } + return [bool]($text -match "^(?i:true|1|yes|y|on)$") +} + function Convert-AgentSafeFileToken { param([string]$Value) $safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_") @@ -63,7 +72,7 @@ function Get-AgentAlertText { function Test-AgentAlertActionable { param([object]$Alert) - $force = [bool](Get-AgentField $Alert "force" $false) + $force = Convert-AgentBool (Get-AgentField $Alert "force" $false) if ($force) { return $true } $text = (Get-AgentAlertText $Alert).ToLowerInvariant() @@ -162,6 +171,14 @@ function Get-AgentControlledApplyForAlert { return [bool]($ModeName -in $remediationModes) } +function Test-AgentAlertSuppressTelegram { + param([object]$Alert) + if (Convert-AgentBool (Get-AgentField $Alert "suppressTelegram" $false)) { return $true } + $text = (Get-AgentAlertText $Alert) + if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true } + return $false +} + function Invoke-AgentAlertRoutingSelfTest { $cases = @( [pscustomobject]@{ @@ -267,6 +284,22 @@ function Invoke-AgentAlertRoutingSelfTest { message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate" } }, + [pscustomobject]@{ + name = "provider freshness do not page" + expectedActionable = $true + expectedMode = "ProviderFreshness" + expectedControlledApply = $false + expectedSuppressTelegram = $true + alert = [pscustomobject]@{ + id = "selftest-provider-freshness-do-not-page" + source = "agent99-routing-selftest" + severity = "warning" + kind = "provider_freshness_signal" + service = "provider_freshness_signal" + message = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate" + suppressTelegram = $true + } + }, [pscustomobject]@{ name = "security alert" expectedActionable = $true @@ -347,11 +380,14 @@ function Invoke-AgentAlertRoutingSelfTest { $actionable = Test-AgentAlertActionable $case.alert $mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null } $controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false } + $suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false } + $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false } $instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null } $ok = [bool]( $actionable -eq $case.expectedActionable -and [string]$mode -eq [string]$case.expectedMode -and - $controlled -eq $case.expectedControlledApply + $controlled -eq $case.expectedControlledApply -and + $suppressTelegram -eq $expectedSuppressTelegram ) $results += [pscustomobject]@{ name = $case.name @@ -363,6 +399,8 @@ function Invoke-AgentAlertRoutingSelfTest { expectedMode = $case.expectedMode controlledApply = $controlled expectedControlledApply = $case.expectedControlledApply + suppressTelegram = $suppressTelegram + expectedSuppressTelegram = $expectedSuppressTelegram instruction = $instruction } } @@ -454,6 +492,7 @@ foreach ($file in $files) { $service = [string](Get-AgentField $alert "service" "unknown") $hostName = [string](Get-AgentField $alert "host" "") $replyChatId = [string](Get-AgentField $alert "replyChatId" "") + $suppressTelegram = Test-AgentAlertSuppressTelegram $alert $requestPath = Join-Path $RequestDir "$queueId.json" $queuePath = Join-Path $QueueDir "$queueId.json" @@ -473,6 +512,7 @@ foreach ($file in $files) { alertService = $service alertHost = $hostName alertPath = $processedPath + suppressTelegram = $suppressTelegram createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8 @@ -493,6 +533,7 @@ foreach ($file in $files) { alertHost = $hostName alertPath = $processedPath replyChatId = $replyChatId + suppressTelegram = $suppressTelegram createdAt = (Get-Date -Format o) } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8 @@ -506,6 +547,7 @@ foreach ($file in $files) { host = $hostName mode = $modeName controlledApply = $controlled + suppressTelegram = $suppressTelegram requestPath = $requestPath queuePath = $queuePath alertPath = $processedPath diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index 21e966dbf..405b51692 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -333,6 +333,15 @@ function Get-AgentTelegramAlertHost { return "" } +function Test-AgentTelegramSuppressPage { + param([object]$Update) + $text = if ($Update -and $Update.PSObject.Properties["text"]) { [string]$Update.text } else { "" } + $chatMatch = if ($Update -and $Update.PSObject.Properties["chat_match"]) { [string]$Update.chat_match } else { "" } + if ($chatMatch -eq "synthetic") { return $true } + if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true } + return $false +} + function New-AgentMonitoringAlertFromTelegram { param([object]$Update) @@ -349,25 +358,33 @@ function New-AgentMonitoringAlertFromTelegram { $alertId = "telegram-monitoring-" + [string]$Update.update_id $alertPath = Join-Path $AlertIncomingDir ($alertId + ".json") $message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text } + $kind = Get-AgentTelegramAlertKind $text + $suppressTelegram = Test-AgentTelegramSuppressPage $Update + $labels = if ($kind -eq "provider_freshness_signal") { + @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green") + } else { + @("telegram", "sre-war-room", "monitoring") + } + if ($suppressTelegram) { + $labels += @("synthetic", "do-not-page") + } $alert = [pscustomobject]@{ id = $alertId source = "telegram-sre-war-room-monitoring" severity = $severity - kind = Get-AgentTelegramAlertKind $text + kind = $kind service = Get-AgentTelegramAlertService $text host = Get-AgentTelegramAlertHost $text title = "AwoooI SRE Telegram monitoring alert" message = $message - labels = if ((Get-AgentTelegramAlertKind $text) -eq "provider_freshness_signal") { - @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green") - } else { - @("telegram", "sre-war-room", "monitoring") - } + labels = $labels telegramUpdateId = $Update.update_id telegramMessageId = $Update.message_id telegramChatTitle = $Update.chat_title telegramChatType = $Update.chat_type replyChatId = [string]$Update.chat_id + suppressTelegram = $suppressTelegram + suppressReason = if ($suppressTelegram) { "telegram_synthetic_or_do_not_page" } else { "" } force = $true createdAt = (Get-Date -Format o) } @@ -389,6 +406,7 @@ function New-AgentMonitoringAlertFromTelegram { kind = $alert.kind service = $alert.service host = $alert.host + suppressTelegram = $suppressTelegram sreInboxExitCode = $sreExitCode } } @@ -414,6 +432,18 @@ function Invoke-AgentTelegramInboxSelfTest { expectedKind = "provider_freshness_signal" expectedService = "unknown" expectedHost = "" + expectedSuppressTelegram = $false + }, + [pscustomobject]@{ + name = "synthetic provider freshness do not page" + text = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate lastSeen missing" + chatMatch = "synthetic" + expectedMonitoring = $true + expectedSeverity = "warning" + expectedKind = "provider_freshness_signal" + expectedService = "unknown" + expectedHost = "" + expectedSuppressTelegram = $true }, [pscustomobject]@{ name = "host perf readback failed" @@ -478,12 +508,16 @@ function Invoke-AgentTelegramInboxSelfTest { $kind = if ($monitoring) { Get-AgentTelegramAlertKind $case.text } else { $null } $service = if ($monitoring) { Get-AgentTelegramAlertService $case.text } else { $null } $hostName = if ($monitoring) { Get-AgentTelegramAlertHost $case.text } else { "" } + $chatMatch = if ($case.PSObject.Properties["chatMatch"]) { [string]$case.chatMatch } else { "configured_chat_id" } + $suppressTelegram = if ($monitoring) { Test-AgentTelegramSuppressPage ([pscustomobject]@{ text = $case.text; chat_match = $chatMatch }) } else { $false } + $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false } $passed = ( $monitoring -eq $case.expectedMonitoring -and [string]$severity -eq [string]$case.expectedSeverity -and [string]$kind -eq [string]$case.expectedKind -and [string]$service -eq [string]$case.expectedService -and - [string]$hostName -eq [string]$case.expectedHost + [string]$hostName -eq [string]$case.expectedHost -and + $suppressTelegram -eq $expectedSuppressTelegram ) $results += [pscustomobject]@{ name = $case.name @@ -498,6 +532,8 @@ function Invoke-AgentTelegramInboxSelfTest { expectedService = $case.expectedService host = $hostName expectedHost = $case.expectedHost + suppressTelegram = $suppressTelegram + expectedSuppressTelegram = $expectedSuppressTelegram } } @@ -584,6 +620,7 @@ foreach ($update in @($updatesResult.updates)) { kind = $alertResult.kind service = $alertResult.service host = $alertResult.host + suppressTelegram = $alertResult.suppressTelegram alertPath = $alertResult.alertPath sreInboxExitCode = $alertResult.sreInboxExitCode }