fix(agent99): suppress synthetic monitoring pages

2026-07-10 12:33:11 +08:00
parent 8884d79e52
commit cf5370b49e
5 changed files with 140 additions and 15 deletions
--- a/HOST-REBOOT-AI-AUTOMATION-SOP.md
+++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md
@@ -462,6 +462,7 @@ Recovery is complete only when all of the following are captured:
  - P0-11: paused Agent99 sensors must pass a no-TG/no-remediation sensor restore gate before schedules are re-enabled. The gate must prove Performance, BackupHealth, and SelfHealth are eligible and must write readback evidence with `sentTelegram=false`, `controlledApply=false`, and `ranRemediation=false`.
  - P0-12: slow SSH readback after reboot must not be collapsed into "host down". Performance and backup metadata checks must record `elapsedMs`, use configurable timeouts, and run a slow retry on timeout before raising `perf_readback_failed` or backup readback failure.
  - P0-13: first-run evidence refresh after a noise freeze must use controlled `-SuppressAlerts` for Perf/Backup/SelfCheck so stale evidence is refreshed without Telegram noise; schedules are then re-enabled without suppression.
+  - P0-14: synthetic or `TEST_DO_NOT_PAGE` monitoring tests must still run the full Telegram -> SRE inbox -> ControlTick -> mode evidence path, but `suppressTelegram=true` and `SuppressReason=do_not_page` must propagate end to end so validation never pages the SRE Telegram group.
 - 2026-07-10 10:54 visual card verifier:
  - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`.
  - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`.
@@ -586,6 +587,24 @@ Recovery is complete only when all of the following are captured:
    - Scheduled inbox readback: `Wooo-Agent99-Telegram-Inbox` last run `2026-07-10T12:14:17+08:00`, last task result `0`, latest evidence `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-121418.json`.
    - Scheduled inbox result: `ok=True`, `reason=no_updates`, `updatesSeen=0`, `processedCount=0`, `alertedCount=0`, `sentTelegram=0`.
    - Scheduled relay readback: `ok=True`, `exitCode=0`, `elapsedMs=35062`, `timeoutSeconds=60`, `slowRetry=False`.
+  - 2026-07-10 12:31 synthetic do-not-page E2E verifier:
+    - Runtime scripts deployed:
+      - `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1`, backup `C:\Wooo\Agent99\bin\agent99-telegram-inbox.ps1.bak-20260710-122314`.
+      - `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1`, backups `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1.bak-20260710-122315` and `C:\Wooo\Agent99\bin\agent99-sre-alert-inbox.ps1.bak-20260710-123108`.
+      - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`, backups `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-122315`, `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-122800`, and `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-123108`.
+      - `C:\Wooo\Agent99\agent99-run.ps1`, backups `C:\Wooo\Agent99\agent99-run.ps1.bak-20260710-122315` and `C:\Wooo\Agent99\agent99-run.ps1.bak-20260710-122800`.
+    - Parser readback: all deployed PowerShell scripts and launcher parsed OK before write.
+    - Safety fix: `suppressTelegram` now uses an explicit bool parser so string `"false"` does not become truthy in Windows PowerShell.
+    - Self-tests:
+      - Telegram inbox classifier: `ok=True`, `caseCount=9`, `passedCount=9`, `failedCount=0`; added `synthetic provider freshness do not page`.
+      - SRE alert routing: `C:\Wooo\Agent99\evidence\agent99-SreAlertRoutingSelfTest-20260710-123126.json`, `ok=True`, `caseCount=13`, `passedCount=13`, `failedCount=0`; added `provider freshness do not page`.
+    - E2E alert id: `telegram-monitoring-1783686686`.
+    - Telegram inbox evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramInbox-20260710-123126.json`, `synthetic=True`, `alertedCount=1`, `suppressTelegram=True`.
+    - SRE inbox evidence: `C:\Wooo\Agent99\evidence\agent99-SreAlertInbox-20260710-123126.json`, `queuedCount=1`, `controlExitCode=0`, queue `suppressTelegram=True`.
+    - ControlTick evidence: `C:\Wooo\Agent99\evidence\agent99-ControlTick-20260710-123127.json`, `processedCount=1`, `pendingCount=0`, command `ok=True`, `exitCode=0`, operator result `sent=False`, `suppressed=True`, `reason=do_not_page`.
+    - ProviderFreshness evidence: `C:\Wooo\Agent99\evidence\agent99-ProviderFreshness-20260710-123128.json`, `status=candidate_created_waiting_direct_readback`, candidate `provider-freshness-20260710-123129`.
+    - ProviderFreshness Telegram evidence: both `agent_start` and `provider_freshness_triage` have `sent=False`, `suppressed=True`, `reason=do_not_page`.
+    - Problem/KM writeback: `problemKey=agent99-sre-alert-inbox_unknown_provider_freshness_signal`, `occurrences=6`, `resolved=5`, `failed=1`, `lastStatus=resolved`, KM path `C:\Wooo\Agent99\playbooks\agent99-incident-km.md`.

 ## Agent99 Monitoring Alert Routing Self-Test Receipt

--- a/agent99-bootstrap.ps1
+++ b/agent99-bootstrap.ps1
@@ -114,12 +114,14 @@ if (-not (Test-Path $configPath)) {
 $launcher = Join-Path $AgentRoot "agent99-run.ps1"
@"
 param(
-  [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "ControlTick")]
+  [ValidateSet("Status", "Recover", "StartVMs", "PublicSmoke", "HarborRepair", "AwoooRepair", "Perf", "LoadShed", "SelfCheck", "BackupCheck", "ProviderFreshness", "SecurityTriage", "ControlTick")]
  [string]`$Mode = "Status",
-  [switch]`$ControlledApply
+  [switch]`$ControlledApply,
+  [switch]`$SuppressAlerts,
+  [string]`$SuppressReason = ""
 )

-& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
+& "$BinDir\agent99-control-plane.ps1" -Mode `$Mode -ControlledApply:`$ControlledApply -SuppressAlerts:`$SuppressAlerts -SuppressReason `$SuppressReason -ConfigPath "$configPath" -EvidenceDir "$EvidenceDir"
 "@ | Set-Content -Encoding UTF8 $launcher

 $submitCmd = Join-Path $AgentRoot "agent99-submit-request.cmd"
--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -6,6 +6,7 @@ param(
  [switch]$SelfTestTelegramDelivery,
  [switch]$SelfTestSensorGate,
  [switch]$SuppressAlerts,
+  [string]$SuppressReason = "",
  [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json",
  [string]$EvidenceDir = "C:\Wooo\Agent99\evidence"
 )
@@ -35,6 +36,16 @@ $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" }
 $script:Events = @()
 $script:TelegramAttempts = @()
 $script:SuppressAgentAlerts = [bool]$SuppressAlerts
+$script:SuppressAgentAlertReason = if ($SuppressReason) { $SuppressReason } elseif ($SuppressAlerts) { "suppress_alerts" } else { "" }
+
+function Convert-AgentBool {
+  param([object]$Value)
+  if ($null -eq $Value) { return $false }
+  if ($Value -is [bool]) { return [bool]$Value }
+  $text = ([string]$Value).Trim()
+  if (-not $text) { return $false }
+  return [bool]($text -match "^(?i:true|1|yes|y|on)$")
+}

 function Get-HostSshUser {
  param([string]$TargetHost)
@@ -813,15 +824,26 @@ function Record-AgentEvent {
    return
  }

-  if ($script:SuppressAgentAlerts) {
-    Write-AgentLog "telegram_suppressed_by_sensor_gate type=$EventType severity=$Severity"
+  $eventSuppressTelegram = $false
+  if ($Data -and $Data.PSObject.Properties["suppressTelegram"]) {
+    $eventSuppressTelegram = Convert-AgentBool $Data.suppressTelegram
+  }
+  if ($script:SuppressAgentAlerts -or $eventSuppressTelegram) {
+    $suppressReason = if ($eventSuppressTelegram) {
+      "do_not_page"
+    } elseif ($script:SuppressAgentAlertReason) {
+      $script:SuppressAgentAlertReason
+    } else {
+      "sensor_gate_no_telegram"
+    }
+    Write-AgentLog "telegram_suppressed reason=$suppressReason type=$EventType severity=$Severity"
    $script:TelegramAttempts += [pscustomobject]@{
      timestamp = (Get-Date -Format o)
      eventType = $EventType
      severity = $Severity
      sent = $false
      suppressed = $true
-      reason = "sensor_gate_no_telegram"
+      reason = $suppressReason
    }
    return
  }
@@ -2459,6 +2481,7 @@ function Invoke-AgentQueuedCommands {
    $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null }
    $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null }
    $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" }
+    $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false }
    if ($modeName -notin $allowedModes) {
      $target = Join-Path $failedDir ("failed-" + $file.Name)
      Move-Item -Force $file.FullName $target
@@ -2476,6 +2499,7 @@ function Invoke-AgentQueuedCommands {
    $stderrPath = Join-Path $EvidenceDir "agent99-command-$id.err"
    $args = @("-NoProfile", "-ExecutionPolicy", "Bypass", "-File", $launcher, "-Mode", $modeName)
    if ($controlled) { $args += "-ControlledApply" }
+    if ($suppressTelegram) { $args += @("-SuppressAlerts", "-SuppressReason", "do_not_page") }
    $process = Start-Process -FilePath "powershell.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru
    $completed = $process.WaitForExit(600000)
    if (-not $completed) {
@@ -2507,6 +2531,7 @@ function Invoke-AgentQueuedCommands {
      alertSeverity = $alertSeverity
      alertService = $alertService
      alertHost = $alertHost
+      suppressTelegram = $suppressTelegram
      ok = [bool]($exitCode -eq 0)
      exitCode = $exitCode
      durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1)
--- a/agent99-sre-alert-inbox.ps1
+++ b/agent99-sre-alert-inbox.ps1
@@ -34,6 +34,15 @@ function Get-AgentField {
  return $Default
 }

+function Convert-AgentBool {
+  param([object]$Value)
+  if ($null -eq $Value) { return $false }
+  if ($Value -is [bool]) { return [bool]$Value }
+  $text = ([string]$Value).Trim()
+  if (-not $text) { return $false }
+  return [bool]($text -match "^(?i:true|1|yes|y|on)$")
+}
+
 function Convert-AgentSafeFileToken {
  param([string]$Value)
  $safe = [regex]::Replace($Value, "[^A-Za-z0-9_.-]", "_")
@@ -63,7 +72,7 @@ function Get-AgentAlertText {

 function Test-AgentAlertActionable {
  param([object]$Alert)
-  $force = [bool](Get-AgentField $Alert "force" $false)
+  $force = Convert-AgentBool (Get-AgentField $Alert "force" $false)
  if ($force) { return $true }

  $text = (Get-AgentAlertText $Alert).ToLowerInvariant()
@@ -162,6 +171,14 @@ function Get-AgentControlledApplyForAlert {
  return [bool]($ModeName -in $remediationModes)
 }

+function Test-AgentAlertSuppressTelegram {
+  param([object]$Alert)
+  if (Convert-AgentBool (Get-AgentField $Alert "suppressTelegram" $false)) { return $true }
+  $text = (Get-AgentAlertText $Alert)
+  if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true }
+  return $false
+}
+
 function Invoke-AgentAlertRoutingSelfTest {
  $cases = @(
    [pscustomobject]@{
@@ -267,6 +284,22 @@ function Invoke-AgentAlertRoutingSelfTest {
        message = "provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
      }
    },
+    [pscustomobject]@{
+      name = "provider freshness do not page"
+      expectedActionable = $true
+      expectedMode = "ProviderFreshness"
+      expectedControlledApply = $false
+      expectedSuppressTelegram = $true
+      alert = [pscustomobject]@{
+        id = "selftest-provider-freshness-do-not-page"
+        source = "agent99-routing-selftest"
+        severity = "warning"
+        kind = "provider_freshness_signal"
+        service = "provider_freshness_signal"
+        message = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate"
+        suppressTelegram = $true
+      }
+    },
    [pscustomobject]@{
      name = "security alert"
      expectedActionable = $true
@@ -347,11 +380,14 @@ function Invoke-AgentAlertRoutingSelfTest {
    $actionable = Test-AgentAlertActionable $case.alert
    $mode = if ($actionable) { Resolve-AgentAlertMode $case.alert } else { $null }
    $controlled = if ($actionable -and $mode) { Get-AgentControlledApplyForAlert $case.alert $mode } else { $false }
+    $suppressTelegram = if ($actionable) { Test-AgentAlertSuppressTelegram $case.alert } else { $false }
+    $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
    $instruction = if ($actionable -and $mode) { New-AgentInstructionFromAlert $case.alert $case.alert.id $mode } else { $null }
    $ok = [bool](
      $actionable -eq $case.expectedActionable -and
      [string]$mode -eq [string]$case.expectedMode -and
-      $controlled -eq $case.expectedControlledApply
+      $controlled -eq $case.expectedControlledApply -and
+      $suppressTelegram -eq $expectedSuppressTelegram
    )
    $results += [pscustomobject]@{
      name = $case.name
@@ -363,6 +399,8 @@ function Invoke-AgentAlertRoutingSelfTest {
      expectedMode = $case.expectedMode
      controlledApply = $controlled
      expectedControlledApply = $case.expectedControlledApply
+      suppressTelegram = $suppressTelegram
+      expectedSuppressTelegram = $expectedSuppressTelegram
      instruction = $instruction
    }
  }
@@ -454,6 +492,7 @@ foreach ($file in $files) {
  $service = [string](Get-AgentField $alert "service" "unknown")
  $hostName = [string](Get-AgentField $alert "host" "")
  $replyChatId = [string](Get-AgentField $alert "replyChatId" "")
+  $suppressTelegram = Test-AgentAlertSuppressTelegram $alert

  $requestPath = Join-Path $RequestDir "$queueId.json"
  $queuePath = Join-Path $QueueDir "$queueId.json"
@@ -473,6 +512,7 @@ foreach ($file in $files) {
    alertService = $service
    alertHost = $hostName
    alertPath = $processedPath
+    suppressTelegram = $suppressTelegram
    createdAt = (Get-Date -Format o)
  } | ConvertTo-Json -Depth 8 | Set-Content -Path $requestPath -Encoding UTF8

@@ -493,6 +533,7 @@ foreach ($file in $files) {
    alertHost = $hostName
    alertPath = $processedPath
    replyChatId = $replyChatId
+    suppressTelegram = $suppressTelegram
    createdAt = (Get-Date -Format o)
  } | ConvertTo-Json -Depth 8 | Set-Content -Path $queuePath -Encoding UTF8

@@ -506,6 +547,7 @@ foreach ($file in $files) {
    host = $hostName
    mode = $modeName
    controlledApply = $controlled
+    suppressTelegram = $suppressTelegram
    requestPath = $requestPath
    queuePath = $queuePath
    alertPath = $processedPath
--- a/agent99-telegram-inbox.ps1
+++ b/agent99-telegram-inbox.ps1
@@ -333,6 +333,15 @@ function Get-AgentTelegramAlertHost {
  return ""
 }

+function Test-AgentTelegramSuppressPage {
+  param([object]$Update)
+  $text = if ($Update -and $Update.PSObject.Properties["text"]) { [string]$Update.text } else { "" }
+  $chatMatch = if ($Update -and $Update.PSObject.Properties["chat_match"]) { [string]$Update.chat_match } else { "" }
+  if ($chatMatch -eq "synthetic") { return $true }
+  if ($text -match "(?i)\b(TEST_DO_NOT_PAGE|DO_NOT_PAGE|NO_PAGE|SYNTHETIC_TEST)\b") { return $true }
+  return $false
+}
+
 function New-AgentMonitoringAlertFromTelegram {
  param([object]$Update)

@@ -349,25 +358,33 @@ function New-AgentMonitoringAlertFromTelegram {
  $alertId = "telegram-monitoring-" + [string]$Update.update_id
  $alertPath = Join-Path $AlertIncomingDir ($alertId + ".json")
  $message = if ($text.Length -gt 2000) { $text.Substring(0, 2000) + "`n...(truncated)" } else { $text }
+  $kind = Get-AgentTelegramAlertKind $text
+  $suppressTelegram = Test-AgentTelegramSuppressPage $Update
+  $labels = if ($kind -eq "provider_freshness_signal") {
+    @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green")
+  } else {
+    @("telegram", "sre-war-room", "monitoring")
+  }
+  if ($suppressTelegram) {
+    $labels += @("synthetic", "do-not-page")
+  }
  $alert = [pscustomobject]@{
    id = $alertId
    source = "telegram-sre-war-room-monitoring"
    severity = $severity
-    kind = Get-AgentTelegramAlertKind $text
+    kind = $kind
    service = Get-AgentTelegramAlertService $text
    host = Get-AgentTelegramAlertHost $text
    title = "AwoooI SRE Telegram monitoring alert"
    message = $message
-    labels = if ((Get-AgentTelegramAlertKind $text) -eq "provider_freshness_signal") {
-      @("telegram", "sre-war-room", "monitoring", "provider-freshness", "no-false-green")
-    } else {
-      @("telegram", "sre-war-room", "monitoring")
-    }
+    labels = $labels
    telegramUpdateId = $Update.update_id
    telegramMessageId = $Update.message_id
    telegramChatTitle = $Update.chat_title
    telegramChatType = $Update.chat_type
    replyChatId = [string]$Update.chat_id
+    suppressTelegram = $suppressTelegram
+    suppressReason = if ($suppressTelegram) { "telegram_synthetic_or_do_not_page" } else { "" }
    force = $true
    createdAt = (Get-Date -Format o)
  }
@@ -389,6 +406,7 @@ function New-AgentMonitoringAlertFromTelegram {
    kind = $alert.kind
    service = $alert.service
    host = $alert.host
+    suppressTelegram = $suppressTelegram
    sreInboxExitCode = $sreExitCode
  }
 }
@@ -414,6 +432,18 @@ function Invoke-AgentTelegramInboxSelfTest {
      expectedKind = "provider_freshness_signal"
      expectedService = "unknown"
      expectedHost = ""
+      expectedSuppressTelegram = $false
+    },
+    [pscustomobject]@{
+      name = "synthetic provider freshness do not page"
+      text = "TEST_DO_NOT_PAGE provider_freshness_signal source_provider_freshness_triage raw_signal_normalized controlled_runtime_candidate lastSeen missing"
+      chatMatch = "synthetic"
+      expectedMonitoring = $true
+      expectedSeverity = "warning"
+      expectedKind = "provider_freshness_signal"
+      expectedService = "unknown"
+      expectedHost = ""
+      expectedSuppressTelegram = $true
    },
    [pscustomobject]@{
      name = "host perf readback failed"
@@ -478,12 +508,16 @@ function Invoke-AgentTelegramInboxSelfTest {
    $kind = if ($monitoring) { Get-AgentTelegramAlertKind $case.text } else { $null }
    $service = if ($monitoring) { Get-AgentTelegramAlertService $case.text } else { $null }
    $hostName = if ($monitoring) { Get-AgentTelegramAlertHost $case.text } else { "" }
+    $chatMatch = if ($case.PSObject.Properties["chatMatch"]) { [string]$case.chatMatch } else { "configured_chat_id" }
+    $suppressTelegram = if ($monitoring) { Test-AgentTelegramSuppressPage ([pscustomobject]@{ text = $case.text; chat_match = $chatMatch }) } else { $false }
+    $expectedSuppressTelegram = if ($case.PSObject.Properties["expectedSuppressTelegram"]) { [bool]$case.expectedSuppressTelegram } else { $false }
    $passed = (
      $monitoring -eq $case.expectedMonitoring -and
      [string]$severity -eq [string]$case.expectedSeverity -and
      [string]$kind -eq [string]$case.expectedKind -and
      [string]$service -eq [string]$case.expectedService -and
-      [string]$hostName -eq [string]$case.expectedHost
+      [string]$hostName -eq [string]$case.expectedHost -and
+      $suppressTelegram -eq $expectedSuppressTelegram
    )
    $results += [pscustomobject]@{
      name = $case.name
@@ -498,6 +532,8 @@ function Invoke-AgentTelegramInboxSelfTest {
      expectedService = $case.expectedService
      host = $hostName
      expectedHost = $case.expectedHost
+      suppressTelegram = $suppressTelegram
+      expectedSuppressTelegram = $expectedSuppressTelegram
    }
  }

@@ -584,6 +620,7 @@ foreach ($update in @($updatesResult.updates)) {
          kind = $alertResult.kind
          service = $alertResult.service
          host = $alertResult.host
+          suppressTelegram = $alertResult.suppressTelegram
          alertPath = $alertResult.alertPath
          sreInboxExitCode = $alertResult.sreInboxExitCode
        }