From 6a8ea96c49f1b53451148c0061bfc196f8fb9cd9 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 12:03:53 +0800 Subject: [PATCH] fix(agent99): gate sensor restore after reboot --- HOST-REBOOT-AI-AUTOMATION-SOP.md | 57 ++++++++++ agent99-control-plane.ps1 | 181 ++++++++++++++++++++++++++++++- 2 files changed, 232 insertions(+), 6 deletions(-) diff --git a/HOST-REBOOT-AI-AUTOMATION-SOP.md b/HOST-REBOOT-AI-AUTOMATION-SOP.md index b97220781..d762095e5 100644 --- a/HOST-REBOOT-AI-AUTOMATION-SOP.md +++ b/HOST-REBOOT-AI-AUTOMATION-SOP.md @@ -459,6 +459,9 @@ Recovery is complete only when all of the following are captured: - P0-8: all Agent99 Telegram alert messages must include Traditional Chinese user-facing labels and human-readable context; professional terms such as `readback_failed`, `SecurityTriage`, `directReadback`, `KM/RAG`, and provider names may remain English when translation would reduce precision. - P0-9: Telegram inbox must not force `controlledApply=true` on every monitoring alert. `controlledApply` is resolved by SRE mode: remediation modes may run controlled apply, evidence-first modes such as `ProviderFreshness`, `BackupCheck`, `SecurityTriage`, and `Status` must remain non-apply unless explicitly allowlisted. - P0-10: Telegram delivery must support actual image/photo delivery through the relay path, not only text plus a local evidence image. + - P0-11: paused Agent99 sensors must pass a no-TG/no-remediation sensor restore gate before schedules are re-enabled. The gate must prove Performance, BackupHealth, and SelfHealth are eligible and must write readback evidence with `sentTelegram=false`, `controlledApply=false`, and `ranRemediation=false`. + - P0-12: slow SSH readback after reboot must not be collapsed into "host down". Performance and backup metadata checks must record `elapsedMs`, use configurable timeouts, and run a slow retry on timeout before raising `perf_readback_failed` or backup readback failure. + - P0-13: first-run evidence refresh after a noise freeze must use controlled `-SuppressAlerts` for Perf/Backup/SelfCheck so stale evidence is refreshed without Telegram noise; schedules are then re-enabled without suppression. - 2026-07-10 10:54 visual card verifier: - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. - Backup before deploy: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-105439`. @@ -501,6 +504,60 @@ Recovery is complete only when all of the following are captured: - Self-test evidence: `C:\Wooo\Agent99\evidence\agent99-TelegramDeliverySelfTest-20260710-112512.json`. - Result: `ok=True`, `sentTelegram=True`, `visualSent=True`, `relay.photoSent=True`, `relay.output=photo_sent_ok`, `messageFormat=incident_card_zh_tw_v3`. - Delivery mode: Windows 99 generated the PNG, copied it to host 110, copied it into relay container `stockplatform-v2-api-1`, and sent it through Telegram `sendPhoto`; text-only fallback was not used. +- 2026-07-10 12:01 sensor restore gate verifier: + - Runtime script deployed: `C:\Wooo\Agent99\bin\agent99-control-plane.ps1`. + - Backups before deploy: + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-114000`. + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-114519`. + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-114816`. + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-115331`. + - `C:\Wooo\Agent99\bin\agent99-control-plane.ps1.bak-20260710-115644`. + - New verifier mode: `-SelfTestSensorGate`. + - New controlled refresh switch: `-SuppressAlerts`. + - SSH readback hardening: + - Performance readback default timeout raised to `45s`. + - Performance timeout gets slow retry with default `90s`. + - Backup file and crontab metadata readback default timeout raised to `45s`. + - SSH evidence now includes `elapsedMs`; performance rows include `readTimeoutSeconds`, `slowReadTimeoutSeconds`, and `readbackSlowRetry`. + - False-negative findings fixed during rollout: + - 110 performance readback was timing out at the previous `25s` boundary even though SSH succeeded. + - Backup file metadata checks were timing out at the previous `15s` boundary. + - Backup crontab readback was timing out at the previous `12s` boundary. + - Suppressed refresh evidence: + - Perf refresh: `C:\Wooo\Agent99\evidence\agent99-refresh-perf-20260710-115704.txt`, exit code `0`. + - Backup refresh: `C:\Wooo\Agent99\evidence\agent99-refresh-backup-20260710-115754.txt`, exit code `0`. + - Self refresh: `C:\Wooo\Agent99\evidence\agent99-refresh-self-20260710-115858.txt`, exit code `0`. + - Final sensor gate evidence: `C:\Wooo\Agent99\evidence\agent99-SensorGateSelfTest-20260710-115927.json`. + - Final gate result: + - `ok=True`. + - `readyToEnableAll=True`. + - `sentTelegram=False`. + - `controlledApply=False`. + - `ranRemediation=False`. + - `performanceGuard=eligible`. + - `backupHealth=eligible`. + - `selfHealth=eligible`. + - Final host performance readback: + - `192.168.0.110`: `ok`, `loadPerCore=0.44`, `memAvailPct=59.93`, `diskUsedPct=83`, `elapsedMs=25834`. + - `192.168.0.112`: `ok`, `loadPerCore=0.01`, `memAvailPct=53.09`, `diskUsedPct=53`, `elapsedMs=345`. + - `192.168.0.120`: `ok`, `loadPerCore=0.07`, `memAvailPct=90.01`, `diskUsedPct=47`, `elapsedMs=709`. + - `192.168.0.121`: `ok`, `loadPerCore=0.02`, `memAvailPct=92.75`, `diskUsedPct=27`, `elapsedMs=1247`. + - `192.168.0.188`: `ok`, `loadPerCore=0.26`, `memAvailPct=83.77`, `diskUsedPct=88`, `elapsedMs=974`. + - Final backup readback: + - `backupSeverity=ok`. + - `targetCritical=0`, `targetWarning=0`. + - `fileCritical=0`, `fileWarning=0`. + - `cronCritical=0`, `cronWarning=0`. + - Coverage: `13` snapshot targets, `7` file checks, `6` cron patterns. + - Final self-health readback: + - `selfSeverity=ok`. + - `taskFailures=0`. + - `evidenceCritical=0`, `evidenceWarning=0`. + - `performanceSeverity=ok`, `telegramSeverity=ok`. + - Restored schedules: + - `Wooo-Agent99-Performance-Guard`: enabled, latest Perf evidence `C:\Wooo\Agent99\evidence\agent99-Perf-20260710-120117.json`, `latestPerfTelegramSent=0`, `latestPerfCritical=0`, `latestPerfWarning=0`. + - `Wooo-Agent99-Backup-Health`: enabled. + - `Wooo-Agent99-Self-Health`: enabled. ## Agent99 Monitoring Alert Routing Self-Test Receipt diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 926dc87d9..1848c3f4f 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4,6 +4,8 @@ param( [switch]$ControlledApply, [switch]$SelfTestTelegramCard, [switch]$SelfTestTelegramDelivery, + [switch]$SelfTestSensorGate, + [switch]$SuppressAlerts, [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", [string]$EvidenceDir = "C:\Wooo\Agent99\evidence" ) @@ -32,6 +34,7 @@ $Config = Load-Config $SshUser = if ($Config.sshUser) { $Config.sshUser } else { "wooo" } $script:Events = @() $script:TelegramAttempts = @() +$script:SuppressAgentAlerts = [bool]$SuppressAlerts function Get-HostSshUser { param([string]$TargetHost) @@ -582,6 +585,91 @@ function Invoke-AgentTelegramDeliverySelfTest { exit 0 } +function Invoke-AgentSensorGateSelfTest { + $previousSuppress = $script:SuppressAgentAlerts + $script:SuppressAgentAlerts = $true + $gateError = $null + $performance = @() + $backupHealth = $null + $selfHealth = $null + + try { + Write-AgentLog "sensor_gate start no_telegram=true controlled_apply=false remediation=false" + $performance = @(Get-HostPerformance) + $backupHealth = Test-AgentBackupHealth + $selfHealth = Test-AgentSelfHealth + } catch { + $gateError = $_.Exception.Message + Write-AgentLog "sensor_gate error=$gateError" + } finally { + $script:SuppressAgentAlerts = $previousSuppress + } + + $perfReadbackFailed = @($performance | Where-Object { $_ -and ((-not $_.ok) -or (@($_.reasons) -contains "perf_readback_failed")) }) + $perfMissingMetrics = @($performance | Where-Object { + $_ -and $_.ok -and (($null -eq $_.loadPerCore) -or ($null -eq $_.memAvailablePercent) -or ($null -eq $_.diskUsedPercent)) + }) + $perfCritical = @($performance | Where-Object { $_ -and $_.severity -eq "critical" }) + $perfWarning = @($performance | Where-Object { $_ -and $_.severity -eq "warning" }) + $sentTelegram = [bool](@($script:TelegramAttempts | Where-Object { $_.sent }).Count -gt 0) + $suppressedTelegram = @($script:TelegramAttempts | Where-Object { $_.PSObject.Properties["suppressed"] -and $_.suppressed }) + $perfReadbackOk = [bool](@($performance).Count -gt 0 -and @($perfReadbackFailed).Count -eq 0 -and @($perfMissingMetrics).Count -eq 0) + $backupOk = [bool]($backupHealth -and $backupHealth.severity -eq "ok") + $selfOk = [bool]($selfHealth -and $selfHealth.severity -eq "ok") + + $performanceDecision = if (-not $perfReadbackOk) { + "hold_readback_or_metric_gap" + } elseif (@($perfCritical).Count -gt 0) { + "hold_critical_needs_remediation" + } elseif (@($perfWarning).Count -gt 0) { + "hold_warning_needs_summary_or_remediation" + } else { + "eligible" + } + $backupDecision = if ($backupOk) { "eligible" } elseif ($backupHealth) { "hold_$($backupHealth.severity)" } else { "hold_readback_failed" } + $selfDecision = if ($selfOk) { "eligible" } elseif ($selfHealth) { "hold_$($selfHealth.severity)" } else { "hold_readback_failed" } + $readyToEnableAll = [bool]($performanceDecision -eq "eligible" -and $backupDecision -eq "eligible" -and $selfDecision -eq "eligible") + $gateSafe = [bool]((-not $sentTelegram) -and (-not $ControlledApply) -and ($null -eq $gateError)) + + $result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = $gateSafe + readyToEnableAll = $readyToEnableAll + sentTelegram = $sentTelegram + suppressedTelegramCount = @($suppressedTelegram).Count + controlledApply = [bool]$ControlledApply + ranRemediation = $false + error = $gateError + decisions = [pscustomobject]@{ + performanceGuard = $performanceDecision + backupHealth = $backupDecision + selfHealth = $selfDecision + } + performance = [pscustomobject]@{ + ok = $perfReadbackOk + hostCount = @($performance).Count + readbackFailedHosts = @($perfReadbackFailed | ForEach-Object { $_.host }) + missingMetricHosts = @($perfMissingMetrics | ForEach-Object { $_.host }) + warningHosts = @($perfWarning | ForEach-Object { $_.host }) + criticalHosts = @($perfCritical | ForEach-Object { $_.host }) + rows = $performance + } + backupHealth = $backupHealth + selfHealth = $selfHealth + telegramAttempts = $script:TelegramAttempts + events = $script:Events + evidenceLog = $logPath + } + + $selfTestPath = Join-Path $EvidenceDir "agent99-SensorGateSelfTest-$stamp.json" + $result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8 + $result | ConvertTo-Json -Depth 12 | Set-Content -Encoding UTF8 $jsonPath + Write-AgentLog "sensor_gate complete ok=$($result.ok) readyToEnableAll=$readyToEnableAll evidence=$selfTestPath" + $result | ConvertTo-Json -Depth 12 + if (-not $gateSafe) { exit 1 } + exit 0 +} + function Send-AgentTelegram { param( [string]$Severity, @@ -725,6 +813,19 @@ function Record-AgentEvent { return } + if ($script:SuppressAgentAlerts) { + Write-AgentLog "telegram_suppressed_by_sensor_gate type=$EventType severity=$Severity" + $script:TelegramAttempts += [pscustomobject]@{ + timestamp = (Get-Date -Format o) + eventType = $EventType + severity = $Severity + sent = $false + suppressed = $true + reason = "sensor_gate_no_telegram" + } + return + } + $alertAll = ($Config.telegram -and $Config.telegram.alertAllOperations) $alertInfo = ($Config.telegram -and $Config.telegram.alertInfoEvents) $alertOperatorCommands = $true @@ -836,12 +937,14 @@ function Invoke-SshText { ) try { + $stopwatch = [Diagnostics.Stopwatch]::StartNew() $id = [guid]::NewGuid().ToString("N") $stdoutPath = Join-Path $env:TEMP "agent99-ssh-$id.out" $stderrPath = Join-Path $env:TEMP "agent99-ssh-$id.err" $process = Start-Process -FilePath "ssh.exe" -ArgumentList $args -NoNewWindow -RedirectStandardOutput $stdoutPath -RedirectStandardError $stderrPath -PassThru if (-not $process.WaitForExit($TimeoutSeconds * 1000)) { + $stopwatch.Stop() $process.Kill() $process.WaitForExit(2000) | Out-Null Remove-Item $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null @@ -849,9 +952,11 @@ function Invoke-SshText { ok = $false exitCode = -2 output = "timeout_after_${TimeoutSeconds}s" + elapsedMs = $stopwatch.ElapsedMilliseconds } } else { $process.WaitForExit() | Out-Null + $stopwatch.Stop() $process.Refresh() $stdout = if (Test-Path $stdoutPath) { Get-Content $stdoutPath -Raw } else { "" } $stderr = if (Test-Path $stderrPath) { Get-Content $stderrPath -Raw } else { "" } @@ -865,6 +970,7 @@ function Invoke-SshText { ok = ($exitCode -eq 0) exitCode = $exitCode output = $text + elapsedMs = $stopwatch.ElapsedMilliseconds } } } catch { @@ -872,6 +978,7 @@ function Invoke-SshText { ok = $false exitCode = -1 output = $_.Exception.Message + elapsedMs = if ($stopwatch) { $stopwatch.ElapsedMilliseconds } else { $null } } } @@ -943,6 +1050,14 @@ function Get-HostPerformance { param([switch]$SuppressAlerts) $thresholds = Get-PerfThresholds + $readTimeoutSeconds = 45 + if ($Config.performance -and $Config.performance.PSObject.Properties["readTimeoutSeconds"]) { + $readTimeoutSeconds = [int]$Config.performance.readTimeoutSeconds + } + $readRetries = 1 + if ($Config.performance -and $Config.performance.PSObject.Properties["readRetries"]) { + $readRetries = [int]$Config.performance.readRetries + } $results = @() $command = @' echo __AGENT99_PERF__ @@ -956,7 +1071,30 @@ df -P / 2>/dev/null '@ foreach ($hostIp in $Config.hosts) { - $readback = Invoke-HostSshText $hostIp $command 25 1 + $hostTimeoutSeconds = $readTimeoutSeconds + if ($Config.performance -and $Config.performance.PSObject.Properties["hostReadTimeoutSeconds"] -and $Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp]) { + $hostTimeoutSeconds = [int]$Config.performance.hostReadTimeoutSeconds.PSObject.Properties[$hostIp].Value + } + $slowRetryUsed = $false + $slowTimeoutSeconds = $null + $readback = Invoke-HostSshText $hostIp $command $hostTimeoutSeconds $readRetries + if (-not $readback.ok -and $readback.exitCode -eq -2) { + $slowTimeoutSeconds = 90 + if ($Config.performance -and $Config.performance.PSObject.Properties["slowReadTimeoutSeconds"]) { + $slowTimeoutSeconds = [int]$Config.performance.slowReadTimeoutSeconds + } + if ($Config.performance -and $Config.performance.PSObject.Properties["hostSlowReadTimeoutSeconds"] -and $Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp]) { + $slowTimeoutSeconds = [int]$Config.performance.hostSlowReadTimeoutSeconds.PSObject.Properties[$hostIp].Value + } + if ($slowTimeoutSeconds -gt $hostTimeoutSeconds) { + Write-AgentLog "perf slow_retry host=$hostIp firstTimeoutSeconds=$hostTimeoutSeconds slowTimeoutSeconds=$slowTimeoutSeconds" + $slowReadback = Invoke-HostSshText $hostIp $command $slowTimeoutSeconds 1 + $slowReadback | Add-Member -NotePropertyName firstAttemptExitCode -NotePropertyValue $readback.exitCode -Force + $slowReadback | Add-Member -NotePropertyName firstAttemptElapsedMs -NotePropertyValue $readback.elapsedMs -Force + $readback = $slowReadback + $slowRetryUsed = $true + } + } $text = $readback.output $cores = 1 $coresMatch = [regex]::Match($text, "CORES=(\d+)") @@ -1050,8 +1188,12 @@ df -P / 2>/dev/null diskUsedPercent = $diskUsedPercent topCpu = $topCpu output = $text + readTimeoutSeconds = $hostTimeoutSeconds + slowReadTimeoutSeconds = $slowTimeoutSeconds + readbackSlowRetry = $slowRetryUsed + readbackElapsedMs = $readback.elapsedMs } - Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route)" + Write-AgentLog "perf host=$hostIp severity=$severity loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent route=$($readback.route) elapsedMs=$($readback.elapsedMs) timeoutSeconds=$hostTimeoutSeconds slowRetry=$slowRetryUsed" if ($severity -in @("warning", "critical") -and -not $SuppressAlerts) { Record-AgentEvent "performance_$severity" $severity "host=$hostIp loadPerCore=$loadPerCore memAvailPct=$memAvailablePercent diskUsedPct=$diskUsedPercent reasons=$($reasons -join ',')" $result -Alert } @@ -1671,7 +1813,14 @@ function Test-AgentBackupFileCheck { $quotedPath = Quote-ShSingle $path $command = "if [ -f $quotedPath ]; then stat -c '__STAT__=%Y|%y|%s|%n' $quotedPath; else echo __MISSING_FILE__=$quotedPath; fi" - $readback = Invoke-HostSshText $hostIp $command 15 1 + $timeoutSeconds = 45 + if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["fileReadTimeoutSeconds"]) { + $timeoutSeconds = [int]$Config.backupHealth.fileReadTimeoutSeconds + } + if ($FileCheck.PSObject.Properties["timeoutSeconds"]) { + $timeoutSeconds = [int]$FileCheck.timeoutSeconds + } + $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 $unix = $null $size = $null if ($readback.ok) { @@ -1710,8 +1859,10 @@ function Test-AgentBackupFileCheck { maxAgeMinutes = $maxAgeMinutes route = $readback.route exitCode = $readback.exitCode + readTimeoutSeconds = $timeoutSeconds + readbackElapsedMs = $readback.elapsedMs } - Write-AgentLog "backup file=$name severity=$severity reason=$reason ageMinutes=$ageMinutes" + Write-AgentLog "backup file=$name severity=$severity reason=$reason ageMinutes=$ageMinutes elapsedMs=$($readback.elapsedMs) timeoutSeconds=$timeoutSeconds" if ($severity -in @("warning", "critical")) { Record-AgentEvent "backup_$reason" $severity "file=$name host=$hostIp ageMinutes=$ageMinutes maxAgeMinutes=$maxAgeMinutes path=$path" $result -Alert } @@ -1726,7 +1877,11 @@ function Test-AgentBackupCronPattern { $quotedPattern = Quote-ShSingle $Pattern $command = "crontab -l 2>/dev/null | grep -F -- $quotedPattern >/dev/null && echo __PRESENT__ || echo __MISSING__" - $readback = Invoke-HostSshText $DefaultHost $command 8 2 + $timeoutSeconds = 45 + if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) { + $timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds + } + $readback = Invoke-HostSshText $DefaultHost $command $timeoutSeconds 2 $present = [bool]($readback.ok -and $readback.output -match "__PRESENT__") $result = [pscustomobject]@{ host = $DefaultHost @@ -1736,6 +1891,8 @@ function Test-AgentBackupCronPattern { reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } route = $readback.route exitCode = $readback.exitCode + readTimeoutSeconds = $timeoutSeconds + readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "backup cron pattern=$Pattern ok=$present" if (-not $present) { @@ -1750,7 +1907,11 @@ function Test-AgentBackupCronPatterns { [string]$DefaultHost ) - $readback = Invoke-HostSshText $DefaultHost "crontab -l 2>/dev/null" 12 2 + $timeoutSeconds = 45 + if ($Config.backupHealth -and $Config.backupHealth.PSObject.Properties["cronReadTimeoutSeconds"]) { + $timeoutSeconds = [int]$Config.backupHealth.cronReadTimeoutSeconds + } + $readback = Invoke-HostSshText $DefaultHost "crontab -l 2>/dev/null" $timeoutSeconds 2 $results = @() if (-not $readback.ok) { foreach ($pattern in $Patterns) { @@ -1762,6 +1923,8 @@ function Test-AgentBackupCronPatterns { reason = "cron_readback_failed" route = $readback.route exitCode = $readback.exitCode + readTimeoutSeconds = $timeoutSeconds + readbackElapsedMs = $readback.elapsedMs } Record-AgentEvent "backup_cron_readback_failed" "warning" "host=$DefaultHost pattern=$pattern" $result -Alert $results += $result @@ -1780,6 +1943,8 @@ function Test-AgentBackupCronPatterns { reason = if ($present) { "cron_pattern_present" } else { "cron_pattern_missing" } route = $readback.route exitCode = $readback.exitCode + readTimeoutSeconds = $timeoutSeconds + readbackElapsedMs = $readback.elapsedMs } Write-AgentLog "backup cron pattern=$pattern ok=$present" if (-not $present) { @@ -2897,6 +3062,10 @@ if ($SelfTestTelegramDelivery) { Invoke-AgentTelegramDeliverySelfTest } +if ($SelfTestSensorGate) { + Invoke-AgentSensorGateSelfTest +} + Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert