From b3a7e1504a115f87da6ad3568bf038450c0f79ec Mon Sep 17 00:00:00 2001 From: Your Name Date: Fri, 17 Jul 2026 10:32:53 +0800 Subject: [PATCH] fix(agent99): decouple fleet pressure from self health --- agent99-control-plane.ps1 | 69 ++++++++++++++++++- ...t_agent99_alert_dispatch_order_contract.py | 11 +++ 2 files changed, 77 insertions(+), 3 deletions(-) diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index b8ebc6a6d..143614a51 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -4382,17 +4382,25 @@ function Test-AgentSelfHealth { $taskFailures = @($tasks | Where-Object { -not $_.ok }).Count $evidenceCritical = @($evidence | Where-Object { $_.severity -eq "critical" }).Count $evidenceWarning = @($evidence | Where-Object { $_.severity -eq "warning" }).Count + $perfReadbackFailures = @($latestPerf.hosts | Where-Object { + -not $_.present -or (@($_.reasons) -contains "perf_readback_failed") + }) + $perfReadbackFailed = [bool]( + $latestPerf.reason -in @("missing_perf_evidence", "perf_evidence_parse_error") -or + $perfReadbackFailures.Count -gt 0 + ) $critical = $taskFailures + $evidenceCritical - if ($latestPerf.severity -eq "critical") { $critical += 1 } + # Fleet pressure belongs to the Perf lane. Self-health only fails when the + # performance sensor/readback is unavailable or incomplete. + if ($perfReadbackFailed) { $critical += 1 } if ($telegram.severity -eq "critical") { $critical += 1 } if ($runtimeManifest.severity -eq "critical") { $critical += 1 } if ($windowsControlBaseline.severity -eq "critical") { $critical += 1 } if ($runningQueue.severity -eq "critical") { $critical += 1 } $warning = $evidenceWarning - if ($latestPerf.severity -eq "warning") { $warning += 1 } $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } - $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity) runtimeManifest=$($runtimeManifest.severity) windowsControlBaseline=$($windowsControlBaseline.severity) staleRunningQueue=$($runningQueue.staleCount)" + $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perfSignal=$($latestPerf.severity) perfReadbackFailures=$($perfReadbackFailures.Count) telegram=$($telegram.severity) runtimeManifest=$($runtimeManifest.severity) windowsControlBaseline=$($windowsControlBaseline.severity) staleRunningQueue=$($runningQueue.staleCount)" Record-AgentEvent "agent_selfcheck" $(if ($severity -eq "ok") { "info" } else { $severity }) $message $null -Alert [pscustomobject]@{ @@ -4410,6 +4418,8 @@ function Test-AgentSelfHealth { evidenceCritical = $evidenceCritical evidenceWarning = $evidenceWarning performanceSeverity = $latestPerf.severity + performanceReadbackOk = [bool](-not $perfReadbackFailed) + performanceReadbackFailureCount = $perfReadbackFailures.Count telegramSeverity = $telegram.severity runtimeManifestSeverity = $runtimeManifest.severity runtimeManifestSourceRevision = $runtimeManifest.sourceRevision @@ -5330,6 +5340,11 @@ function Get-AgentOutcomeContract { } "SelfCheck" { $selfOk = [bool]($data.selfHealth -and $data.selfHealth.severity -eq "ok") + $fleetPerformanceClear = [bool]( + $data.selfHealth -and + $data.selfHealth.latestPerformance -and + $data.selfHealth.latestPerformance.severity -eq "ok" + ) $windowsBaseline = if ($data.selfHealth -and $data.selfHealth.PSObject.Properties["windowsControlBaseline"]) { $data.selfHealth.windowsControlBaseline } else { $null } $windowsBaselineOk = [bool]( $windowsBaseline -and @@ -5340,6 +5355,7 @@ function Get-AgentOutcomeContract { $windowsBaseline.evidenceSha256 -match "^[0-9a-f]{64}$" ) $checks += New-AgentOutcomeCheck "agent_self_health_ok" $selfOk + $checks += New-AgentOutcomeCheck "fleet_performance_signal_clear" $fleetPerformanceClear $false "severity=$([string]$data.selfHealth.latestPerformance.severity)" $checks += New-AgentOutcomeCheck "windows99_control_baseline_no_write" $windowsBaselineOk if ($windowsBaselineOk) { $modeEvidenceRefs["windows_control_baseline_evidence_ref"] = [string]$windowsBaseline.evidenceRef @@ -5509,6 +5525,46 @@ function Invoke-AgentOutcomeContractSelfTest { public = @([pscustomobject]@{ url = "https://example.invalid"; status = 502; non502 = $false; ok = $false }) } } + $selfCheckFleetWarningEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-selfcheck-fleet-warning.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "SelfCheck" + selfHealth = [pscustomobject]@{ + severity = "ok" + latestPerformance = [pscustomobject]@{ severity = "warning" } + windowsControlBaseline = [pscustomobject]@{ + ok = $true + runtimeWritePerformed = $false + reason = "windows_control_baseline_verified_no_write" + evidenceRef = "agent99-WindowsControlBaseline-selftest.json" + evidenceSha256 = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + } + } + } + } + $selfCheckControlPlaneFailedEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-selfcheck-control-plane-failed.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "SelfCheck" + selfHealth = [pscustomobject]@{ + severity = "critical" + latestPerformance = [pscustomobject]@{ severity = "ok" } + windowsControlBaseline = [pscustomobject]@{ + ok = $true + runtimeWritePerformed = $false + reason = "windows_control_baseline_verified_no_write" + evidenceRef = "agent99-WindowsControlBaseline-selftest.json" + evidenceSha256 = "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb" + } + } + } + } $providerFixtureNow = [DateTimeOffset]::Parse("2026-07-16T14:00:00Z").UtcDateTime function New-AgentProviderSelfTestResponse { param([string]$Provider, [string]$LatestReceivedAt, [int]$SourceCount) @@ -5580,6 +5636,8 @@ function Invoke-AgentOutcomeContractSelfTest { $providerMixedDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerMixedEvidence $startedAt $true $false "" "mode_verifier_can_resolve" $providerFreshResolved = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerFreshEvidence $startedAt $true $false "" "mode_verifier_can_resolve" $providerGenericDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerGenericEvidence $startedAt $true $false "" "mode_verifier_can_resolve" + $selfCheckFleetWarningResolved = Get-AgentOutcomeContract "SelfCheck" 0 $selfCheckFleetWarningEvidence $startedAt $true $false "" "mode_verifier_can_resolve" + $selfCheckControlPlaneFailed = Get-AgentOutcomeContract "SelfCheck" 0 $selfCheckControlPlaneFailedEvidence $startedAt $true $false "" "mode_verifier_can_resolve" $securityCandidate = Get-AgentOutcomeContract "SecurityTriage" 0 $securityEvidence $startedAt $true $false $ok = [bool]( $operatorResolved.state -eq "resolved" -and $operatorResolved.verifierPassed -and @@ -5591,6 +5649,9 @@ function Invoke-AgentOutcomeContractSelfTest { $providerMixedDegraded.state -eq "degraded" -and -not $providerMixedDegraded.verifierPassed -and $providerFreshResolved.state -eq "resolved" -and $providerFreshResolved.verifierPassed -and $providerFreshResolved.sourceEventResolved -and $providerGenericDegraded.state -eq "degraded" -and -not $providerGenericDegraded.verifierPassed -and + $selfCheckFleetWarningResolved.state -eq "resolved" -and $selfCheckFleetWarningResolved.verifierPassed -and $selfCheckFleetWarningResolved.sourceEventResolved -and + @($selfCheckFleetWarningResolved.checks | Where-Object { $_.name -eq "fleet_performance_signal_clear" -and -not $_.passed -and -not $_.required }).Count -eq 1 -and + $selfCheckControlPlaneFailed.state -eq "degraded" -and -not $selfCheckControlPlaneFailed.verifierPassed -and $securityCandidate.state -eq "candidate_ready" ) $result = [pscustomobject]@{ @@ -5607,6 +5668,8 @@ function Invoke-AgentOutcomeContractSelfTest { providerMixedDegraded = $providerMixedDegraded providerFreshResolved = $providerFreshResolved providerGenericDegraded = $providerGenericDegraded + selfCheckFleetWarningResolved = $selfCheckFleetWarningResolved + selfCheckControlPlaneFailed = $selfCheckControlPlaneFailed securityCandidate = $securityCandidate } } diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py index 73abb7632..1df3ae716 100644 --- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -243,9 +243,20 @@ def test_agent99_selfcheck_verifies_windows99_control_baseline_no_write() -> Non assert '"agent99-WindowsControlBaseline-$runId.json"' not in verifier assert 'receipt.evidenceIdentity.runToken -eq $evidenceRunToken' in verifier assert "$windowsControlBaseline = Test-AgentWindowsControlBaseline" in selfcheck + assert "$perfReadbackFailures = @($latestPerf.hosts" in selfcheck + assert '(@($_.reasons) -contains "perf_readback_failed")' in selfcheck + assert "if ($perfReadbackFailed) { $critical += 1 }" in selfcheck + assert 'if ($latestPerf.severity -eq "warning") { $warning += 1 }' not in selfcheck + assert "performanceReadbackOk = [bool](-not $perfReadbackFailed)" in selfcheck assert 'windowsControlBaseline = $windowsControlBaseline' in selfcheck assert 'windowsControlBaselineSeverity = $windowsControlBaseline.severity' in selfcheck + assert ( + 'New-AgentOutcomeCheck "fleet_performance_signal_clear" ' + "$fleetPerformanceClear $false" in source + ) assert 'New-AgentOutcomeCheck "windows99_control_baseline_no_write"' in source + assert '$selfCheckFleetWarningResolved.state -eq "resolved"' in source + assert '$selfCheckControlPlaneFailed.state -eq "degraded"' in source assert '"windows_control_baseline_evidence_ref"' in source