From 85cb7fba426ad193ed81fc4b389267f2a7e23607 Mon Sep 17 00:00:00 2001 From: ogt Date: Fri, 10 Jul 2026 16:05:54 +0800 Subject: [PATCH] fix(agent99): require verified outcomes before resolution --- AGENT99-AI-WORKSTATION-SOP.md | 11 + agent99-bootstrap.ps1 | 83 ++- agent99-control-plane.ps1 | 527 +++++++++++++++++- ..._agent99_control_plane_outcome_contract.py | 59 ++ ...ise-ai-automation-work-items.snapshot.json | 20 +- 5 files changed, 669 insertions(+), 31 deletions(-) create mode 100644 apps/api/tests/test_agent99_control_plane_outcome_contract.py diff --git a/AGENT99-AI-WORKSTATION-SOP.md b/AGENT99-AI-WORKSTATION-SOP.md index b20123aec..6ad3e7317 100644 --- a/AGENT99-AI-WORKSTATION-SOP.md +++ b/AGENT99-AI-WORKSTATION-SOP.md @@ -15,6 +15,17 @@ This SOP defines how Agent99 operates. The canonical work ledger defines what re Host 99 is not the only availability observer. An external watcher and maintenance surface must remain available when 99 or the entire local failure domain is down. +## 2026-07-10 P0-001 Outcome Truth Contract + +- `exitCode=0` means only that the PowerShell transport completed. +- A command is `resolved` only when its mode-specific post-condition verifier passes, its evidence is fresh and mode-matched, and any alert-source event has an explicit resolved readback. +- Other legal outcomes are `candidate_ready`, `verifying`, `degraded`, `blocked`, and `failed`. +- Alert-originated work without `sourceEventResolved=true` remains `verifying`, even when the repair command and post-condition verifier succeeded. +- `problem-counts.json` v2 preserves the legacy `totalResolved` field but labels it as possibly transport-derived. New trustworthy closure is counted separately in `totalVerifiedResolved` and per-problem `verifiedResolved`. +- Bootstrap writes `state\runtime-manifest.json` with source revision, source/runtime SHA-256, content match, file count and mismatch count. Self-health is critical when this manifest is missing, invalid, stale by hash, or incomplete. +- All `.ps1` runtime files are normalized to UTF-8 BOM by bootstrap so Windows PowerShell 5 cannot corrupt Traditional Chinese strings during parsing. +- Staging verification on 99 must pass `-SelfTestOutcomeContract` and `runtimeMatched=true` before production Agent99 files are replaced. + It must act as: - Secretary: reminders, status summaries, evidence packets, operator updates. diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 8537512f3..430d2ba91 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -1,5 +1,6 @@ param( [string]$AgentRoot = "C:\Wooo\Agent99", + [string]$SourceRevision = "", [switch]$InstallScheduledTasks ) @@ -15,8 +16,9 @@ $RequestDir = Join-Path $AgentRoot "requests" $AlertDir = Join-Path $AgentRoot "alerts" $PlaybookDir = Join-Path $AgentRoot "playbooks" $DashboardDir = Join-Path $AgentRoot "dashboard" +$StateDir = Join-Path $AgentRoot "state" -foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir, $EvidenceDir, $LogDir, $QueueDir, $RequestDir, $AlertDir, $PlaybookDir, $DashboardDir)) { +foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir, $EvidenceDir, $LogDir, $QueueDir, $RequestDir, $AlertDir, $PlaybookDir, $DashboardDir, $StateDir)) { New-Item -ItemType Directory -Force -Path $dir | Out-Null } @@ -28,18 +30,79 @@ function Copy-AgentFile { $srcFull = [System.IO.Path]::GetFullPath($src) $dstFull = [System.IO.Path]::GetFullPath($dst) if ($srcFull -ne $dstFull) { - Copy-Item -Force $src $dst + if ([System.IO.Path]::GetExtension($src).ToLowerInvariant() -eq ".ps1") { + $content = [System.IO.File]::ReadAllText($src) + $utf8Bom = [System.Text.UTF8Encoding]::new($true) + [System.IO.File]::WriteAllText($dst, $content, $utf8Bom) + } else { + Copy-Item -Force $src $dst + } } } } -Copy-AgentFile "agent99-control-plane.ps1" -Copy-AgentFile "agent99-register-tasks.ps1" -Copy-AgentFile "agent99-submit-request.ps1" -Copy-AgentFile "agent99-telegram-inbox.ps1" -Copy-AgentFile "agent99-sre-alert-inbox.ps1" -Copy-AgentFile "agent99-sre-alert-relay.ps1" -Copy-AgentFile "host-reboot-scorecard.ps1" +$agentFiles = @( + "agent99-control-plane.ps1", + "agent99-register-tasks.ps1", + "agent99-submit-request.ps1", + "agent99-telegram-inbox.ps1", + "agent99-sre-alert-inbox.ps1", + "agent99-sre-alert-relay.ps1", + "host-reboot-scorecard.ps1" +) +foreach ($agentFile in $agentFiles) { + Copy-AgentFile $agentFile +} + +$resolvedSourceRevision = $SourceRevision +if (-not $resolvedSourceRevision) { + try { + $gitRevision = & git -C $SourceRoot rev-parse HEAD 2>$null + if ($LASTEXITCODE -eq 0 -and $gitRevision) { + $resolvedSourceRevision = ([string]$gitRevision).Trim() + } + } catch { + $resolvedSourceRevision = "" + } +} +if (-not $resolvedSourceRevision) { $resolvedSourceRevision = "unknown" } + +$manifestRows = @() +foreach ($agentFile in $agentFiles) { + $sourcePath = Join-Path $SourceRoot $agentFile + $runtimePath = Join-Path $BinDir $agentFile + $sourceExists = Test-Path $sourcePath + $runtimeExists = Test-Path $runtimePath + $sourceSha256 = if ($sourceExists) { (Get-FileHash -Algorithm SHA256 -LiteralPath $sourcePath).Hash.ToLowerInvariant() } else { $null } + $runtimeSha256 = if ($runtimeExists) { (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimePath).Hash.ToLowerInvariant() } else { $null } + $contentMatched = if ($sourceExists -and $runtimeExists) { [System.IO.File]::ReadAllText($sourcePath) -ceq [System.IO.File]::ReadAllText($runtimePath) } else { $false } + $manifestRows += [pscustomobject]@{ + name = $agentFile + sourceExists = $sourceExists + runtimeExists = $runtimeExists + sourceSha256 = $sourceSha256 + runtimeSha256 = $runtimeSha256 + contentMatched = [bool]$contentMatched + matched = [bool]$contentMatched + } +} +$runtimeMatched = [bool](@($manifestRows).Count -eq @($agentFiles).Count -and @($manifestRows | Where-Object { -not $_.matched }).Count -eq 0) +$runtimeManifest = [pscustomobject]@{ + schemaVersion = "agent99_runtime_manifest_v1" + generatedAt = (Get-Date -Format o) + sourceRevision = $resolvedSourceRevision + runtimeMatched = $runtimeMatched + fileCount = @($manifestRows).Count + mismatchCount = @($manifestRows | Where-Object { -not $_.matched }).Count + files = $manifestRows +} +$manifestPath = Join-Path $StateDir "runtime-manifest.json" +$manifestEvidencePath = Join-Path $EvidenceDir ("agent99-RuntimeManifest-" + (Get-Date -Format "yyyyMMdd-HHmmss") + ".json") +$runtimeManifest | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $manifestPath +$runtimeManifest | ConvertTo-Json -Depth 8 | Set-Content -Encoding UTF8 $manifestEvidencePath +if (-not $runtimeMatched) { + throw "Agent99 runtime manifest mismatch; inspect $manifestEvidencePath" +} $exampleConfig = Join-Path $SourceRoot "agent99.config.example.json" $configPath = Join-Path $ConfigDir "agent99.config.json" @@ -158,6 +221,8 @@ Write-Host "Agent99 bootstrap complete" Write-Host "AgentRoot: $AgentRoot" Write-Host "Config: $configPath" Write-Host "Evidence: $EvidenceDir" +Write-Host "Runtime manifest: $manifestPath" +Write-Host "Source revision: $resolvedSourceRevision" if ($vmrun) { Write-Host "vmrun: $vmrun" } else { diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 2342f929c..2534c181f 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -5,6 +5,7 @@ param( [switch]$SelfTestTelegramCard, [switch]$SelfTestTelegramDelivery, [switch]$SelfTestSensorGate, + [switch]$SelfTestOutcomeContract, [switch]$SuppressAlerts, [string]$SuppressReason = "", [string]$ConfigPath = "C:\Wooo\Agent99\config\agent99.config.json", @@ -350,7 +351,16 @@ function Format-AgentTelegramText { $instruction = if ($Data -and $Data.PSObject.Properties["instruction"]) { $Data.instruction } else { $null } $exitCode = if ($Data -and $Data.PSObject.Properties["exitCode"]) { $Data.exitCode } else { $null } $evidencePath = if ($Data -and $Data.PSObject.Properties["evidence"]) { $Data.evidence } else { $null } - $resultLabel = if ($exitCode -eq 0 -or [string](Get-AgentObjectValue $Data "ok" "") -match "^(?i:true|1)$") { "已完成 / resolved" } else { "仍需處理 / failed" } + $outcome = if ($Data -and $Data.PSObject.Properties["outcome"]) { $Data.outcome } else { $null } + $outcomeState = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } else { "failed" } + $resultLabel = switch ($outcomeState) { + "resolved" { "已驗證解決 / verified resolved" } + "candidate_ready" { "修復候選已建立 / candidate ready" } + "verifying" { "等待來源事件驗證 / verifying" } + "degraded" { "仍有異常 / degraded" } + "blocked" { "受阻擋 / blocked" } + default { "執行失敗 / failed" } + } $actionText = switch ($modeName) { "ProviderFreshness" { "建立 provider freshness candidate,要求 provider window、last seen、direct reference、verifier readback;不切 provider。" } "BackupCheck" { "讀回備份 freshness、status/log、snapshot metadata、cron contract;不讀備份內容或 secrets。" } @@ -366,12 +376,15 @@ function Format-AgentTelegramText { $lines += "事件 Event: Agent99 自動化結果 / automation result" $lines += "模式 Mode: $modeName" $lines += "來源 Source: $source" - $lines += "結果 Result: $resultLabel, exitCode=$exitCode" + $lines += "結果 Result: $resultLabel, transportExitCode=$exitCode" + if ($outcome) { + $lines += "驗證 Verifier: name=$($outcome.verifierName), passed=$($outcome.verifierPassed), sourceEventResolved=$($outcome.sourceEventResolved)" + } $lines += "Agent99 動作 Action: $actionText" if ($Data -and $Data.PSObject.Properties["problem"]) { $problem = $Data.problem if ($problem -and $problem.PSObject.Properties["occurrences"]) { - $lines += "問題計數 Problem count: occurrences=$($problem.occurrences), resolved=$($problem.resolved), failed=$($problem.failed)" + $lines += "問題計數 Problem count: occurrences=$($problem.occurrences), verifiedResolved=$($problem.verifiedResolved), failed=$($problem.failed), degraded=$($problem.degraded), blocked=$($problem.blocked), verifying=$($problem.verifying), candidateReady=$($problem.candidateReady)" if ($problem.PSObject.Properties["kmPath"]) { $lines += "KM: $($problem.kmPath)" } @@ -604,11 +617,22 @@ function Invoke-AgentTelegramCardSelfTest { ok = $true exitCode = 0 evidence = $jsonPath + outcome = [pscustomobject]@{ + state = "resolved" + verifierName = "selftest-verifier" + verifierPassed = $true + sourceEventResolved = $true + } problem = [pscustomobject]@{ key = "selftest-provider-freshness" occurrences = 2 resolved = 2 + verifiedResolved = 2 failed = 0 + degraded = 0 + blocked = 0 + verifying = 0 + candidateReady = 0 kmPath = (Join-Path $Config.agentRoot "playbooks\agent99-incident-km.md") } } @@ -616,7 +640,7 @@ function Invoke-AgentTelegramCardSelfTest { $operatorImagePath = New-AgentTelegramCardImage "info" "operator_command_result" $operatorText $operatorSample $operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample $operatorTextOk = [bool]( - $operatorText -match "已完成 / resolved" -and + $operatorText -match "已驗證解決 / verified resolved" -and $operatorText -match "Agent99 動作 Action" -and $operatorText -match "問題計數 Problem count" -and $operatorText -match "KM:" @@ -1721,11 +1745,82 @@ function Test-AgentRecentTelegramDelivery { } } +function Test-AgentRuntimeManifest { + $manifestPath = Join-Path $Config.agentRoot "state\runtime-manifest.json" + if (-not (Test-Path $manifestPath)) { + return [pscustomobject]@{ + ok = $false + severity = "critical" + reason = "runtime_manifest_missing" + manifestPath = $manifestPath + sourceRevision = $null + driftCount = 0 + missingCount = 1 + files = @() + } + } + + try { + $manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json + } catch { + return [pscustomobject]@{ + ok = $false + severity = "critical" + reason = "runtime_manifest_parse_failed" + manifestPath = $manifestPath + sourceRevision = $null + driftCount = 0 + missingCount = 0 + files = @() + error = $_.Exception.Message + } + } + + $rows = @() + foreach ($entry in @($manifest.files)) { + $name = [string]$entry.name + $runtimePath = Join-Path $PSScriptRoot $name + $exists = Test-Path $runtimePath + $actualSha256 = $null + if ($exists) { + try { + $actualSha256 = (Get-FileHash -Algorithm SHA256 -LiteralPath $runtimePath).Hash.ToLowerInvariant() + } catch { + $actualSha256 = $null + } + } + $expectedSha256 = if ($entry.runtimeSha256) { ([string]$entry.runtimeSha256).ToLowerInvariant() } elseif ($entry.sourceSha256) { ([string]$entry.sourceSha256).ToLowerInvariant() } else { "" } + $matched = [bool]($exists -and $actualSha256 -and $expectedSha256 -and $actualSha256 -eq $expectedSha256) + $rows += [pscustomobject]@{ + name = $name + exists = $exists + matched = $matched + expectedSha256 = $expectedSha256 + actualSha256 = $actualSha256 + } + } + $missingCount = @($rows | Where-Object { -not $_.exists }).Count + $driftCount = @($rows | Where-Object { $_.exists -and -not $_.matched }).Count + $ready = [bool](@($rows).Count -gt 0 -and $missingCount -eq 0 -and $driftCount -eq 0 -and $manifest.runtimeMatched -eq $true) + [pscustomobject]@{ + ok = $ready + severity = if ($ready) { "ok" } else { "critical" } + reason = if ($ready) { "runtime_manifest_matched" } else { "runtime_source_drift_or_missing" } + manifestPath = $manifestPath + sourceRevision = $manifest.sourceRevision + generatedAt = $manifest.generatedAt + driftCount = $driftCount + missingCount = $missingCount + files = $rows + } +} + function Test-AgentSelfHealth { $selfConfig = Get-AgentSelfHealthConfig $tasks = Test-AgentScheduledTaskHealth $selfConfig.scheduledTasks $evidence = Test-AgentEvidenceFreshness $selfConfig.evidenceFreshness $latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig.requiredHosts + $runtimeManifest = Test-AgentRuntimeManifest $telegram = if ($selfConfig.requireRecentTelegramDelivery) { Test-AgentRecentTelegramDelivery $selfConfig.evidenceFreshness } else { @@ -1738,11 +1833,12 @@ function Test-AgentSelfHealth { $critical = $taskFailures + $evidenceCritical if ($latestPerf.severity -eq "critical") { $critical += 1 } if ($telegram.severity -eq "critical") { $critical += 1 } + if ($runtimeManifest.severity -eq "critical") { $critical += 1 } $warning = $evidenceWarning if ($latestPerf.severity -eq "warning") { $warning += 1 } $severity = if ($critical -gt 0) { "critical" } elseif ($warning -gt 0) { "warning" } else { "ok" } - $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity)" + $message = "selfHealth severity=$severity taskFailures=$taskFailures evidenceCritical=$evidenceCritical evidenceWarning=$evidenceWarning perf=$($latestPerf.severity) telegram=$($telegram.severity) runtimeManifest=$($runtimeManifest.severity)" Record-AgentEvent "agent_selfcheck" $(if ($severity -eq "ok") { "info" } else { $severity }) $message $null -Alert [pscustomobject]@{ @@ -1752,12 +1848,16 @@ function Test-AgentSelfHealth { evidenceFreshness = $evidence latestPerformance = $latestPerf telegramDelivery = $telegram + runtimeManifest = $runtimeManifest summary = [pscustomobject]@{ taskFailures = $taskFailures evidenceCritical = $evidenceCritical evidenceWarning = $evidenceWarning performanceSeverity = $latestPerf.severity telegramSeverity = $telegram.severity + runtimeManifestSeverity = $runtimeManifest.severity + runtimeManifestSourceRevision = $runtimeManifest.sourceRevision + runtimeManifestDriftCount = $runtimeManifest.driftCount } } } @@ -2146,15 +2246,316 @@ function Convert-AgentSafeKey { return $safe } +function New-AgentOutcomeCheck { + param( + [string]$Name, + [bool]$Passed, + [bool]$Required = $true, + [string]$Detail = "" + ) + [pscustomobject]@{ + name = $Name + passed = $Passed + required = $Required + detail = $Detail + } +} + +function Get-AgentOutcomeContract { + param( + [string]$ModeName, + [int]$TransportExitCode, + [object]$LatestEvidence, + [datetime]$StartedAt, + [bool]$SourceEventResolutionRequired, + [object]$SourceEventResolved = $null, + [string]$SourceEventEvidence = "" + ) + + $checks = @() + $transportOk = [bool]($TransportExitCode -eq 0) + $evidenceExists = [bool]($LatestEvidence -and $LatestEvidence.exists) + $evidenceParsed = [bool]($evidenceExists -and -not $LatestEvidence.parseError -and $LatestEvidence.data) + $evidenceFresh = $false + if ($evidenceExists -and $LatestEvidence.lastWriteTime) { + try { + $evidenceTime = [datetime]::Parse([string]$LatestEvidence.lastWriteTime) + $evidenceFresh = [bool]($evidenceTime -ge $StartedAt.AddSeconds(-5)) + } catch { + $evidenceFresh = $false + } + } + $data = if ($evidenceParsed) { $LatestEvidence.data } else { $null } + $evidenceMode = if ($data -and $data.PSObject.Properties["mode"]) { [string]$data.mode } else { "" } + $modeMatches = [bool]($evidenceMode -eq $ModeName) + + $checks += New-AgentOutcomeCheck "transport_completed" $transportOk $true "exitCode=$TransportExitCode" + $checks += New-AgentOutcomeCheck "evidence_exists" $evidenceExists $true ([string]$LatestEvidence.path) + $checks += New-AgentOutcomeCheck "evidence_parsed" $evidenceParsed $true ([string]$LatestEvidence.parseError) + $checks += New-AgentOutcomeCheck "evidence_fresh" $evidenceFresh $true ([string]$LatestEvidence.lastWriteTime) + $checks += New-AgentOutcomeCheck "evidence_mode_matches" $modeMatches $true "expected=$ModeName actual=$evidenceMode" + + $verifierName = "agent99-$($ModeName.ToLowerInvariant())-post-condition-v1" + $modeVerifierPassed = $false + $successState = "resolved" + $failureState = "degraded" + + if ($data) { + switch ($ModeName) { + "Status" { + $expectedHosts = @($Config.hosts).Count + $hostRows = @($data.hosts) + $hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0) + $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) + $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) + $publicRows = @($data.public) + $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) + $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) + $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" + $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk + $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk + $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk + $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false + $modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk) + } + "Recover" { + $expectedVms = @($Config.vms).Count + $vmRows = @($data.vmResults) + $vmsOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0) + $expectedHosts = @($Config.hosts).Count + $hostRows = @($data.hosts) + $hostsOk = [bool]($expectedHosts -gt 0 -and $hostRows.Count -ge $expectedHosts -and @($hostRows | Where-Object { -not $_.ping }).Count -eq 0) + $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) + $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) + $publicRows = @($data.public) + $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) + $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" + $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" + $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk + $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk + $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk + $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk) + } + "StartVMs" { + $expectedVms = @($Config.vms).Count + $vmRows = @($data.vmResults) + $commandOk = [bool]($expectedVms -gt 0 -and $vmRows.Count -ge $expectedVms -and @($vmRows | Where-Object { -not $_.ok }).Count -eq 0) + $unreachable = @() + foreach ($vm in @($Config.vms)) { + if (-not $vm.host) { $unreachable += [string]$vm.name; continue } + $reachable = Test-Connection -ComputerName ([string]$vm.host) -Count 1 -Quiet -ErrorAction SilentlyContinue + if (-not $reachable) { $unreachable += [string]$vm.name } + } + $guestsReachable = [bool]($expectedVms -gt 0 -and @($unreachable).Count -eq 0) + $checks += New-AgentOutcomeCheck "vm_start_commands_ok" $commandOk $true "expected=$expectedVms actual=$($vmRows.Count)" + $checks += New-AgentOutcomeCheck "vm_guests_reachable" $guestsReachable $true "unreachable=$($unreachable -join ',')" + $modeVerifierPassed = [bool]($commandOk -and $guestsReachable) + } + "PublicSmoke" { + $rows = @($data.public) + $routesOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or -not $_.non502 }).Count -eq 0) + $checks += New-AgentOutcomeCheck "public_routes_non502" $routesOk $true "count=$($rows.Count)" + $modeVerifierPassed = $routesOk + } + "HarborRepair" { + $harborOk = [bool]($data.harbor -and $data.harbor.redisUp -and $data.harbor.coreHealthy -and $data.harbor.registryHealthy) + $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk + $modeVerifierPassed = $harborOk + } + "AwoooRepair" { + $awoooOk = [bool]($data.awoooi -and $data.awoooi.deployReady -and -not $data.awoooi.imagePullFailure -and -not $data.awoooi.crashFailure) + $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk + $modeVerifierPassed = $awoooOk + } + "Perf" { + $rows = @($data.performance) + $readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0) + $pressureClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0) + $checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk + $checks += New-AgentOutcomeCheck "performance_pressure_cleared" $pressureClear + $modeVerifierPassed = [bool]($readbackOk -and $pressureClear) + } + "LoadShed" { + $rows = @($data.performance) + $readbackOk = [bool]($rows.Count -gt 0 -and @($rows | Where-Object { -not $_.ok -or (@($_.reasons) -contains "perf_readback_failed") }).Count -eq 0) + $executed = @($data.loadShedding | Where-Object { -not $_.skipped }) + $postVerifyOk = [bool]($executed.Count -gt 0 -and @($executed | Where-Object { -not $_.postVerifyOk }).Count -eq 0) + $alreadyClear = [bool]($readbackOk -and @($rows | Where-Object { $_.severity -in @("warning", "critical") }).Count -eq 0) + $checks += New-AgentOutcomeCheck "performance_readback_complete" $readbackOk + $checks += New-AgentOutcomeCheck "load_shed_post_verify" ([bool]($postVerifyOk -or $alreadyClear)) $true "executed=$($executed.Count)" + $modeVerifierPassed = [bool]($readbackOk -and ($postVerifyOk -or $alreadyClear)) + } + "SelfCheck" { + $selfOk = [bool]($data.selfHealth -and $data.selfHealth.severity -eq "ok") + $checks += New-AgentOutcomeCheck "agent_self_health_ok" $selfOk + $modeVerifierPassed = $selfOk + } + "BackupCheck" { + $backupOk = [bool]($data.backupHealth -and $data.backupHealth.severity -eq "ok") + $checks += New-AgentOutcomeCheck "backup_health_ok" $backupOk + $modeVerifierPassed = $backupOk + } + "ProviderFreshness" { + $provider = $data.providerFreshness + $directOk = [bool]($provider -and $provider.status -eq "direct_readback_ok" -and @($provider.missingFields).Count -eq 0 -and $provider.lastSeen -and $provider.verifierReadback -and $provider.verifierReadback.ok -eq $true -and [int]$provider.verifierReadback.sourceEventTotal -gt 0) + $checks += New-AgentOutcomeCheck "provider_freshness_direct_readback" $directOk $true ([string]$provider.status) + $modeVerifierPassed = $directOk + } + "SecurityTriage" { + $candidateReady = [bool]($data.securityTriage -and $data.securityTriage.status -eq "security_triage_candidate_created") + $checks += New-AgentOutcomeCheck "security_candidate_created" $candidateReady + $modeVerifierPassed = $candidateReady + $successState = "candidate_ready" + } + "ControlTick" { + $controlOk = [bool]($data.controlTick -and $data.controlTick.ok) + $checks += New-AgentOutcomeCheck "control_tick_completed" $controlOk + $modeVerifierPassed = $controlOk + } + default { + $checks += New-AgentOutcomeCheck "mode_verifier_defined" $false + $failureState = "blocked" + } + } + } + + $basePassed = [bool]($transportOk -and $evidenceExists -and $evidenceParsed -and $evidenceFresh -and $modeMatches) + $verifierPassed = [bool]($basePassed -and $modeVerifierPassed) + $sourceEventResolvedBool = if (-not $SourceEventResolutionRequired) { $true } else { [bool]($SourceEventResolved -eq $true) } + $state = if (-not $transportOk) { + "failed" + } elseif (-not $basePassed) { + "blocked" + } elseif (-not $modeVerifierPassed) { + $failureState + } elseif ($successState -eq "candidate_ready") { + "candidate_ready" + } elseif ($SourceEventResolutionRequired -and -not $sourceEventResolvedBool) { + "verifying" + } else { + "resolved" + } + $failedChecks = @($checks | Where-Object { $_.required -and -not $_.passed } | ForEach-Object { $_.name }) + + [pscustomobject]@{ + schemaVersion = "agent99_outcome_contract_v1" + state = $state + resolved = [bool]($state -eq "resolved") + transportOk = $transportOk + transportExitCode = $TransportExitCode + verifierName = $verifierName + verifierPassed = $verifierPassed + evidenceFresh = $evidenceFresh + evidenceModeMatched = $modeMatches + sourceEventResolutionRequired = $SourceEventResolutionRequired + sourceEventResolved = $sourceEventResolvedBool + sourceEventEvidence = if ($SourceEventEvidence) { $SourceEventEvidence } else { $null } + failedChecks = $failedChecks + checks = $checks + verifiedAt = (Get-Date -Format o) + } +} + +function Invoke-AgentOutcomeContractSelfTest { + $startedAt = (Get-Date).AddSeconds(-1) + $publicOkEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-public-ok.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "PublicSmoke" + public = @([pscustomobject]@{ url = "https://example.invalid"; status = 200; non502 = $true; ok = $true }) + } + } + $publicFailedEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-public-failed.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "PublicSmoke" + public = @([pscustomobject]@{ url = "https://example.invalid"; status = 502; non502 = $false; ok = $false }) + } + } + $providerEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-provider.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "ProviderFreshness" + providerFreshness = [pscustomobject]@{ + status = "direct_readback_ok_no_source_events" + missingFields = @("lastSeen") + lastSeen = $null + verifierReadback = [pscustomobject]@{ ok = $true; sourceEventTotal = 0 } + } + } + } + $securityEvidence = [pscustomobject]@{ + exists = $true + path = "selftest-security.json" + lastWriteTime = (Get-Date -Format o) + parseError = $null + data = [pscustomobject]@{ + mode = "SecurityTriage" + securityTriage = [pscustomobject]@{ status = "security_triage_candidate_created"; hitCount = 1 } + } + } + + $operatorResolved = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $false + $alertVerifying = Get-AgentOutcomeContract "PublicSmoke" 0 $publicOkEvidence $startedAt $true $false "selftest-source-event" + $failedPostCondition = Get-AgentOutcomeContract "PublicSmoke" 0 $publicFailedEvidence $startedAt $false + $transportFailed = Get-AgentOutcomeContract "PublicSmoke" 7 $publicOkEvidence $startedAt $false + $providerDegraded = Get-AgentOutcomeContract "ProviderFreshness" 0 $providerEvidence $startedAt $true $false + $securityCandidate = Get-AgentOutcomeContract "SecurityTriage" 0 $securityEvidence $startedAt $true $false + $ok = [bool]( + $operatorResolved.state -eq "resolved" -and $operatorResolved.verifierPassed -and + $alertVerifying.state -eq "verifying" -and -not $alertVerifying.resolved -and + $failedPostCondition.state -eq "degraded" -and -not $failedPostCondition.verifierPassed -and + $transportFailed.state -eq "failed" -and + $providerDegraded.state -eq "degraded" -and + $securityCandidate.state -eq "candidate_ready" + ) + $result = [pscustomobject]@{ + timestamp = (Get-Date -Format o) + ok = $ok + contract = "agent99_outcome_contract_v1" + cases = [pscustomobject]@{ + operatorResolved = $operatorResolved + alertVerifying = $alertVerifying + failedPostCondition = $failedPostCondition + transportFailed = $transportFailed + providerDegraded = $providerDegraded + securityCandidate = $securityCandidate + } + } + $selfTestPath = Join-Path $EvidenceDir "agent99-OutcomeContractSelfTest-$stamp.json" + $result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8 + $result | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8 + Write-AgentLog "outcome_contract_selftest ok=$ok evidence=$selfTestPath" + $result | ConvertTo-Json -Depth 12 + if (-not $ok) { exit 1 } + exit 0 +} + function Get-AgentProblemStats { $stateDir = Join-Path $Config.agentRoot "state" $statsPath = Join-Path $stateDir "problem-counts.json" if (-not (Test-Path $statsPath)) { return [pscustomobject]@{ path = $statsPath + schemaVersion = "agent99_problem_counts_v2" + outcomeContractActivatedAt = $null totalEvents = 0 totalResolved = 0 + totalVerifiedResolved = 0 totalFailed = 0 + totalDegraded = 0 + totalBlocked = 0 + totalVerifying = 0 + totalCandidateReady = 0 problems = @() } } @@ -2162,18 +2563,32 @@ function Get-AgentProblemStats { $data = Get-Content $statsPath -Raw | ConvertFrom-Json return [pscustomobject]@{ path = $statsPath + schemaVersion = if ($data.schemaVersion) { [string]$data.schemaVersion } else { "agent99_problem_counts_legacy_v1" } + outcomeContractActivatedAt = if ($data.outcomeContractActivatedAt) { [string]$data.outcomeContractActivatedAt } else { $null } totalEvents = if ($data.totalEvents) { [int]$data.totalEvents } else { 0 } totalResolved = if ($data.totalResolved) { [int]$data.totalResolved } else { 0 } + totalVerifiedResolved = if ($data.totalVerifiedResolved) { [int]$data.totalVerifiedResolved } else { 0 } totalFailed = if ($data.totalFailed) { [int]$data.totalFailed } else { 0 } + totalDegraded = if ($data.totalDegraded) { [int]$data.totalDegraded } else { 0 } + totalBlocked = if ($data.totalBlocked) { [int]$data.totalBlocked } else { 0 } + totalVerifying = if ($data.totalVerifying) { [int]$data.totalVerifying } else { 0 } + totalCandidateReady = if ($data.totalCandidateReady) { [int]$data.totalCandidateReady } else { 0 } problems = @($data.problems) } } catch { Write-AgentLog "problem_stats_parse_failed path=$statsPath error=$($_.Exception.Message)" return [pscustomobject]@{ path = $statsPath + schemaVersion = "agent99_problem_counts_v2" + outcomeContractActivatedAt = $null totalEvents = 0 totalResolved = 0 + totalVerifiedResolved = 0 totalFailed = 0 + totalDegraded = 0 + totalBlocked = 0 + totalVerifying = 0 + totalCandidateReady = 0 problems = @() parseError = $_.Exception.Message } @@ -2200,7 +2615,13 @@ function Update-AgentProblemKnowledge { $severity = if ($Result.alertSeverity) { [string]$Result.alertSeverity } elseif ($Result.ok) { "info" } else { "critical" } $key = Convert-AgentSafeKey (($source, $service, $hostName, $kind | Where-Object { $_ }) -join "|") $now = Get-Date -Format o - $statusText = if ($Result.ok) { "resolved" } else { "failed" } + $outcome = if ($Result.PSObject.Properties["outcome"]) { $Result.outcome } else { $null } + $statusText = if ($outcome -and $outcome.PSObject.Properties["state"]) { [string]$outcome.state } elseif ($Result.ok) { "resolved" } else { "failed" } + $isResolved = [bool]($statusText -eq "resolved") + $isFailed = [bool]($statusText -eq "failed") + if (-not $Result.alertSeverity) { + $severity = if ($isResolved) { "info" } elseif ($isFailed) { "critical" } else { "warning" } + } $event = [pscustomobject]@{ timestamp = $now @@ -2216,8 +2637,13 @@ function Update-AgentProblemKnowledge { host = $hostName mode = $Result.mode instruction = $Result.instruction - ok = [bool]$Result.ok + ok = $isResolved + transportOk = if ($Result.PSObject.Properties["transportOk"]) { [bool]$Result.transportOk } else { [bool]($Result.exitCode -eq 0) } exitCode = $Result.exitCode + outcomeState = $statusText + verifierName = if ($outcome) { $outcome.verifierName } else { $null } + verifierPassed = if ($outcome) { [bool]$outcome.verifierPassed } else { $false } + sourceEventResolved = if ($outcome) { [bool]$outcome.sourceEventResolved } else { $false } evidence = $Result.evidence } @@ -2237,7 +2663,12 @@ function Update-AgentProblemKnowledge { severity = $severity occurrences = 0 resolved = 0 + verifiedResolved = 0 failed = 0 + degraded = 0 + blocked = 0 + verifying = 0 + candidateReady = 0 lastStatus = $null firstSeenAt = $now lastSeenAt = $null @@ -2249,12 +2680,27 @@ function Update-AgentProblemKnowledge { $problems += $existing } + foreach ($counterName in @("verifiedResolved", "degraded", "blocked", "verifying", "candidateReady")) { + if (-not $existing.PSObject.Properties[$counterName]) { + $existing | Add-Member -MemberType NoteProperty -Name $counterName -Value 0 + } + } + $existing.occurrences = [int]$existing.occurrences + 1 - if ($Result.ok) { + if ($isResolved) { $existing.resolved = [int]$existing.resolved + 1 + $existing.verifiedResolved = [int]$existing.verifiedResolved + 1 $existing.lastResolvedAt = $now - } else { + } elseif ($isFailed) { $existing.failed = [int]$existing.failed + 1 + } elseif ($statusText -eq "degraded") { + $existing.degraded = [int]$existing.degraded + 1 + } elseif ($statusText -eq "blocked") { + $existing.blocked = [int]$existing.blocked + 1 + } elseif ($statusText -eq "verifying") { + $existing.verifying = [int]$existing.verifying + 1 + } elseif ($statusText -eq "candidate_ready") { + $existing.candidateReady = [int]$existing.candidateReady + 1 } $existing.lastStatus = $statusText $existing.lastSeenAt = $now @@ -2264,20 +2710,34 @@ function Update-AgentProblemKnowledge { $existing.severity = $severity $newTotalEvents = [int]$stats.totalEvents + 1 - $newTotalResolved = [int]$stats.totalResolved + $(if ($Result.ok) { 1 } else { 0 }) - $newTotalFailed = [int]$stats.totalFailed + $(if ($Result.ok) { 0 } else { 1 }) + $newTotalResolved = [int]$stats.totalResolved + $(if ($isResolved) { 1 } else { 0 }) + $newTotalVerifiedResolved = [int]$stats.totalVerifiedResolved + $(if ($isResolved) { 1 } else { 0 }) + $newTotalFailed = [int]$stats.totalFailed + $(if ($isFailed) { 1 } else { 0 }) + $newTotalDegraded = [int]$stats.totalDegraded + $(if ($statusText -eq "degraded") { 1 } else { 0 }) + $newTotalBlocked = [int]$stats.totalBlocked + $(if ($statusText -eq "blocked") { 1 } else { 0 }) + $newTotalVerifying = [int]$stats.totalVerifying + $(if ($statusText -eq "verifying") { 1 } else { 0 }) + $newTotalCandidateReady = [int]$stats.totalCandidateReady + $(if ($statusText -eq "candidate_ready") { 1 } else { 0 }) + $outcomeContractActivatedAt = if ($stats.outcomeContractActivatedAt) { $stats.outcomeContractActivatedAt } else { $now } $updated = [pscustomobject]@{ + schemaVersion = "agent99_problem_counts_v2" generatedAt = $now + outcomeContractActivatedAt = $outcomeContractActivatedAt + legacyResolvedCountMayIncludeTransportOnly = [bool]($stats.schemaVersion -ne "agent99_problem_counts_v2" -or ([int]$stats.totalResolved -gt [int]$stats.totalVerifiedResolved)) totalEvents = $newTotalEvents totalResolved = $newTotalResolved + totalVerifiedResolved = $newTotalVerifiedResolved totalFailed = $newTotalFailed + totalDegraded = $newTotalDegraded + totalBlocked = $newTotalBlocked + totalVerifying = $newTotalVerifying + totalCandidateReady = $newTotalCandidateReady problems = @($problems | Sort-Object @{ Expression = { [int]$_.occurrences }; Descending = $true }, @{ Expression = { $_.lastSeenAt }; Descending = $true }) } $updated | ConvertTo-Json -Depth 10 | Set-Content -Path $statsPath -Encoding UTF8 $rows = @($updated.problems | Select-Object -First 50 | ForEach-Object { - "- key=$($_.key); occurrences=$($_.occurrences); resolved=$($_.resolved); failed=$($_.failed); lastStatus=$($_.lastStatus); lastMode=$($_.lastMode); evidence=$($_.lastEvidence)" + "- key=$($_.key); occurrences=$($_.occurrences); verifiedResolved=$($_.verifiedResolved); failed=$($_.failed); degraded=$($_.degraded); blocked=$($_.blocked); verifying=$($_.verifying); candidateReady=$($_.candidateReady); lastStatus=$($_.lastStatus); lastMode=$($_.lastMode); evidence=$($_.lastEvidence)" }) $km = @( "# Agent99 Incident KM", @@ -2287,8 +2747,13 @@ function Update-AgentProblemKnowledge { "## Totals", "", "- totalEvents: $($updated.totalEvents)", - "- totalResolved: $($updated.totalResolved)", + "- totalResolvedLegacyAndVerified: $($updated.totalResolved)", + "- totalVerifiedResolved: $($updated.totalVerifiedResolved)", "- totalFailed: $($updated.totalFailed)", + "- totalDegraded: $($updated.totalDegraded)", + "- totalBlocked: $($updated.totalBlocked)", + "- totalVerifying: $($updated.totalVerifying)", + "- totalCandidateReady: $($updated.totalCandidateReady)", "", "## Recurring Problems", "", @@ -2300,7 +2765,12 @@ function Update-AgentProblemKnowledge { key = $key occurrences = $existing.occurrences resolved = $existing.resolved + verifiedResolved = $existing.verifiedResolved failed = $existing.failed + degraded = $existing.degraded + blocked = $existing.blocked + verifying = $existing.verifying + candidateReady = $existing.candidateReady lastStatus = $existing.lastStatus statsPath = $statsPath eventsPath = $eventsPath @@ -2388,7 +2858,12 @@ function Get-AgentDashboardSummary { path = $problemStats.path totalEvents = $problemStats.totalEvents totalResolved = $problemStats.totalResolved + totalVerifiedResolved = $problemStats.totalVerifiedResolved totalFailed = $problemStats.totalFailed + totalDegraded = $problemStats.totalDegraded + totalBlocked = $problemStats.totalBlocked + totalVerifying = $problemStats.totalVerifying + totalCandidateReady = $problemStats.totalCandidateReady top = @($problemStats.problems | Select-Object -First 10) } tasks = $tasks @@ -2434,7 +2909,7 @@ function Write-AgentDashboard { "$(Convert-AgentHtml $ev.name)$(Convert-AgentHtml $ev.file)$(Convert-AgentHtml $ev.ageMinutes)$(Convert-AgentHtml $ev.parseError)" } $problemRows = foreach ($problem in @($Summary.problemCounts.top)) { - "$(Convert-AgentHtml $problem.key)$(Convert-AgentHtml $problem.occurrences)$(Convert-AgentHtml $problem.resolved)$(Convert-AgentHtml $problem.failed)$(Convert-AgentHtml $problem.lastStatus)$(Convert-AgentHtml $problem.lastMode)" + "$(Convert-AgentHtml $problem.key)$(Convert-AgentHtml $problem.occurrences)$(Convert-AgentHtml $problem.verifiedResolved)$(Convert-AgentHtml $problem.failed)$(Convert-AgentHtml $problem.degraded)$(Convert-AgentHtml $problem.blocked)$(Convert-AgentHtml $problem.verifying)$(Convert-AgentHtml $problem.candidateReady)$(Convert-AgentHtml $problem.lastStatus)$(Convert-AgentHtml $problem.lastMode)" } $html = @" @@ -2469,12 +2944,14 @@ function Write-AgentDashboard {
Queue Pending
$(Convert-AgentHtml $Summary.queue.pending)
Alerts Pending
$(Convert-AgentHtml $Summary.alerts.pending)
Problem Events
$(Convert-AgentHtml $Summary.problemCounts.totalEvents)
-
Resolved
$(Convert-AgentHtml $Summary.problemCounts.totalResolved)
+
Verified Resolved
$(Convert-AgentHtml $Summary.problemCounts.totalVerifiedResolved)
Failed
$(Convert-AgentHtml $Summary.problemCounts.totalFailed)
+
Degraded
$(Convert-AgentHtml $Summary.problemCounts.totalDegraded)
+
Verifying
$(Convert-AgentHtml $Summary.problemCounts.totalVerifying)

Agent root: $(Convert-AgentHtml $Summary.agentRoot)
Queue: $(Convert-AgentHtml $Summary.queue.path)
Alerts: $(Convert-AgentHtml $Summary.alerts.incomingPath)
Problem counts: $(Convert-AgentHtml $Summary.problemCounts.path)
Evidence: $(Convert-AgentHtml $Summary.evidenceDir)

Problem Counts

- $($problemRows -join "`n")
ProblemOccurrencesResolvedFailedLast StatusLast Mode
+ $($problemRows -join "`n")
ProblemOccurrencesVerified ResolvedFailedDegradedBlockedVerifyingCandidate ReadyLast StatusLast Mode

Scheduled Tasks

$($taskRows -join "`n")
TaskStateLast ResultLast RunNext Run

Hosts

@@ -2566,6 +3043,9 @@ function Invoke-AgentQueuedCommands { $alertSeverity = if ($command.PSObject.Properties["alertSeverity"]) { [string]$command.alertSeverity } else { $null } $alertService = if ($command.PSObject.Properties["alertService"]) { [string]$command.alertService } else { $null } $alertHost = if ($command.PSObject.Properties["alertHost"]) { [string]$command.alertHost } else { $null } + $sourceEventResolutionRequired = if ($command.PSObject.Properties["sourceEventResolutionRequired"]) { Convert-AgentBool $command.sourceEventResolutionRequired } else { [bool]$alertId } + $sourceEventResolved = if ($command.PSObject.Properties["sourceEventResolved"]) { [object](Convert-AgentBool $command.sourceEventResolved) } else { $null } + $sourceEventEvidence = if ($command.PSObject.Properties["sourceEventEvidence"]) { [string]$command.sourceEventEvidence } else { "" } $replyChatId = if ($command.PSObject.Properties["replyChatId"]) { [string]$command.replyChatId } else { "" } $suppressTelegram = if ($command.PSObject.Properties["suppressTelegram"]) { Convert-AgentBool $command.suppressTelegram } else { $false } if ($modeName -notin $allowedModes) { @@ -2604,6 +3084,7 @@ function Invoke-AgentQueuedCommands { } else { -3 } + $outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence $result = [pscustomobject]@{ id = $id mode = $modeName @@ -2618,7 +3099,12 @@ function Invoke-AgentQueuedCommands { alertService = $alertService alertHost = $alertHost suppressTelegram = $suppressTelegram - ok = [bool]($exitCode -eq 0) + transportOk = [bool]($exitCode -eq 0) + ok = [bool]$outcome.resolved + outcomeState = $outcome.state + verifierPassed = [bool]$outcome.verifierPassed + sourceEventResolved = [bool]$outcome.sourceEventResolved + outcome = $outcome exitCode = $exitCode durationSeconds = [math]::Round(((Get-Date) - $startTime).TotalSeconds, 1) evidence = $latest.path @@ -2634,7 +3120,8 @@ function Invoke-AgentQueuedCommands { if ($replyChatId) { $recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force } - Record-AgentEvent "operator_command_result" $(if ($exitCode -eq 0) { "info" } else { "critical" }) "id=$id mode=$modeName exitCode=$exitCode evidence=$($latest.path)" $recordData -Alert + $outcomeSeverity = if ($outcome.state -eq "resolved") { "info" } elseif ($outcome.state -eq "failed") { "critical" } else { "warning" } + Record-AgentEvent "operator_command_result" $outcomeSeverity "id=$id mode=$modeName outcome=$($outcome.state) transportExitCode=$exitCode verifierPassed=$($outcome.verifierPassed) sourceEventResolved=$($outcome.sourceEventResolved) evidence=$($latest.path)" $recordData -Alert $results += $result } $results @@ -3177,6 +3664,10 @@ if ($SelfTestSensorGate) { Invoke-AgentSensorGateSelfTest } +if ($SelfTestOutcomeContract) { + Invoke-AgentOutcomeContractSelfTest +} + Write-AgentLog "agent99_start mode=$Mode controlledApply=$ControlledApply config=$ConfigPath" Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledApply host=$env:COMPUTERNAME" $null -Alert diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py new file mode 100644 index 000000000..afd50e60d --- /dev/null +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -0,0 +1,59 @@ +from __future__ import annotations + +from pathlib import Path + +_REPO_ROOT = Path(__file__).resolve().parents[3] +_CONTROL_PLANE = _REPO_ROOT / "agent99-control-plane.ps1" +_BOOTSTRAP = _REPO_ROOT / "agent99-bootstrap.ps1" + + +def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None: + source = _CONTROL_PLANE.read_text(encoding="utf-8") + + assert "function Get-AgentOutcomeContract" in source + assert 'schemaVersion = "agent99_outcome_contract_v1"' in source + assert "ok = [bool]$outcome.resolved" in source + assert "outcomeState = $outcome.state" in source + assert "verifierPassed = [bool]$outcome.verifierPassed" in source + assert "sourceEventResolutionRequired" in source + assert '"verifying"' in source + assert '"candidate_ready"' in source + assert "ok = [bool]($exitCode -eq 0)" not in source + + +def test_agent99_problem_counts_separate_verified_resolution() -> None: + source = _CONTROL_PLANE.read_text(encoding="utf-8") + + assert 'schemaVersion = "agent99_problem_counts_v2"' in source + assert "totalVerifiedResolved" in source + assert "legacyResolvedCountMayIncludeTransportOnly" in source + assert "verifiedResolved" in source + assert "totalDegraded" in source + assert "totalBlocked" in source + assert "totalVerifying" in source + assert "totalCandidateReady" in source + + +def test_agent99_runtime_manifest_is_generated_and_self_checked() -> None: + control_source = _CONTROL_PLANE.read_text(encoding="utf-8") + bootstrap_source = _BOOTSTRAP.read_text(encoding="utf-8") + + assert "function Test-AgentRuntimeManifest" in control_source + assert 'reason = "runtime_manifest_missing"' in control_source + assert "runtimeManifest = Test-AgentRuntimeManifest" in control_source + assert 'schemaVersion = "agent99_runtime_manifest_v1"' in bootstrap_source + assert 'Join-Path $StateDir "runtime-manifest.json"' in bootstrap_source + assert "Get-FileHash -Algorithm SHA256" in bootstrap_source + assert "if (-not $runtimeMatched)" in bootstrap_source + + +def test_agent99_outcome_contract_has_runtime_self_test() -> None: + source = _CONTROL_PLANE.read_text(encoding="utf-8") + + assert "[switch]$SelfTestOutcomeContract" in source + assert "function Invoke-AgentOutcomeContractSelfTest" in source + assert '$operatorResolved.state -eq "resolved"' in source + assert '$alertVerifying.state -eq "verifying"' in source + assert '$failedPostCondition.state -eq "degraded"' in source + assert '$transportFailed.state -eq "failed"' in source + assert '$securityCandidate.state -eq "candidate_ready"' in source diff --git a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json index 90694affb..85406b7a0 100644 --- a/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json +++ b/docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json @@ -1,7 +1,7 @@ { "schema_version": "agent99_enterprise_ai_automation_work_items_v1", - "generated_at": "2026-07-10T15:05:00+08:00", - "status": "in_progress_p0_agent_outcome_truth", + "generated_at": "2026-07-10T16:00:26+08:00", + "status": "in_progress_p0_agent_outcome_staging_verified_runtime_apply_pending", "scope_complete": false, "north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.", "canonical_sources": { @@ -20,7 +20,19 @@ "id": "AG99-P0-001", "title": "Agent outcome truth and source-runtime reconciliation", "reason": "Agent99 currently treats command exit success as incident resolution and the Windows runtime scripts do not consistently match the current Gitea source. All later automation depends on trustworthy outcomes.", - "next_action": "Implement mode-specific outcome evaluation, require verifier and source-event resolution for resolved, add regression tests, then deploy and compare source/runtime hashes on host 99." + "implementation_status": "source_implemented_staging_verified_runtime_apply_pending", + "staging_verifier": { + "host": "99", + "outcome_contract": "agent99_outcome_contract_v1", + "outcome_self_test_ok": true, + "runtime_manifest": "agent99_runtime_manifest_v1", + "source_revision": "p0-001-staging-latest", + "runtime_manifest_matched": true, + "runtime_manifest_file_count": 7, + "runtime_manifest_mismatch_count": 0, + "evidence": "C:\\Wooo\\Agent99\\staging\\p0-001-runtime\\evidence\\agent99-OutcomeContractSelfTest-20260710-160255.json" + }, + "next_action": "Commit and deploy the verified outcome contract to host 99, rerun the runtime manifest and outcome self-test, then replay one operator smoke and one alert-source event without counting transport-only success as resolved." }, "current_runtime_truth": { "observed_at": "2026-07-10T15:04:27+08:00", @@ -271,7 +283,7 @@ "verifier": "For every mode, require post-condition pass plus source event resolution; compare source and runtime manifest hashes.", "callback": "Write final outcome to AWOOOI incident and AwoooP Run/Verifier using the same trace id.", "acceptance": "No false-resolved regression cases; source/runtime drift count is zero; unresolved evidence stays degraded or blocked.", - "next_action": "Patch agent99-control-plane.ps1 outcome evaluation and add focused regression tests." + "next_action": "Deploy the staging-verified outcome contract to host 99, confirm source/runtime manifest hashes, then replay operator and alert-source outcomes." }, { "id": "AG99-P0-002",