@@ -5,6 +5,7 @@ param(
[ switch ] $SelfTestTelegramCard ,
[ switch ] $SelfTestTelegramDelivery ,
[ switch ] $SelfTestSensorGate ,
[ switch ] $SelfTestOutcomeContract ,
[ switch ] $SuppressAlerts ,
[ string ] $SuppressReason = " " ,
[ string ] $ConfigPath = " C:\Wooo\Agent99\config\agent99.config.json " ,
@@ -350,7 +351,16 @@ function Format-AgentTelegramText {
$instruction = if ( $Data -and $Data . PSObject . Properties [ " instruction " ] ) { $Data . instruction } else { $null }
$exitCode = if ( $Data -and $Data . PSObject . Properties [ " exitCode " ] ) { $Data . exitCode } else { $null }
$evidencePath = if ( $Data -and $Data . PSObject . Properties [ " evidence " ] ) { $Data . evidence } else { $null }
$resultLabel = if ( $exitCode -eq 0 -or [ string ] ( Get-AgentObjectValue $Data " ok " " " ) -match " ^(?i:true|1) $ " ) { " 已完成 / resolved " } else { " 仍需處理 / failed " }
$outcome = if ( $Data -and $Data . PSObject . Properties [ " outcome " ] ) { $Data . outcome } else { $null }
$outcomeState = if ( $outcome -and $outcome . PSObject . Properties [ " state " ] ) { [ string ] $outcome . state } else { " failed " }
$resultLabel = switch ( $outcomeState ) {
" resolved " { " 已驗證解決 / verified resolved " }
" candidate_ready " { " 修復候選已建立 / candidate ready " }
" verifying " { " 等待來源事件驗證 / verifying " }
" degraded " { " 仍有異常 / degraded " }
" blocked " { " 受阻擋 / blocked " }
default { " 執行失敗 / failed " }
}
$actionText = switch ( $modeName ) {
" ProviderFreshness " { " 建立 provider freshness candidate， ； " }
" BackupCheck " { " 讀回備份 freshness、status/log、snapshot metadata、cron contract； " }
@@ -366,12 +376,15 @@ function Format-AgentTelegramText {
$lines + = " 事件 Event: Agent99 自動化結果 / automation result "
$lines + = " 模式 Mode: $modeName "
$lines + = " 來源 Source: $source "
$lines + = " 結果 Result: $resultLabel , e xitCode= $exitCode "
$lines + = " 結果 Result: $resultLabel , transportE xitCode= $exitCode "
if ( $outcome ) {
$lines + = " 驗證 Verifier: name= $( $outcome . verifierName ) , passed= $( $outcome . verifierPassed ) , sourceEventResolved= $( $outcome . sourceEventResolved ) "
}
$lines + = " Agent99 動作 Action: $actionText "
if ( $Data -and $Data . PSObject . Properties [ " problem " ] ) {
$problem = $Data . problem
if ( $problem -and $problem . PSObject . Properties [ " occurrences " ] ) {
$lines + = " 問題計數 Problem count: occurrences= $( $problem . occurrences ) , r esolved= $( $problem . r esolved) , failed= $( $problem . failed ) "
$lines + = " 問題計數 Problem count: occurrences= $( $problem . occurrences ) , verifiedR esolved= $( $problem . verifiedR esolved) , failed= $( $problem . failed ) , degraded= $( $problem . degraded ) , blocked= $( $problem . blocked ) , verifying= $( $problem . verifying ) , candidateReady= $( $problem . candidateReady )
if ( $problem . PSObject . Properties [ " kmPath " ] ) {
$lines + = " KM: $( $problem . kmPath ) "
}
@@ -604,11 +617,22 @@ function Invoke-AgentTelegramCardSelfTest {
ok = $true
exitCode = 0
evidence = $jsonPath
outcome = [ pscustomobject ] @ {
state = " resolved "
verifierName = " selftest-verifier "
verifierPassed = $true
sourceEventResolved = $true
}
problem = [ pscustomobject ] @ {
key = " selftest-provider-freshness "
occurrences = 2
resolved = 2
verifiedResolved = 2
failed = 0
degraded = 0
blocked = 0
verifying = 0
candidateReady = 0
kmPath = ( Join-Path $Config . agentRoot " playbooks\agent99-incident-km.md " )
}
}
@@ -616,7 +640,7 @@ function Invoke-AgentTelegramCardSelfTest {
$operatorImagePath = New-AgentTelegramCardImage " info " " operator_command_result " $operatorText $operatorSample
$operatorDecisionOk = Test-AgentSreOperatorResultAlertEnabled $operatorSample
$operatorTextOk = [ bool ] (
$operatorText -match " 已完成 / resolved " -and
$operatorText -match " 已驗證解決 / verified resolved " -and
$operatorText -match " Agent99 動作 Action " -and
$operatorText -match " 問題計數 Problem count " -and
$operatorText -match " KM: "
@@ -1721,11 +1745,82 @@ function Test-AgentRecentTelegramDelivery {
}
}
function Test-AgentRuntimeManifest {
$manifestPath = Join-Path $Config . agentRoot " state\runtime-manifest.json "
if ( -not ( Test-Path $manifestPath ) ) {
return [ pscustomobject ] @ {
ok = $false
severity = " critical "
reason = " runtime_manifest_missing "
manifestPath = $manifestPath
sourceRevision = $null
driftCount = 0
missingCount = 1
files = @ ( )
}
}
try {
$manifest = Get-Content $manifestPath -Raw | ConvertFrom-Json
} catch {
return [ pscustomobject ] @ {
ok = $false
severity = " critical "
reason = " runtime_manifest_parse_failed "
manifestPath = $manifestPath
sourceRevision = $null
driftCount = 0
missingCount = 0
files = @ ( )
error = $_ . Exception . Message
}
}
$rows = @ ( )
foreach ( $entry in @ ( $manifest . files ) ) {
$name = [ string ] $entry . name
$runtimePath = Join-Path $PSScriptRoot $name
$exists = Test-Path $runtimePath
$actualSha256 = $null
if ( $exists ) {
try {
$actualSha256 = ( Get-FileHash -Algorithm SHA256 -LiteralPath $runtimePath ) . Hash . ToLowerInvariant ( )
} catch {
$actualSha256 = $null
}
}
$expectedSha256 = if ( $entry . runtimeSha256 ) { ( [ string ] $entry . runtimeSha256 ) . ToLowerInvariant ( ) } elseif ( $entry . sourceSha256 ) { ( [ string ] $entry . sourceSha256 ) . ToLowerInvariant ( ) } else { " " }
$matched = [ bool ] ( $exists -and $actualSha256 -and $expectedSha256 -and $actualSha256 -eq $expectedSha256 )
$rows + = [ pscustomobject ] @ {
name = $name
exists = $exists
matched = $matched
expectedSha256 = $expectedSha256
actualSha256 = $actualSha256
}
}
$missingCount = @ ( $rows | Where-Object { -not $_ . exists } ) . Count
$driftCount = @ ( $rows | Where-Object { $_ . exists -and -not $_ . matched } ) . Count
$ready = [ bool ] ( @ ( $rows ) . Count -gt 0 -and $missingCount -eq 0 -and $driftCount -eq 0 -and $manifest . runtimeMatched -eq $true )
[ pscustomobject ] @ {
ok = $ready
severity = if ( $ready ) { " ok " } else { " critical " }
reason = if ( $ready ) { " runtime_manifest_matched " } else { " runtime_source_drift_or_missing " }
manifestPath = $manifestPath
sourceRevision = $manifest . sourceRevision
generatedAt = $manifest . generatedAt
driftCount = $driftCount
missingCount = $missingCount
files = $rows
}
}
function Test-AgentSelfHealth {
$selfConfig = Get-AgentSelfHealthConfig
$tasks = Test-AgentScheduledTaskHealth $selfConfig . scheduledTasks
$evidence = Test-AgentEvidenceFreshness $selfConfig . evidenceFreshness
$latestPerf = Test-AgentLatestPerformanceEvidence $selfConfig . requiredHosts
$runtimeManifest = Test-AgentRuntimeManifest
$telegram = if ( $selfConfig . requireRecentTelegramDelivery ) {
Test-AgentRecentTelegramDelivery $selfConfig . evidenceFreshness
} else {
@@ -1738,11 +1833,12 @@ function Test-AgentSelfHealth {
$critical = $taskFailures + $evidenceCritical
if ( $latestPerf . severity -eq " critical " ) { $critical + = 1 }
if ( $telegram . severity -eq " critical " ) { $critical + = 1 }
if ( $runtimeManifest . severity -eq " critical " ) { $critical + = 1 }
$warning = $evidenceWarning
if ( $latestPerf . severity -eq " warning " ) { $warning + = 1 }
$severity = if ( $critical -gt 0 ) { " critical " } elseif ( $warning -gt 0 ) { " warning " } else { " ok " }
$message = " selfHealth severity= $severity taskFailures= $taskFailures evidenceCritical= $evidenceCritical evidenceWarning= $evidenceWarning perf= $( $latestPerf . severity ) telegram= $( $telegram . severity ) "
$message = " selfHealth severity= $severity taskFailures= $taskFailures evidenceCritical= $evidenceCritical evidenceWarning= $evidenceWarning perf= $( $latestPerf . severity ) telegram= $( $telegram . severity ) runtimeManifest= $( $runtimeManifest . severity )
Record-AgentEvent " agent_selfcheck " $ ( if ( $severity -eq " ok " ) { " info " } else { $severity } ) $message $null -Alert
[ pscustomobject ] @ {
@@ -1752,12 +1848,16 @@ function Test-AgentSelfHealth {
evidenceFreshness = $evidence
latestPerformance = $latestPerf
telegramDelivery = $telegram
runtimeManifest = $runtimeManifest
summary = [ pscustomobject ] @ {
taskFailures = $taskFailures
evidenceCritical = $evidenceCritical
evidenceWarning = $evidenceWarning
performanceSeverity = $latestPerf . severity
telegramSeverity = $telegram . severity
runtimeManifestSeverity = $runtimeManifest . severity
runtimeManifestSourceRevision = $runtimeManifest . sourceRevision
runtimeManifestDriftCount = $runtimeManifest . driftCount
}
}
}
@@ -2146,15 +2246,316 @@ function Convert-AgentSafeKey {
return $safe
}
function New-AgentOutcomeCheck {
param (
[ string ] $Name ,
[ bool ] $Passed ,
[ bool ] $Required = $true ,
[ string ] $Detail = " "
)
[ pscustomobject ] @ {
name = $Name
passed = $Passed
required = $Required
detail = $Detail
}
}
function Get-AgentOutcomeContract {
param (
[ string ] $ModeName ,
[ int ] $TransportExitCode ,
[ object ] $LatestEvidence ,
[ datetime ] $StartedAt ,
[ bool ] $SourceEventResolutionRequired ,
[ object ] $SourceEventResolved = $null ,
[ string ] $SourceEventEvidence = " "
)
$checks = @ ( )
$transportOk = [ bool ] ( $TransportExitCode -eq 0 )
$evidenceExists = [ bool ] ( $LatestEvidence -and $LatestEvidence . exists )
$evidenceParsed = [ bool ] ( $evidenceExists -and -not $LatestEvidence . parseError -and $LatestEvidence . data )
$evidenceFresh = $false
if ( $evidenceExists -and $LatestEvidence . lastWriteTime ) {
try {
$evidenceTime = [ datetime ] :: Parse ( [ string ] $LatestEvidence . lastWriteTime )
$evidenceFresh = [ bool ] ( $evidenceTime -ge $StartedAt . AddSeconds ( -5 ) )
} catch {
$evidenceFresh = $false
}
}
$data = if ( $evidenceParsed ) { $LatestEvidence . data } else { $null }
$evidenceMode = if ( $data -and $data . PSObject . Properties [ " mode " ] ) { [ string ] $data . mode } else { " " }
$modeMatches = [ bool ] ( $evidenceMode -eq $ModeName )
$checks + = New-AgentOutcomeCheck " transport_completed " $transportOk $true " exitCode= $TransportExitCode "
$checks + = New-AgentOutcomeCheck " evidence_exists " $evidenceExists $true ( [ string ] $LatestEvidence . path )
$checks + = New-AgentOutcomeCheck " evidence_parsed " $evidenceParsed $true ( [ string ] $LatestEvidence . parseError )
$checks + = New-AgentOutcomeCheck " evidence_fresh " $evidenceFresh $true ( [ string ] $LatestEvidence . lastWriteTime )
$checks + = New-AgentOutcomeCheck " evidence_mode_matches " $modeMatches $true " expected= $ModeName actual= $evidenceMode "
$verifierName = " agent99- $( $ModeName . ToLowerInvariant ( ) ) -post-condition-v1 "
$modeVerifierPassed = $false
$successState = " resolved "
$failureState = " degraded "
if ( $data ) {
switch ( $ModeName ) {
" Status " {
$expectedHosts = @ ( $Config . hosts ) . Count
$hostRows = @ ( $data . hosts )
$hostsOk = [ bool ] ( $expectedHosts -gt 0 -and $hostRows . Count -ge $expectedHosts -and @ ( $hostRows | Where-Object { -not $_ . ping } ) . Count -eq 0 )
$harborOk = [ bool ] ( $data . harbor -and $data . harbor . redisUp -and $data . harbor . coreHealthy -and $data . harbor . registryHealthy )
$awoooOk = [ bool ] ( $data . awoooi -and $data . awoooi . deployReady -and -not $data . awoooi . imagePullFailure -and -not $data . awoooi . crashFailure )
$publicRows = @ ( $data . public )
$publicOk = [ bool ] ( $publicRows . Count -gt 0 -and @ ( $publicRows | Where-Object { -not $_ . ok } ) . Count -eq 0 )
$aiOk = [ bool ] ( @ ( $data . aiServices | Where-Object { -not $_ . ok } ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " required_hosts_reachable " $hostsOk $true " expected= $expectedHosts actual= $( $hostRows . Count ) "
$checks + = New-AgentOutcomeCheck " harbor_healthy " $harborOk
$checks + = New-AgentOutcomeCheck " awoooi_deployments_ready " $awoooOk
$checks + = New-AgentOutcomeCheck " public_routes_healthy " $publicOk
$checks + = New-AgentOutcomeCheck " ai_services_healthy " $aiOk $false
$modeVerifierPassed = [ bool ] ( $hostsOk -and $harborOk -and $awoooOk -and $publicOk )
}
" Recover " {
$expectedVms = @ ( $Config . vms ) . Count
$vmRows = @ ( $data . vmResults )
$vmsOk = [ bool ] ( $expectedVms -gt 0 -and $vmRows . Count -ge $expectedVms -and @ ( $vmRows | Where-Object { -not $_ . ok } ) . Count -eq 0 )
$expectedHosts = @ ( $Config . hosts ) . Count
$hostRows = @ ( $data . hosts )
$hostsOk = [ bool ] ( $expectedHosts -gt 0 -and $hostRows . Count -ge $expectedHosts -and @ ( $hostRows | Where-Object { -not $_ . ping } ) . Count -eq 0 )
$harborOk = [ bool ] ( $data . harbor -and $data . harbor . redisUp -and $data . harbor . coreHealthy -and $data . harbor . registryHealthy )
$awoooOk = [ bool ] ( $data . awoooi -and $data . awoooi . deployReady -and -not $data . awoooi . imagePullFailure -and -not $data . awoooi . crashFailure )
$publicRows = @ ( $data . public )
$publicOk = [ bool ] ( $publicRows . Count -gt 0 -and @ ( $publicRows | Where-Object { -not $_ . ok } ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " configured_vms_ready " $vmsOk $true " expected= $expectedVms actual= $( $vmRows . Count ) "
$checks + = New-AgentOutcomeCheck " required_hosts_reachable " $hostsOk $true " expected= $expectedHosts actual= $( $hostRows . Count ) "
$checks + = New-AgentOutcomeCheck " harbor_healthy " $harborOk
$checks + = New-AgentOutcomeCheck " awoooi_deployments_ready " $awoooOk
$checks + = New-AgentOutcomeCheck " public_routes_healthy " $publicOk
$modeVerifierPassed = [ bool ] ( $vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk )
}
" StartVMs " {
$expectedVms = @ ( $Config . vms ) . Count
$vmRows = @ ( $data . vmResults )
$commandOk = [ bool ] ( $expectedVms -gt 0 -and $vmRows . Count -ge $expectedVms -and @ ( $vmRows | Where-Object { -not $_ . ok } ) . Count -eq 0 )
$unreachable = @ ( )
foreach ( $vm in @ ( $Config . vms ) ) {
if ( -not $vm . host ) { $unreachable + = [ string ] $vm . name ; continue }
$reachable = Test-Connection -ComputerName ( [ string ] $vm . host ) -Count 1 -Quiet -ErrorAction SilentlyContinue
if ( -not $reachable ) { $unreachable + = [ string ] $vm . name }
}
$guestsReachable = [ bool ] ( $expectedVms -gt 0 -and @ ( $unreachable ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " vm_start_commands_ok " $commandOk $true " expected= $expectedVms actual= $( $vmRows . Count ) "
$checks + = New-AgentOutcomeCheck " vm_guests_reachable " $guestsReachable $true " unreachable= $( $unreachable -join ',' ) "
$modeVerifierPassed = [ bool ] ( $commandOk -and $guestsReachable )
}
" PublicSmoke " {
$rows = @ ( $data . public )
$routesOk = [ bool ] ( $rows . Count -gt 0 -and @ ( $rows | Where-Object { -not $_ . ok -or -not $_ . non502 } ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " public_routes_non502 " $routesOk $true " count= $( $rows . Count ) "
$modeVerifierPassed = $routesOk
}
" HarborRepair " {
$harborOk = [ bool ] ( $data . harbor -and $data . harbor . redisUp -and $data . harbor . coreHealthy -and $data . harbor . registryHealthy )
$checks + = New-AgentOutcomeCheck " harbor_healthy " $harborOk
$modeVerifierPassed = $harborOk
}
" AwoooRepair " {
$awoooOk = [ bool ] ( $data . awoooi -and $data . awoooi . deployReady -and -not $data . awoooi . imagePullFailure -and -not $data . awoooi . crashFailure )
$checks + = New-AgentOutcomeCheck " awoooi_deployments_ready " $awoooOk
$modeVerifierPassed = $awoooOk
}
" Perf " {
$rows = @ ( $data . performance )
$readbackOk = [ bool ] ( $rows . Count -gt 0 -and @ ( $rows | Where-Object { -not $_ . ok -or ( @ ( $_ . reasons ) -contains " perf_readback_failed " ) } ) . Count -eq 0 )
$pressureClear = [ bool ] ( $readbackOk -and @ ( $rows | Where-Object { $_ . severity -in @ ( " warning " , " critical " ) } ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " performance_readback_complete " $readbackOk
$checks + = New-AgentOutcomeCheck " performance_pressure_cleared " $pressureClear
$modeVerifierPassed = [ bool ] ( $readbackOk -and $pressureClear )
}
" LoadShed " {
$rows = @ ( $data . performance )
$readbackOk = [ bool ] ( $rows . Count -gt 0 -and @ ( $rows | Where-Object { -not $_ . ok -or ( @ ( $_ . reasons ) -contains " perf_readback_failed " ) } ) . Count -eq 0 )
$executed = @ ( $data . loadShedding | Where-Object { -not $_ . skipped } )
$postVerifyOk = [ bool ] ( $executed . Count -gt 0 -and @ ( $executed | Where-Object { -not $_ . postVerifyOk } ) . Count -eq 0 )
$alreadyClear = [ bool ] ( $readbackOk -and @ ( $rows | Where-Object { $_ . severity -in @ ( " warning " , " critical " ) } ) . Count -eq 0 )
$checks + = New-AgentOutcomeCheck " performance_readback_complete " $readbackOk
$checks + = New-AgentOutcomeCheck " load_shed_post_verify " ( [ bool ] ( $postVerifyOk -or $alreadyClear ) ) $true " executed= $( $executed . Count ) "
$modeVerifierPassed = [ bool ] ( $readbackOk -and ( $postVerifyOk -or $alreadyClear ) )
}
" SelfCheck " {
$selfOk = [ bool ] ( $data . selfHealth -and $data . selfHealth . severity -eq " ok " )
$checks + = New-AgentOutcomeCheck " agent_self_health_ok " $selfOk
$modeVerifierPassed = $selfOk
}
" BackupCheck " {
$backupOk = [ bool ] ( $data . backupHealth -and $data . backupHealth . severity -eq " ok " )
$checks + = New-AgentOutcomeCheck " backup_health_ok " $backupOk
$modeVerifierPassed = $backupOk
}
" ProviderFreshness " {
$provider = $data . providerFreshness
$directOk = [ bool ] ( $provider -and $provider . status -eq " direct_readback_ok " -and @ ( $provider . missingFields ) . Count -eq 0 -and $provider . lastSeen -and $provider . verifierReadback -and $provider . verifierReadback . ok -eq $true -and [ int ] $provider . verifierReadback . sourceEventTotal -gt 0 )
$checks + = New-AgentOutcomeCheck " provider_freshness_direct_readback " $directOk $true ( [ string ] $provider . status )
$modeVerifierPassed = $directOk
}
" SecurityTriage " {
$candidateReady = [ bool ] ( $data . securityTriage -and $data . securityTriage . status -eq " security_triage_candidate_created " )
$checks + = New-AgentOutcomeCheck " security_candidate_created " $candidateReady
$modeVerifierPassed = $candidateReady
$successState = " candidate_ready "
}
" ControlTick " {
$controlOk = [ bool ] ( $data . controlTick -and $data . controlTick . ok )
$checks + = New-AgentOutcomeCheck " control_tick_completed " $controlOk
$modeVerifierPassed = $controlOk
}
default {
$checks + = New-AgentOutcomeCheck " mode_verifier_defined " $false
$failureState = " blocked "
}
}
}
$basePassed = [ bool ] ( $transportOk -and $evidenceExists -and $evidenceParsed -and $evidenceFresh -and $modeMatches )
$verifierPassed = [ bool ] ( $basePassed -and $modeVerifierPassed )
$sourceEventResolvedBool = if ( -not $SourceEventResolutionRequired ) { $true } else { [ bool ] ( $SourceEventResolved -eq $true ) }
$state = if ( -not $transportOk ) {
" failed "
} elseif ( -not $basePassed ) {
" blocked "
} elseif ( -not $modeVerifierPassed ) {
$failureState
} elseif ( $successState -eq " candidate_ready " ) {
" candidate_ready "
} elseif ( $SourceEventResolutionRequired -and -not $sourceEventResolvedBool ) {
" verifying "
} else {
" resolved "
}
$failedChecks = @ ( $checks | Where-Object { $_ . required -and -not $_ . passed } | ForEach-Object { $_ . name } )
[ pscustomobject ] @ {
schemaVersion = " agent99_outcome_contract_v1 "
state = $state
resolved = [ bool ] ( $state -eq " resolved " )
transportOk = $transportOk
transportExitCode = $TransportExitCode
verifierName = $verifierName
verifierPassed = $verifierPassed
evidenceFresh = $evidenceFresh
evidenceModeMatched = $modeMatches
sourceEventResolutionRequired = $SourceEventResolutionRequired
sourceEventResolved = $sourceEventResolvedBool
sourceEventEvidence = if ( $SourceEventEvidence ) { $SourceEventEvidence } else { $null }
failedChecks = $failedChecks
checks = $checks
verifiedAt = ( Get-Date -Format o )
}
}
function Invoke-AgentOutcomeContractSelfTest {
$startedAt = ( Get-Date ) . AddSeconds ( -1 )
$publicOkEvidence = [ pscustomobject ] @ {
exists = $true
path = " selftest-public-ok.json "
lastWriteTime = ( Get-Date -Format o )
parseError = $null
data = [ pscustomobject ] @ {
mode = " PublicSmoke "
public = @ ( [ pscustomobject ] @ { url = " https://example.invalid " ; status = 200 ; non502 = $true ; ok = $true } )
}
}
$publicFailedEvidence = [ pscustomobject ] @ {
exists = $true
path = " selftest-public-failed.json "
lastWriteTime = ( Get-Date -Format o )
parseError = $null
data = [ pscustomobject ] @ {
mode = " PublicSmoke "
public = @ ( [ pscustomobject ] @ { url = " https://example.invalid " ; status = 502 ; non502 = $false ; ok = $false } )
}
}
$providerEvidence = [ pscustomobject ] @ {
exists = $true
path = " selftest-provider.json "
lastWriteTime = ( Get-Date -Format o )
parseError = $null
data = [ pscustomobject ] @ {
mode = " ProviderFreshness "
providerFreshness = [ pscustomobject ] @ {
status = " direct_readback_ok_no_source_events "
missingFields = @ ( " lastSeen " )
lastSeen = $null
verifierReadback = [ pscustomobject ] @ { ok = $true ; sourceEventTotal = 0 }
}
}
}
$securityEvidence = [ pscustomobject ] @ {
exists = $true
path = " selftest-security.json "
lastWriteTime = ( Get-Date -Format o )
parseError = $null
data = [ pscustomobject ] @ {
mode = " SecurityTriage "
securityTriage = [ pscustomobject ] @ { status = " security_triage_candidate_created " ; hitCount = 1 }
}
}
$operatorResolved = Get-AgentOutcomeContract " PublicSmoke " 0 $publicOkEvidence $startedAt $false
$alertVerifying = Get-AgentOutcomeContract " PublicSmoke " 0 $publicOkEvidence $startedAt $true $false " selftest-source-event "
$failedPostCondition = Get-AgentOutcomeContract " PublicSmoke " 0 $publicFailedEvidence $startedAt $false
$transportFailed = Get-AgentOutcomeContract " PublicSmoke " 7 $publicOkEvidence $startedAt $false
$providerDegraded = Get-AgentOutcomeContract " ProviderFreshness " 0 $providerEvidence $startedAt $true $false
$securityCandidate = Get-AgentOutcomeContract " SecurityTriage " 0 $securityEvidence $startedAt $true $false
$ok = [ bool ] (
$operatorResolved . state -eq " resolved " -and $operatorResolved . verifierPassed -and
$alertVerifying . state -eq " verifying " -and -not $alertVerifying . resolved -and
$failedPostCondition . state -eq " degraded " -and -not $failedPostCondition . verifierPassed -and
$transportFailed . state -eq " failed " -and
$providerDegraded . state -eq " degraded " -and
$securityCandidate . state -eq " candidate_ready "
)
$result = [ pscustomobject ] @ {
timestamp = ( Get-Date -Format o )
ok = $ok
contract = " agent99_outcome_contract_v1 "
cases = [ pscustomobject ] @ {
operatorResolved = $operatorResolved
alertVerifying = $alertVerifying
failedPostCondition = $failedPostCondition
transportFailed = $transportFailed
providerDegraded = $providerDegraded
securityCandidate = $securityCandidate
}
}
$selfTestPath = Join-Path $EvidenceDir " agent99-OutcomeContractSelfTest- $stamp .json "
$result | ConvertTo-Json -Depth 12 | Set-Content -Path $selfTestPath -Encoding UTF8
$result | ConvertTo-Json -Depth 12 | Set-Content -Path $jsonPath -Encoding UTF8
Write-AgentLog " outcome_contract_selftest ok= $ok evidence= $selfTestPath "
$result | ConvertTo-Json -Depth 12
if ( -not $ok ) { exit 1 }
exit 0
}
function Get-AgentProblemStats {
$stateDir = Join-Path $Config . agentRoot " state "
$statsPath = Join-Path $stateDir " problem-counts.json "
if ( -not ( Test-Path $statsPath ) ) {
return [ pscustomobject ] @ {
path = $statsPath
schemaVersion = " agent99_problem_counts_v2 "
outcomeContractActivatedAt = $null
totalEvents = 0
totalResolved = 0
totalVerifiedResolved = 0
totalFailed = 0
totalDegraded = 0
totalBlocked = 0
totalVerifying = 0
totalCandidateReady = 0
problems = @ ( )
}
}
@@ -2162,18 +2563,32 @@ function Get-AgentProblemStats {
$data = Get-Content $statsPath -Raw | ConvertFrom-Json
return [ pscustomobject ] @ {
path = $statsPath
schemaVersion = if ( $data . schemaVersion ) { [ string ] $data . schemaVersion } else { " agent99_problem_counts_legacy_v1 " }
outcomeContractActivatedAt = if ( $data . outcomeContractActivatedAt ) { [ string ] $data . outcomeContractActivatedAt } else { $null }
totalEvents = if ( $data . totalEvents ) { [ int ] $data . totalEvents } else { 0 }
totalResolved = if ( $data . totalResolved ) { [ int ] $data . totalResolved } else { 0 }
totalVerifiedResolved = if ( $data . totalVerifiedResolved ) { [ int ] $data . totalVerifiedResolved } else { 0 }
totalFailed = if ( $data . totalFailed ) { [ int ] $data . totalFailed } else { 0 }
totalDegraded = if ( $data . totalDegraded ) { [ int ] $data . totalDegraded } else { 0 }
totalBlocked = if ( $data . totalBlocked ) { [ int ] $data . totalBlocked } else { 0 }
totalVerifying = if ( $data . totalVerifying ) { [ int ] $data . totalVerifying } else { 0 }
totalCandidateReady = if ( $data . totalCandidateReady ) { [ int ] $data . totalCandidateReady } else { 0 }
problems = @ ( $data . problems )
}
} catch {
Write-AgentLog " problem_stats_parse_failed path= $statsPath error= $( $_ . Exception . Message ) "
return [ pscustomobject ] @ {
path = $statsPath
schemaVersion = " agent99_problem_counts_v2 "
outcomeContractActivatedAt = $null
totalEvents = 0
totalResolved = 0
totalVerifiedResolved = 0
totalFailed = 0
totalDegraded = 0
totalBlocked = 0
totalVerifying = 0
totalCandidateReady = 0
problems = @ ( )
parseError = $_ . Exception . Message
}
@@ -2200,7 +2615,13 @@ function Update-AgentProblemKnowledge {
$severity = if ( $Result . alertSeverity ) { [ string ] $Result . alertSeverity } elseif ( $Result . ok ) { " info " } else { " critical " }
$key = Convert-AgentSafeKey ( ( $source , $service , $hostName , $kind | Where-Object { $_ } ) -join " | " )
$now = Get-Date -Format o
$statusText = if ( $Result . ok ) { " resolved " } else { " failed " }
$outcome = if ( $Result . PSObject . Properties [ " outcome " ] ) { $Result . outcome } else { $null }
$statusText = if ( $outcome -and $outcome . PSObject . Properties [ " state " ] ) { [ string ] $outcome . state } elseif ( $Result . ok ) { " resolved " } else { " failed " }
$isResolved = [ bool ] ( $statusText -eq " resolved " )
$isFailed = [ bool ] ( $statusText -eq " failed " )
if ( -not $Result . alertSeverity ) {
$severity = if ( $isResolved ) { " info " } elseif ( $isFailed ) { " critical " } else { " warning " }
}
$event = [ pscustomobject ] @ {
timestamp = $now
@@ -2216,8 +2637,13 @@ function Update-AgentProblemKnowledge {
host = $hostName
mode = $Result . mode
instruction = $Result . instruction
ok = [ bool ] $Result . ok
ok = $isResolved
transportOk = if ( $Result . PSObject . Properties [ " transportOk " ] ) { [ bool ] $Result . transportOk } else { [ bool ] ( $Result . exitCode -eq 0 ) }
exitCode = $Result . exitCode
outcomeState = $statusText
verifierName = if ( $outcome ) { $outcome . verifierName } else { $null }
verifierPassed = if ( $outcome ) { [ bool ] $outcome . verifierPassed } else { $false }
sourceEventResolved = if ( $outcome ) { [ bool ] $outcome . sourceEventResolved } else { $false }
evidence = $Result . evidence
}
@@ -2237,7 +2663,12 @@ function Update-AgentProblemKnowledge {
severity = $severity
occurrences = 0
resolved = 0
verifiedResolved = 0
failed = 0
degraded = 0
blocked = 0
verifying = 0
candidateReady = 0
lastStatus = $null
firstSeenAt = $now
lastSeenAt = $null
@@ -2249,12 +2680,27 @@ function Update-AgentProblemKnowledge {
$problems + = $existing
}
foreach ( $counterName in @ ( " verifiedResolved " , " degraded " , " blocked " , " verifying " , " candidateReady " ) ) {
if ( -not $existing . PSObject . Properties [ $counterName ] ) {
$existing | Add-Member -MemberType NoteProperty -Name $counterName -Value 0
}
}
$existing . occurrences = [ int ] $existing . occurrences + 1
if ( $Result . ok ) {
if ( $is Resolved ) {
$existing . resolved = [ int ] $existing . resolved + 1
$existing . verifiedResolved = [ int ] $existing . verifiedResolved + 1
$existing . lastResolvedAt = $now
} else {
} elseif ( $isFailed ) {
$existing . failed = [ int ] $existing . failed + 1
} elseif ( $statusText -eq " degraded " ) {
$existing . degraded = [ int ] $existing . degraded + 1
} elseif ( $statusText -eq " blocked " ) {
$existing . blocked = [ int ] $existing . blocked + 1
} elseif ( $statusText -eq " verifying " ) {
$existing . verifying = [ int ] $existing . verifying + 1
} elseif ( $statusText -eq " candidate_ready " ) {
$existing . candidateReady = [ int ] $existing . candidateReady + 1
}
$existing . lastStatus = $statusText
$existing . lastSeenAt = $now
@@ -2264,20 +2710,34 @@ function Update-AgentProblemKnowledge {
$existing . severity = $severity
$newTotalEvents = [ int ] $stats . totalEvents + 1
$newTotalResolved = [ int ] $stats . totalResolved + $ ( if ( $Result . ok ) { 1 } else { 0 } )
$newTotalFail ed = [ int ] $stats . totalFail ed + $ ( if ( $Result . ok ) { 0 } else { 1 } )
$newTotalResolved = [ int ] $stats . totalResolved + $ ( if ( $is Resolved ) { 1 } else { 0 } )
$newTotalVerifiedResolv ed = [ int ] $stats . totalVerifiedResolv ed + $ ( if ( $is Resolved ) { 1 } else { 0 } )
$newTotalFailed = [ int ] $stats . totalFailed + $ ( if ( $isFailed ) { 1 } else { 0 } )
$newTotalDegraded = [ int ] $stats . totalDegraded + $ ( if ( $statusText -eq " degraded " ) { 1 } else { 0 } )
$newTotalBlocked = [ int ] $stats . totalBlocked + $ ( if ( $statusText -eq " blocked " ) { 1 } else { 0 } )
$newTotalVerifying = [ int ] $stats . totalVerifying + $ ( if ( $statusText -eq " verifying " ) { 1 } else { 0 } )
$newTotalCandidateReady = [ int ] $stats . totalCandidateReady + $ ( if ( $statusText -eq " candidate_ready " ) { 1 } else { 0 } )
$outcomeContractActivatedAt = if ( $stats . outcomeContractActivatedAt ) { $stats . outcomeContractActivatedAt } else { $now }
$updated = [ pscustomobject ] @ {
schemaVersion = " agent99_problem_counts_v2 "
generatedAt = $now
outcomeContractActivatedAt = $outcomeContractActivatedAt
legacyResolvedCountMayIncludeTransportOnly = [ bool ] ( $stats . schemaVersion -ne " agent99_problem_counts_v2 " -or ( [ int ] $stats . totalResolved -gt [ int ] $stats . totalVerifiedResolved ) )
totalEvents = $newTotalEvents
totalResolved = $newTotalResolved
totalVerifiedResolved = $newTotalVerifiedResolved
totalFailed = $newTotalFailed
totalDegraded = $newTotalDegraded
totalBlocked = $newTotalBlocked
totalVerifying = $newTotalVerifying
totalCandidateReady = $newTotalCandidateReady
problems = @ ( $problems | Sort-Object @ { Expression = { [ int ] $_ . occurrences } ; Descending = $true } , @ { Expression = { $_ . lastSeenAt } ; Descending = $true } )
}
$updated | ConvertTo-Json -Depth 10 | Set-Content -Path $statsPath -Encoding UTF8
$rows = @ ( $updated . problems | Select-Object -First 50 | ForEach-Object {
" - key= $( $_ . key ) ; occurrences= $( $_ . occurrences ) ; r esolved= $( $_ . r esolved) ; failed= $( $_ . failed ) ; lastStatus= $( $_ . lastStatus ) ; lastMode= $( $_ . lastMode ) ; evidence= $( $_ . lastEvidence ) "
" - key= $( $_ . key ) ; occurrences= $( $_ . occurrences ) ; verifiedR esolved= $( $_ . verifiedR esolved) ; failed= $( $_ . failed ) ; degraded= $( $_ . degraded ) ; blocked= $( $_ . blocked ) ; verifying= $( $_ . verifying ) ; candidateReady= $( $_ . candidateReady ) ; lastStatus= $( $_ . lastStatus ) ; lastMode= $( $_ . lastMode ) ; evidence= $( $_ . lastEvidence ) "
} )
$km = @ (
" # Agent99 Incident KM " ,
@@ -2287,8 +2747,13 @@ function Update-AgentProblemKnowledge {
" ## Totals " ,
" " ,
" - totalEvents: $( $updated . totalEvents ) " ,
" - totalResolved: $( $updated . totalResolved ) " ,
" - totalResolvedLegacyAndVerified : $( $updated . totalResolved ) " ,
" - totalVerifiedResolved: $( $updated . totalVerifiedResolved ) " ,
" - totalFailed: $( $updated . totalFailed ) " ,
" - totalDegraded: $( $updated . totalDegraded ) " ,
" - totalBlocked: $( $updated . totalBlocked ) " ,
" - totalVerifying: $( $updated . totalVerifying ) " ,
" - totalCandidateReady: $( $updated . totalCandidateReady ) " ,
" " ,
" ## Recurring Problems " ,
" " ,
@@ -2300,7 +2765,12 @@ function Update-AgentProblemKnowledge {
key = $key
occurrences = $existing . occurrences
resolved = $existing . resolved
verifiedResolved = $existing . verifiedResolved
failed = $existing . failed
degraded = $existing . degraded
blocked = $existing . blocked
verifying = $existing . verifying
candidateReady = $existing . candidateReady
lastStatus = $existing . lastStatus
statsPath = $statsPath
eventsPath = $eventsPath
@@ -2388,7 +2858,12 @@ function Get-AgentDashboardSummary {
path = $problemStats . path
totalEvents = $problemStats . totalEvents
totalResolved = $problemStats . totalResolved
totalVerifiedResolved = $problemStats . totalVerifiedResolved
totalFailed = $problemStats . totalFailed
totalDegraded = $problemStats . totalDegraded
totalBlocked = $problemStats . totalBlocked
totalVerifying = $problemStats . totalVerifying
totalCandidateReady = $problemStats . totalCandidateReady
top = @ ( $problemStats . problems | Select-Object -First 10 )
}
tasks = $tasks
@@ -2434,7 +2909,7 @@ function Write-AgentDashboard {
" <tr><td> $( Convert-AgentHtml $ev . name ) </td><td> $( Convert-AgentHtml $ev . file ) </td><td> $( Convert-AgentHtml $ev . ageMinutes ) </td><td> $( Convert-AgentHtml $ev . parseError ) </td></tr> "
}
$problemRows = foreach ( $problem in @ ( $Summary . problemCounts . top ) ) {
" <tr><td> $( Convert-AgentHtml $problem . key ) </td><td> $( Convert-AgentHtml $problem . occurrences ) </td><td> $( Convert-AgentHtml $problem . r esolved) </td><td> $( Convert-AgentHtml $problem . failed ) </td><td> $( Convert-AgentHtml $problem . lastStatus ) </td><td> $( Convert-AgentHtml $problem . lastMode ) </td></tr> "
" <tr><td> $( Convert-AgentHtml $problem . key ) </td><td> $( Convert-AgentHtml $problem . occurrences ) </td><td> $( Convert-AgentHtml $problem . verifiedR esolved) </td><td> $( Convert-AgentHtml $problem . failed ) </td><td> $( Convert-AgentHtml $problem . degraded ) </td><td> $( Convert-AgentHtml $problem . blocked ) </td><td> $( Convert-AgentHtml $problem . verifying ) </td><td> $( Convert-AgentHtml $problem . candidateReady ) </td><td> $( Convert-AgentHtml $problem . lastStatus ) </td><td> $( Convert-AgentHtml $problem . lastMode ) </td></tr> "
}
$html = @"
@@ -2469,12 +2944,14 @@ function Write-AgentDashboard {
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > Q u e u e P e n d i n g < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . queue . pending ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > A l e r t s P e n d i n g < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . alerts . pending ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > P r o b l e m E v e n t s < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalEvents ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > R e s o l v e d < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalResolved ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > V e r i f i e d e s o l v e d < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalVerified Resolved ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > F a i l e d < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalFailed ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > D e g r a d e d < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalDegraded ) < / d i v > < / d i v >
< d i v c l a s s = " c a r d " > < d i v c l a s s = " l a b e l " > V e r i f y i n g < / d i v > < d i v c l a s s = " v a l u e " > $( Convert-AgentHtml $Summary . problemCounts . totalVerifying ) < / d i v > < / d i v >
< / d i v >
< p c l a s s = " p a t h " > A g e n t r o o t : $( Convert-AgentHtml $Summary . agentRoot ) < b r > Q u e u e : $( Convert-AgentHtml $Summary . queue . path ) < b r > A l e r t s : $( Convert-AgentHtml $Summary . alerts . incomingPath ) < b r > P r o b l e m c o u n t s : $( Convert-AgentHtml $Summary . problemCounts . path ) < b r > E v i d e n c e : $( Convert-AgentHtml $Summary . evidenceDir ) < / p >
< h 2 > P r o b l e m C o u n t s < / h 2 >
< t a b l e > < t r > < t h > P r o b l e m < / t h > < t h > O c c u r r e n c e s < / t h > < t h > R e s o l v e d < / t h > < t h > F a i l e d < / t h > < t h > L a s t S t a t u s < / t h > < t h > L a s t M o d e < / t h > < / t r > $( $problemRows -join " `n " ) < / t a b l e >
< t a b l e > < t r > < t h > P r o b l e m < / t h > < t h > O c c u r r e n c e s < / t h > < t h > V e r i f i e d e s o l v e d < / t h > < t h > F a i l e d < / t h > < t h > D e g r a d e d < / t h > < t h > B l o c k e d < / t h > < t h > V e r i f y i n g < / t h > < t h > C a n d i d a t e R e a d y < / t h > < t h > a s t S t a t u s < / t h > < t h > L a s t M o d e < / t h > < / t r > $( $problemRows -join " `n " ) < / t a b l e >
< h 2 > S c h e d u l e d T a s k s < / h 2 >
< t a b l e > < t r > < t h > T a s k < / t h > < t h > S t a t e < / t h > < t h > L a s t R e s u l t < / t h > < t h > L a s t R u n < / t h > < t h > N e x t R u n < / t h > < / t r > $( $taskRows -join " `n " ) < / t a b l e >
< h 2 > H o s t s < / h 2 >
@@ -2566,6 +3043,9 @@ function Invoke-AgentQueuedCommands {
$alertSeverity = if ( $command . PSObject . Properties [ " alertSeverity " ] ) { [ string ] $command . alertSeverity } else { $null }
$alertService = if ( $command . PSObject . Properties [ " alertService " ] ) { [ string ] $command . alertService } else { $null }
$alertHost = if ( $command . PSObject . Properties [ " alertHost " ] ) { [ string ] $command . alertHost } else { $null }
$sourceEventResolutionRequired = if ( $command . PSObject . Properties [ " sourceEventResolutionRequired " ] ) { Convert-AgentBool $command . sourceEventResolutionRequired } else { [ bool ] $alertId }
$sourceEventResolved = if ( $command . PSObject . Properties [ " sourceEventResolved " ] ) { [ object ] ( Convert-AgentBool $command . sourceEventResolved ) } else { $null }
$sourceEventEvidence = if ( $command . PSObject . Properties [ " sourceEventEvidence " ] ) { [ string ] $command . sourceEventEvidence } else { " " }
$replyChatId = if ( $command . PSObject . Properties [ " replyChatId " ] ) { [ string ] $command . replyChatId } else { " " }
$suppressTelegram = if ( $command . PSObject . Properties [ " suppressTelegram " ] ) { Convert-AgentBool $command . suppressTelegram } else { $false }
if ( $modeName -notin $allowedModes ) {
@@ -2604,6 +3084,7 @@ function Invoke-AgentQueuedCommands {
} else {
-3
}
$outcome = Get-AgentOutcomeContract -ModeName $modeName -TransportExitCode $exitCode -LatestEvidence $latest -StartedAt $startTime -SourceEventResolutionRequired $sourceEventResolutionRequired -SourceEventResolved $sourceEventResolved -SourceEventEvidence $sourceEventEvidence
$result = [ pscustomobject ] @ {
id = $id
mode = $modeName
@@ -2618,7 +3099,12 @@ function Invoke-AgentQueuedCommands {
alertService = $alertService
alertHost = $alertHost
suppressTelegram = $suppressTelegram
o k= [ bool ] ( $exitCode -eq 0 )
transportO k= [ bool ] ( $exitCode -eq 0 )
ok = [ bool ] $outcome . resolved
outcomeState = $outcome . state
verifierPassed = [ bool ] $outcome . verifierPassed
sourceEventResolved = [ bool ] $outcome . sourceEventResolved
outcome = $outcome
exitCode = $exitCode
durationSeconds = [ math ] :: Round ( ( ( Get-Date ) - $startTime ) . TotalSeconds , 1 )
evidence = $latest . path
@@ -2634,7 +3120,8 @@ function Invoke-AgentQueuedCommands {
if ( $replyChatId ) {
$recordData | Add-Member -MemberType NoteProperty -Name replyChatId -Value $replyChatId -Force
}
Record-AgentEvent " operator_command_result " $ ( if ( $exitCode -eq 0 ) { " info " } else { " critical " } ) " id= $id mode= $modeName exitCode= $exitCode evidence= $( $latest . path ) " $recordData -Alert
$outcomeSeverity = if ( $outcome . state -eq " resolved " ) { " info " } elseif ( $outcome . state -eq " failed " ) { " critical " } else { " warning " }
Record-AgentEvent " operator_command_result " $outcomeSeverity " id= $id mode= $modeName outcome= $( $outcome . state ) transportExitCode= $exitCode verifierPassed= $( $outcome . verifierPassed ) sourceEventResolved= $( $outcome . sourceEventResolved ) evidence= $( $latest . path ) " $recordData -Alert
$results + = $result
}
$results
@@ -3177,6 +3664,10 @@ if ($SelfTestSensorGate) {
Invoke-AgentSensorGateSelfTest
}
if ( $SelfTestOutcomeContract ) {
Invoke-AgentOutcomeContractSelfTest
}
Write-AgentLog " agent99_start mode= $Mode controlledApply= $ControlledApply config= $ConfigPath "
Record-AgentEvent " agent_start " " info " " mode= $Mode controlledApply= $ControlledApply host= $env:COMPUTERNAME " $null -Alert
ogt