30e0be8987
All checks were successful
CD Pipeline / select-latest-carrier (push) Successful in 47s
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 4m53s
CD Pipeline / revalidate-deploy-carrier (push) Successful in 29s
CD Pipeline / build-and-deploy (push) Successful in 14m53s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 0s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 9s
CD Pipeline / revalidate-post-deploy-carrier (push) Successful in 47s
CD Pipeline / post-deploy-checks (push) Successful in 3m10s
1093 lines
48 KiB
PowerShell
1093 lines
48 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Verify")]
|
|
[string]$Mode = "Verify",
|
|
[string]$AgentRoot = "C:\Wooo\Agent99",
|
|
[ValidateSet("FullRuntime", "PromotionReserve")]
|
|
[string]$ReadinessScope = "FullRuntime",
|
|
[ValidateRange(0, 20)]
|
|
[int]$MinimumEvidenceFreshnessReserveMinutes = 0
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Get-AgentLivePreflightDecision {
|
|
param(
|
|
[bool]$AgentRootPresent,
|
|
[object]$Manifest,
|
|
[bool]$ManifestCheckRequired,
|
|
[int]$ExpectedRuntimeFileCount,
|
|
[int]$TaskFailureCount,
|
|
[int]$RelayListenerCount,
|
|
[int]$RequiredEvidenceStaleCount,
|
|
[int]$RequiredEvidenceReserveFailureCount,
|
|
[int]$RequiredEvidenceParseFailureCount,
|
|
[int]$RequiredEvidenceContentFailureCount,
|
|
[string]$SelfHealthSeverity,
|
|
[int]$SelfHealthPerformanceIssueCount,
|
|
[int]$StaleAlertExecutionCount,
|
|
[int]$AlertIncomingCount,
|
|
[int]$StaleAlertIncomingCount,
|
|
[int]$PendingCommandBlockingCount,
|
|
[int]$PromotionDeferredCommandCount
|
|
)
|
|
|
|
$blockingReasons = @()
|
|
$warningReasons = @()
|
|
$manifestExists = [bool]($Manifest -and $Manifest.exists)
|
|
$manifestParseReady = [bool](
|
|
$manifestExists -and
|
|
-not ([string]$Manifest.parseError)
|
|
)
|
|
$manifestIdentityReady = [bool](
|
|
$manifestParseReady -and
|
|
[string]$Manifest.sourceRevision -match "^[a-fA-F0-9]{40}$" -and
|
|
[int]$Manifest.fileCount -eq $ExpectedRuntimeFileCount
|
|
)
|
|
$manifestContentReady = [bool](
|
|
$manifestParseReady -and
|
|
$Manifest.runtimeMatched -eq $true -and
|
|
[int]$Manifest.mismatchCount -eq 0
|
|
)
|
|
|
|
if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" }
|
|
if ($ManifestCheckRequired) {
|
|
if (-not $manifestExists) {
|
|
$blockingReasons += "runtime_manifest_missing"
|
|
} elseif (-not $manifestParseReady) {
|
|
$blockingReasons += "runtime_manifest_parse_failed"
|
|
} else {
|
|
if (-not $manifestIdentityReady) {
|
|
$blockingReasons += "runtime_manifest_identity_invalid"
|
|
}
|
|
if (-not $manifestContentReady) {
|
|
$blockingReasons += "runtime_manifest_mismatch"
|
|
}
|
|
}
|
|
}
|
|
if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
|
if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" }
|
|
if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" }
|
|
if ($RequiredEvidenceReserveFailureCount -gt 0) { $blockingReasons += "required_evidence_freshness_reserve_insufficient" }
|
|
if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
|
|
if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
|
|
|
|
$normalizedSelfHealth = ([string]$SelfHealthSeverity).Trim().ToLowerInvariant()
|
|
switch ($normalizedSelfHealth) {
|
|
"ok" {}
|
|
"warning" {
|
|
$warningReasons += "self_health_warning"
|
|
if ($SelfHealthPerformanceIssueCount -gt 0) {
|
|
$warningReasons += "performance_warning"
|
|
}
|
|
}
|
|
# Self-health also aggregates external host performance and Telegram
|
|
# delivery. Keep the preflight exit degraded, but emit the receiver's
|
|
# canonical external-health reason so unrelated asset pressure cannot be
|
|
# mistaken for a runtime-plane integrity failure.
|
|
"critical" { $blockingReasons += "self_health_not_ok" }
|
|
default { $blockingReasons += "self_health_unclassified" }
|
|
}
|
|
|
|
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
|
|
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandBlockingCount -gt 0) {
|
|
$blockingReasons += "pending_work_before_reboot"
|
|
} elseif ($AlertIncomingCount -gt 0) {
|
|
$warningReasons += "fresh_alert_pending_next_inbox_tick"
|
|
}
|
|
if ($PromotionDeferredCommandCount -gt 0) {
|
|
$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
deploymentEligible = [bool]($blockingReasons.Count -eq 0)
|
|
blockingReasons = @($blockingReasons)
|
|
warningReasons = @($warningReasons)
|
|
warningCount = [int]$warningReasons.Count
|
|
manifestCheckRequired = $ManifestCheckRequired
|
|
manifestIdentityReady = $manifestIdentityReady
|
|
manifestContentReady = $manifestContentReady
|
|
selfHealthClassification = $normalizedSelfHealth
|
|
}
|
|
}
|
|
|
|
function Get-TaskReadback {
|
|
param(
|
|
[string]$TaskPath,
|
|
[string]$TaskName,
|
|
[string[]]$RequiredArgumentMarkers
|
|
)
|
|
|
|
try {
|
|
$tasks = @(Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop)
|
|
if ($tasks.Count -ne 1) { throw "exact_root_task_count_invalid" }
|
|
$task = $tasks[0]
|
|
$info = Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName -ErrorAction Stop
|
|
$lastResult = [int64]$info.LastTaskResult
|
|
$state = [string]$task.State
|
|
$actions = @($task.Actions)
|
|
$actionDefinitionReady = [bool](
|
|
$actions.Count -eq 1 -and
|
|
[string]$actions[0].Execute -match '(?i)\\WindowsPowerShell\\v1\.0\\powershell\.exe$'
|
|
)
|
|
if ($actionDefinitionReady) {
|
|
foreach ($marker in @($RequiredArgumentMarkers)) {
|
|
if (
|
|
-not $marker -or
|
|
([string]$actions[0].Arguments).IndexOf(
|
|
$marker,
|
|
[StringComparison]::OrdinalIgnoreCase
|
|
) -lt 0
|
|
) {
|
|
$actionDefinitionReady = $false
|
|
break
|
|
}
|
|
}
|
|
}
|
|
$principalReady = [bool](
|
|
[string]$task.Principal.UserId -match '(?i)(^|\\)Administrator$' -and
|
|
[string]$task.Principal.LogonType -eq "S4U" -and
|
|
[string]$task.Principal.RunLevel -eq "Highest"
|
|
)
|
|
$definitionReady = [bool]($actionDefinitionReady -and $principalReady)
|
|
$healthy = [bool](
|
|
$task.Settings.Enabled -and
|
|
$definitionReady -and
|
|
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
|
|
)
|
|
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
taskPath = $TaskPath
|
|
exists = $true
|
|
enabled = [bool]$task.Settings.Enabled
|
|
state = $state
|
|
lastTaskResult = $lastResult
|
|
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
|
|
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
|
|
actionDefinitionReady = $actionDefinitionReady
|
|
principalReady = $principalReady
|
|
definitionReady = $definitionReady
|
|
healthy = $healthy
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
taskPath = $TaskPath
|
|
exists = $false
|
|
enabled = $false
|
|
state = "Missing"
|
|
lastTaskResult = $null
|
|
lastRunTime = ""
|
|
nextRunTime = ""
|
|
actionDefinitionReady = $false
|
|
principalReady = $false
|
|
definitionReady = $false
|
|
healthy = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-DirectoryReadback {
|
|
param(
|
|
[string]$RelativePath,
|
|
[switch]$RootQueueFilesOnly
|
|
)
|
|
|
|
$path = Join-Path $AgentRoot $RelativePath
|
|
$exists = [bool](Test-Path -LiteralPath $path -PathType Container)
|
|
$count = 0
|
|
$staleCount = 0
|
|
$oldest = $null
|
|
$newest = $null
|
|
if ($exists) {
|
|
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
|
|
$path,
|
|
"*.json",
|
|
[IO.SearchOption]::TopDirectoryOnly
|
|
)) {
|
|
$name = [IO.Path]::GetFileName($candidatePath)
|
|
if ($RootQueueFilesOnly -and $name -match "^(processed|failed|running)-") {
|
|
continue
|
|
}
|
|
$file = [IO.FileInfo]::new($candidatePath)
|
|
$count += 1
|
|
if (-not $oldest -or $file.LastWriteTimeUtc -lt $oldest.LastWriteTimeUtc) {
|
|
$oldest = $file
|
|
}
|
|
if (-not $newest -or $file.LastWriteTimeUtc -gt $newest.LastWriteTimeUtc) {
|
|
$newest = $file
|
|
}
|
|
if (((Get-Date) - $file.LastWriteTime).TotalMinutes -gt 5) {
|
|
$staleCount += 1
|
|
}
|
|
}
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
relativePath = $RelativePath
|
|
exists = $exists
|
|
count = [int]$count
|
|
oldest = if ($oldest) { $oldest.LastWriteTime.ToString("o") } else { "" }
|
|
newest = if ($newest) { $newest.LastWriteTime.ToString("o") } else { "" }
|
|
staleOverFiveMinutes = [int]$staleCount
|
|
}
|
|
}
|
|
|
|
function Get-AgentBoundedNewestFiles {
|
|
param(
|
|
[string]$DirectoryPath,
|
|
[string]$Pattern,
|
|
[ValidateRange(1, 256)]
|
|
[int]$CandidateWindow = 32
|
|
)
|
|
|
|
# Evidence freshness is authoritative from filesystem UTC mtime. Some
|
|
# evidence names are stable aliases and are not chronologically sortable.
|
|
# Keep only the newest bounded window in a native .NET set so a large
|
|
# evidence directory never becomes a PowerShell object-sort workload.
|
|
$selected = [Collections.Generic.SortedSet[string]]::new(
|
|
[StringComparer]::OrdinalIgnoreCase
|
|
)
|
|
$enumeratedCount = 0
|
|
if (Test-Path -LiteralPath $DirectoryPath -PathType Container) {
|
|
foreach ($candidatePath in [IO.Directory]::EnumerateFiles(
|
|
$DirectoryPath,
|
|
$Pattern,
|
|
[IO.SearchOption]::TopDirectoryOnly
|
|
)) {
|
|
$enumeratedCount += 1
|
|
$candidate = [IO.FileInfo]::new($candidatePath)
|
|
$sortKey = (
|
|
$candidate.LastWriteTimeUtc.Ticks.ToString(
|
|
"D19",
|
|
[Globalization.CultureInfo]::InvariantCulture
|
|
) + "|" + $candidate.FullName
|
|
)
|
|
$null = $selected.Add($sortKey)
|
|
if ($selected.Count -gt $CandidateWindow) {
|
|
$null = $selected.Remove($selected.Min)
|
|
}
|
|
}
|
|
}
|
|
$selectedKeys = [string[]]@($selected)
|
|
[Array]::Reverse($selectedKeys)
|
|
|
|
return [pscustomobject]@{
|
|
files = @($selectedKeys | ForEach-Object {
|
|
[IO.FileInfo]::new($_.Substring(20))
|
|
})
|
|
enumeratedCount = [int]$enumeratedCount
|
|
candidateWindow = [int]$CandidateWindow
|
|
windowTruncated = [bool]($enumeratedCount -gt $selected.Count)
|
|
}
|
|
}
|
|
|
|
function Get-AgentControlLoopMaintenanceReadback {
|
|
$evidenceDir = Join-Path $AgentRoot "evidence"
|
|
$selection = Get-AgentBoundedNewestFiles `
|
|
$evidenceDir `
|
|
"agent99-control-loop-maintenance-2*.json" `
|
|
8
|
|
$file = @($selection.files | Select-Object -First 1)
|
|
$receipt = $null
|
|
$intent = $null
|
|
$parseError = ""
|
|
if ($file.Count -eq 1) {
|
|
try {
|
|
$receipt = Get-Content -LiteralPath $file[0].FullName -Raw | ConvertFrom-Json
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
|
|
$intentRef = if ($receipt -and $receipt.PSObject.Properties["operationIntentRef"]) {
|
|
[string]$receipt.operationIntentRef
|
|
} else { "" }
|
|
$intentRefSafe = [bool](
|
|
$intentRef -match "^agent99-control-loop-maintenance-intent-[0-9]{8}-[0-9]{6}-[0-9]{3}-[0-9a-f]{8}\.json$"
|
|
)
|
|
$intentPath = if ($intentRefSafe) { Join-Path $evidenceDir $intentRef } else { "" }
|
|
if ($intentPath -and (Test-Path -LiteralPath $intentPath -PathType Leaf)) {
|
|
try {
|
|
$intent = Get-Content -LiteralPath $intentPath -Raw | ConvertFrom-Json
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
$legacyPropertySet = @(
|
|
"blockers", "controlStopped", "expectedControlCreated", "expectedControlPid",
|
|
"expectedRecoverCreated", "expectedRecoverParentPid", "expectedRecoverPid",
|
|
"hostRebootPerformed", "leaseMetadataRemoved", "mode", "precheckPassed",
|
|
"recoverStopped", "runtimeWritePerformed", "schemaVersion", "secretValueRead",
|
|
"taskEnabled", "taskName", "terminal", "timestamp", "vmPowerChangePerformed",
|
|
"vmwareVmxProcessTerminated", "vmxLockDeleted"
|
|
)
|
|
$receiptPropertySet = if ($receipt) {
|
|
@($receipt.PSObject.Properties.Name | Sort-Object)
|
|
} else { @() }
|
|
$legacyPropertySetMatches = [bool](
|
|
$receiptPropertySet.Count -eq $legacyPropertySet.Count -and
|
|
(@($receiptPropertySet) -join ",") -eq (@($legacyPropertySet | Sort-Object) -join ",")
|
|
)
|
|
$legacyIdentityFieldsValid = [bool](
|
|
$receipt -and
|
|
$receipt.expectedControlPid -is [int] -and [int]$receipt.expectedControlPid -gt 0 -and
|
|
$receipt.expectedRecoverPid -is [int] -and [int]$receipt.expectedRecoverPid -gt 0 -and
|
|
$receipt.expectedRecoverParentPid -is [int] -and [int]$receipt.expectedRecoverParentPid -gt 0 -and
|
|
[string]$receipt.expectedControlCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
|
|
[string]$receipt.expectedRecoverCreated -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T" -and
|
|
[string]$receipt.timestamp -match "^[0-9]{4}-[0-9]{2}-[0-9]{2}T"
|
|
)
|
|
$legacyReceipt = [bool](
|
|
$receipt -and
|
|
-not $receipt.PSObject.Properties["operationIntentRef"] -and
|
|
$receipt.PSObject.Properties["taskEnabled"] -and
|
|
$legacyPropertySetMatches -and
|
|
$legacyIdentityFieldsValid -and
|
|
@($receipt.blockers).Count -eq 0
|
|
)
|
|
|
|
$ageMinutes = if ($file.Count -eq 1) {
|
|
[math]::Round(((Get-Date) - $file[0].LastWriteTime).TotalMinutes, 1)
|
|
} else { $null }
|
|
$activeControlProcesses = @(Get-CimInstance Win32_Process -Filter "Name='powershell.exe'" -ErrorAction SilentlyContinue |
|
|
Where-Object {
|
|
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+ControlTick(\s|$)' -or
|
|
([string]$_.CommandLine) -match '(?i)agent99-run\.ps1"?\s+-Mode\s+Recover\s+-ControlledApply(\s|$)'
|
|
})
|
|
$leasePresent = [bool](Test-Path -LiteralPath (Join-Path $AgentRoot "state\recovery-transport-lease.json") -PathType Leaf)
|
|
$recoveryMutexActive = $true
|
|
$mutex = $null
|
|
$mutexAcquired = $false
|
|
try {
|
|
$mutex = [Threading.Mutex]::new($false, "Global\WoooAgent99RecoveryTransportV1")
|
|
try {
|
|
$mutexAcquired = [bool]$mutex.WaitOne(0)
|
|
} catch [Threading.AbandonedMutexException] {
|
|
$mutexAcquired = $true
|
|
}
|
|
$recoveryMutexActive = [bool](-not $mutexAcquired)
|
|
} finally {
|
|
if ($mutexAcquired -and $mutex) { try { $mutex.ReleaseMutex() } catch {} }
|
|
if ($mutex) { $mutex.Dispose() }
|
|
}
|
|
|
|
$commonReceiptFieldsValid = [bool](
|
|
$receipt -and
|
|
[string]$receipt.schemaVersion -eq "agent99_control_loop_maintenance_v2" -and
|
|
[string]$receipt.mode -eq "Apply" -and
|
|
[string]$receipt.taskName -eq "Wooo-Agent99-Control-Loop" -and
|
|
[string]$receipt.terminal -eq "control_loop_maintenance_frozen" -and
|
|
$receipt.precheckPassed -is [bool] -and $receipt.precheckPassed -and
|
|
$receipt.controlStopped -is [bool] -and $receipt.controlStopped -and
|
|
$receipt.recoverStopped -is [bool] -and $receipt.recoverStopped -and
|
|
$receipt.leaseMetadataRemoved -is [bool] -and $receipt.leaseMetadataRemoved -and
|
|
$receipt.runtimeWritePerformed -is [bool] -and $receipt.runtimeWritePerformed -and
|
|
$receipt.vmPowerChangePerformed -is [bool] -and -not $receipt.vmPowerChangePerformed -and
|
|
$receipt.vmwareVmxProcessTerminated -is [bool] -and -not $receipt.vmwareVmxProcessTerminated -and
|
|
$receipt.vmxLockDeleted -is [bool] -and -not $receipt.vmxLockDeleted -and
|
|
$receipt.hostRebootPerformed -is [bool] -and -not $receipt.hostRebootPerformed -and
|
|
$receipt.secretValueRead -is [bool] -and -not $receipt.secretValueRead
|
|
)
|
|
$modernReceiptFieldsValid = [bool](
|
|
$commonReceiptFieldsValid -and
|
|
-not $legacyReceipt -and
|
|
$receipt.taskEnabledAfter -is [bool] -and -not $receipt.taskEnabledAfter
|
|
)
|
|
$legacyReceiptFieldsValid = [bool](
|
|
$commonReceiptFieldsValid -and
|
|
$legacyReceipt -and
|
|
$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled
|
|
)
|
|
$receiptFieldsValid = [bool]($modernReceiptFieldsValid -or $legacyReceiptFieldsValid)
|
|
$intentFieldsValid = [bool](
|
|
$legacyReceipt -or
|
|
(
|
|
$intent -and
|
|
[string]$intent.schemaVersion -eq "agent99_control_loop_maintenance_intent_v1" -and
|
|
[string]$intent.taskName -eq "Wooo-Agent99-Control-Loop" -and
|
|
$intent.vmPowerChangePerformed -is [bool] -and -not $intent.vmPowerChangePerformed -and
|
|
$intent.vmwareVmxProcessTerminated -is [bool] -and -not $intent.vmwareVmxProcessTerminated -and
|
|
$intent.secretValueRead -is [bool] -and -not $intent.secretValueRead -and
|
|
(@($intent.plannedActions) -join ",") -eq "disable_control_task,stop_exact_control_tree,stop_exact_recover_tree,remove_exact_recovery_lease"
|
|
)
|
|
)
|
|
$valid = [bool](
|
|
-not $parseError -and
|
|
$file.Count -eq 1 -and
|
|
$null -ne $ageMinutes -and
|
|
$ageMinutes -ge 0 -and
|
|
$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 }) -and
|
|
($legacyReceipt -or $intentRefSafe) -and
|
|
$receiptFieldsValid -and
|
|
$intentFieldsValid -and
|
|
$activeControlProcesses.Count -eq 0 -and
|
|
-not $leasePresent -and
|
|
-not $recoveryMutexActive
|
|
)
|
|
|
|
return [pscustomobject]@{
|
|
valid = $valid
|
|
receiptFile = if ($file.Count -eq 1) { $file[0].Name } else { "" }
|
|
ageMinutes = $ageMinutes
|
|
parseError = $parseError
|
|
intentRefPresent = $intentRefSafe
|
|
contractVersion = if ($legacyReceipt) { "legacy_v2_exact_upgrade_bridge" } else { "intent_bound_v2" }
|
|
receiptTimestamp = if ($receipt) { [string]$receipt.timestamp } else { "" }
|
|
receiptLastWriteTime = if ($file.Count -eq 1) { $file[0].LastWriteTime.ToString("o") } else { "" }
|
|
legacyPropertySetMatches = $legacyPropertySetMatches
|
|
legacyIdentityFieldsValid = $legacyIdentityFieldsValid
|
|
receiptFieldsValid = $receiptFieldsValid
|
|
intentFieldsValid = $intentFieldsValid
|
|
activeControlProcessCount = [int]$activeControlProcesses.Count
|
|
recoveryLeasePresent = $leasePresent
|
|
recoveryMutexActive = $recoveryMutexActive
|
|
secretValueRead = $false
|
|
remoteWritePerformed = $false
|
|
}
|
|
}
|
|
|
|
function Get-AgentPendingCommandPromotionReadback {
|
|
param(
|
|
[object]$MaintenanceReadback,
|
|
[ValidateSet("FullRuntime", "PromotionReserve")]
|
|
[string]$Scope
|
|
)
|
|
|
|
$queueRoot = Join-Path $AgentRoot "queue"
|
|
$files = @()
|
|
if (Test-Path -LiteralPath $queueRoot -PathType Container) {
|
|
$files = @([IO.Directory]::EnumerateFiles(
|
|
$queueRoot,
|
|
"*.json",
|
|
[IO.SearchOption]::TopDirectoryOnly
|
|
) | Where-Object {
|
|
[IO.Path]::GetFileName($_) -notmatch "^(processed|failed|running)-"
|
|
})
|
|
}
|
|
|
|
$observedCount = [int]$files.Count
|
|
$result = [ordered]@{
|
|
scope = $Scope
|
|
observedCount = $observedCount
|
|
deferredCount = 0
|
|
blockingCount = $observedCount
|
|
eligibleMaintenanceAutoRecoverCount = 0
|
|
boundedLimit = 8
|
|
contractValid = [bool]($observedCount -eq 0)
|
|
reason = if ($observedCount -eq 0) { "queue_empty" } else { "pending_commands_require_runtime" }
|
|
items = @()
|
|
rawInstructionStored = $false
|
|
secretValueRead = $false
|
|
remoteWritePerformed = $false
|
|
}
|
|
if ($observedCount -eq 0) {
|
|
return [pscustomobject]$result
|
|
}
|
|
if (
|
|
-not $MaintenanceReadback -or
|
|
-not $MaintenanceReadback.receiptFieldsValid -or
|
|
-not $MaintenanceReadback.intentFieldsValid
|
|
) {
|
|
$result.reason = "maintenance_contract_invalid"
|
|
return [pscustomobject]$result
|
|
}
|
|
if ($Scope -eq "PromotionReserve" -and -not $MaintenanceReadback.valid) {
|
|
$result.reason = "maintenance_current_state_invalid"
|
|
return [pscustomobject]$result
|
|
}
|
|
if ($observedCount -gt $result.boundedLimit) {
|
|
$result.reason = "maintenance_pending_command_limit_exceeded"
|
|
return [pscustomobject]$result
|
|
}
|
|
|
|
$maintenanceTimestamp = [datetimeoffset]::MinValue
|
|
$maintenanceTimestampValid = [datetimeoffset]::TryParse(
|
|
[string]$MaintenanceReadback.receiptTimestamp,
|
|
[ref]$maintenanceTimestamp
|
|
)
|
|
if (-not $maintenanceTimestampValid) {
|
|
$result.reason = "maintenance_timestamp_invalid"
|
|
return [pscustomobject]$result
|
|
}
|
|
|
|
$requiredPropertySet = @(
|
|
"approvalId", "automationRunId", "canonicalDigest", "controlledApply",
|
|
"correlationKey", "createdAt", "dispatchIdentitySchemaVersion",
|
|
"dispatchRouteId", "executionGeneration", "externalId", "id",
|
|
"idempotencyKey", "incidentId", "instruction", "lockOwner", "mode",
|
|
"observation", "projectId", "reason", "requestedBy", "singleFlightKey",
|
|
"source", "sourceFingerprint", "traceId", "workItemId"
|
|
) | Sort-Object
|
|
$safeItems = @()
|
|
$allValid = $true
|
|
foreach ($path in $files) {
|
|
$file = [IO.FileInfo]::new($path)
|
|
$payload = $null
|
|
$parseError = ""
|
|
$sha256 = ""
|
|
try {
|
|
$bytes = [IO.File]::ReadAllBytes($path)
|
|
$sha = [Security.Cryptography.SHA256]::Create()
|
|
try {
|
|
$sha256 = -join @($sha.ComputeHash($bytes) | ForEach-Object { $_.ToString("x2") })
|
|
} finally {
|
|
$sha.Dispose()
|
|
}
|
|
$jsonOffset = if (
|
|
$bytes.Length -ge 3 -and
|
|
$bytes[0] -eq 0xef -and
|
|
$bytes[1] -eq 0xbb -and
|
|
$bytes[2] -eq 0xbf
|
|
) { 3 } else { 0 }
|
|
$jsonText = [Text.Encoding]::UTF8.GetString(
|
|
$bytes,
|
|
$jsonOffset,
|
|
$bytes.Length - $jsonOffset
|
|
)
|
|
$payload = $jsonText | ConvertFrom-Json
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
}
|
|
$propertySet = if ($payload) {
|
|
@($payload.PSObject.Properties.Name | Sort-Object)
|
|
} else { @() }
|
|
$propertySetMatches = [bool](
|
|
$propertySet.Count -eq $requiredPropertySet.Count -and
|
|
(@($propertySet) -join ",") -eq (@($requiredPropertySet) -join ",")
|
|
)
|
|
$createdAt = [datetimeoffset]::MinValue
|
|
$createdAtValid = [bool](
|
|
$payload -and
|
|
[datetimeoffset]::TryParse([string]$payload.createdAt, [ref]$createdAt)
|
|
)
|
|
$ageMinutes = if ($createdAtValid) {
|
|
[math]::Round(([datetimeoffset]::Now - $createdAt).TotalMinutes, 1)
|
|
} else { $null }
|
|
$fileBaseName = [IO.Path]::GetFileNameWithoutExtension($file.Name)
|
|
$nativeStringFields = @(
|
|
"approvalId", "automationRunId", "canonicalDigest", "correlationKey",
|
|
"createdAt", "dispatchIdentitySchemaVersion", "dispatchRouteId",
|
|
"executionGeneration", "externalId", "id", "idempotencyKey",
|
|
"incidentId", "instruction", "lockOwner", "mode", "projectId",
|
|
"reason", "requestedBy", "singleFlightKey", "source",
|
|
"sourceFingerprint", "traceId", "workItemId"
|
|
)
|
|
$nativeStringTypesValid = [bool](
|
|
$payload -and
|
|
@($nativeStringFields | Where-Object { $payload.$_ -isnot [string] }).Count -eq 0
|
|
)
|
|
$identityValid = [bool](
|
|
-not $parseError -and
|
|
$propertySetMatches -and
|
|
$nativeStringTypesValid -and
|
|
$payload.controlledApply -is [bool] -and $payload.controlledApply -and
|
|
$payload.observation -is [pscustomobject] -and
|
|
[string]$payload.id -eq $fileBaseName -and
|
|
[string]$payload.id -match "^auto-recover-[0-9]{8}-[0-9]{6}-[0-9a-f]{8}$" -and
|
|
[string]$payload.mode -eq "Recover" -and
|
|
[string]$payload.source -eq "agent99-recovery-observer" -and
|
|
[string]$payload.dispatchIdentitySchemaVersion -eq "agent99_controlled_dispatch_identity_v1" -and
|
|
[string]$payload.dispatchRouteId -eq "agent99_local_observer" -and
|
|
[string]$payload.projectId -eq "awoooi" -and
|
|
[string]$payload.requestedBy -eq "Agent99" -and
|
|
[string]$payload.reason -eq "host_unreachable_detected" -and
|
|
[string]$payload.externalId -eq "host_unreachable_detected" -and
|
|
[string]$payload.approvalId -eq "" -and
|
|
[string]$payload.automationRunId -match "^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" -and
|
|
[string]$payload.lockOwner -eq [string]$payload.automationRunId -and
|
|
[string]$payload.executionGeneration -match "^[1-9][0-9]*$" -and
|
|
[string]$payload.canonicalDigest -match "^[0-9a-f]{64}$" -and
|
|
[string]$payload.sourceFingerprint -match "^[0-9a-f]{64}$" -and
|
|
[string]$payload.correlationKey -match "^agent99-controlled:[0-9a-f]{64}$" -and
|
|
[string]$payload.idempotencyKey -eq [string]$payload.correlationKey -and
|
|
[string]$payload.singleFlightKey -match "^agent99-controlled-flight:[0-9a-f]{64}$" -and
|
|
[string]$payload.incidentId -match "^agent99-auto-[0-9a-f]{20}$" -and
|
|
[string]$payload.workItemId -eq "agent99-auto:$($payload.incidentId):host_unreachable_detected" -and
|
|
[string]$payload.traceId -match "^00-[0-9a-f]{32}-[0-9a-f]{16}-0[01]$" -and
|
|
-not [string]::IsNullOrWhiteSpace([string]$payload.instruction) -and
|
|
$createdAtValid -and
|
|
$createdAt -ge $maintenanceTimestamp -and
|
|
$ageMinutes -ge 0 -and
|
|
$ageMinutes -le 120
|
|
)
|
|
if (-not $identityValid) { $allValid = $false }
|
|
$safeItems += [pscustomobject]@{
|
|
file = $file.Name
|
|
id = if ($payload) { [string]$payload.id } else { "" }
|
|
mode = if ($payload) { [string]$payload.mode } else { "" }
|
|
source = if ($payload) { [string]$payload.source } else { "" }
|
|
workItemId = if ($payload) { [string]$payload.workItemId } else { "" }
|
|
createdAt = if ($payload) { [string]$payload.createdAt } else { "" }
|
|
ageMinutes = $ageMinutes
|
|
sha256 = $sha256
|
|
identity = if ($payload -and $sha256) { "$([string]$payload.id)|$sha256" } else { "" }
|
|
parseError = $parseError
|
|
propertySetMatches = $propertySetMatches
|
|
identityValid = $identityValid
|
|
instructionContentReadback = $false
|
|
}
|
|
}
|
|
|
|
$result.items = $safeItems
|
|
if ($allValid) {
|
|
$result.eligibleMaintenanceAutoRecoverCount = $observedCount
|
|
$result.contractValid = $true
|
|
if ($Scope -eq "PromotionReserve") {
|
|
$result.deferredCount = $observedCount
|
|
$result.blockingCount = 0
|
|
$result.reason = "bounded_maintenance_auto_recover_deferred_until_post_promotion"
|
|
} else {
|
|
$result.reason = "full_runtime_requires_maintenance_auto_recover_to_drain"
|
|
}
|
|
} else {
|
|
$result.reason = "maintenance_pending_command_contract_invalid"
|
|
}
|
|
return [pscustomobject]$result
|
|
}
|
|
|
|
function Get-EvidenceReadback {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Pattern,
|
|
[int]$MaxAgeMinutes,
|
|
[bool]$Required,
|
|
[int]$MinimumFreshnessReserveMinutes = 0
|
|
)
|
|
|
|
$evidenceDir = Join-Path $AgentRoot "evidence"
|
|
$candidateSelection = Get-AgentBoundedNewestFiles $evidenceDir $Pattern
|
|
$candidates = @($candidateSelection.files)
|
|
$deferredCandidates = @()
|
|
$file = $null
|
|
foreach ($candidate in $candidates) {
|
|
$candidateDeferred = $false
|
|
try {
|
|
$candidateEvidence = Get-Content -LiteralPath $candidate.FullName -Raw | ConvertFrom-Json
|
|
$candidateSchemaVersion = if ($candidateEvidence.PSObject.Properties["schemaVersion"]) {
|
|
[string]$candidateEvidence.schemaVersion
|
|
} else {
|
|
""
|
|
}
|
|
$candidateDeferred = [bool](
|
|
($candidateEvidence.PSObject.Properties["deferred"] -and [bool]$candidateEvidence.deferred) -or
|
|
$candidateSchemaVersion -in @(
|
|
"agent99_background_mode_deferred_v1",
|
|
"agent99_recovery_transport_deferred_v1"
|
|
)
|
|
)
|
|
} catch {
|
|
# A malformed newest non-deferred candidate must remain visible and fail closed.
|
|
$file = $candidate
|
|
break
|
|
}
|
|
if ($candidateDeferred) {
|
|
$deferredCandidates += $candidate
|
|
continue
|
|
}
|
|
$file = $candidate
|
|
break
|
|
}
|
|
$candidateWindowExhausted = [bool](
|
|
-not $file -and
|
|
$candidateSelection.windowTruncated -and
|
|
$deferredCandidates.Count -eq $candidates.Count
|
|
)
|
|
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
|
$fresh = [bool](
|
|
$file -and
|
|
$ageMinutes -ge 0 -and
|
|
$ageMinutes -le $MaxAgeMinutes
|
|
)
|
|
$freshnessHeadroomMinutes = if ($file) { [math]::Round($MaxAgeMinutes - $ageMinutes, 1) } else { $null }
|
|
$freshnessReserveReady = [bool](
|
|
$fresh -and
|
|
$null -ne $freshnessHeadroomMinutes -and
|
|
$freshnessHeadroomMinutes -ge $MinimumFreshnessReserveMinutes
|
|
)
|
|
$safeSummary = $null
|
|
$parseError = if ($candidateWindowExhausted) { "EvidenceCandidateWindowExhausted" } else { "" }
|
|
$contentHealthy = $true
|
|
if ($file) {
|
|
try {
|
|
$rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
|
|
$hostRows = if ($rawEvidence.PSObject.Properties["hosts"]) { @($rawEvidence.hosts | ForEach-Object {
|
|
[pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping }
|
|
}) } else { @() }
|
|
$vmRows = if ($rawEvidence.PSObject.Properties["vmResults"]) { @($rawEvidence.vmResults | ForEach-Object {
|
|
[pscustomobject]@{
|
|
alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" }
|
|
ok = [bool]$_.ok
|
|
skipped = [bool]$_.skipped
|
|
}
|
|
}) } else { @() }
|
|
$publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() }
|
|
$performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() }
|
|
$telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() }
|
|
$selfHealthPerformance = if (
|
|
$rawEvidence.PSObject.Properties["selfHealth"] -and
|
|
$rawEvidence.selfHealth -and
|
|
$rawEvidence.selfHealth.PSObject.Properties["latestPerformance"]
|
|
) { $rawEvidence.selfHealth.latestPerformance } else { $null }
|
|
$selfHealthPerformanceSeverity = if ($selfHealthPerformance) {
|
|
([string]$selfHealthPerformance.severity).Trim().ToLowerInvariant()
|
|
} else { "" }
|
|
if ($selfHealthPerformanceSeverity -notin @("ok", "warning", "critical")) {
|
|
$selfHealthPerformanceSeverity = "unknown"
|
|
}
|
|
$selfHealthPerformanceRows = if ($selfHealthPerformance) {
|
|
@($selfHealthPerformance.hosts | ForEach-Object {
|
|
$safeHost = ([string]$_.host).Trim()
|
|
if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$") {
|
|
$safeHost = "unknown"
|
|
}
|
|
$safeSeverity = ([string]$_.severity).Trim().ToLowerInvariant()
|
|
if ($safeSeverity -notin @("ok", "warning", "critical", "missing")) {
|
|
$safeSeverity = "unknown"
|
|
}
|
|
$safeReasons = @($_.reasons | ForEach-Object {
|
|
$reason = ([string]$_).Trim().ToLowerInvariant()
|
|
if ($reason -match "^[a-z0-9_]{1,80}$") { $reason }
|
|
})
|
|
[pscustomobject]@{
|
|
host = $safeHost
|
|
severity = $safeSeverity
|
|
reasons = $safeReasons
|
|
}
|
|
})
|
|
} else { @() }
|
|
$safeSummary = if ($Name -eq "sre_alert_inbox") {
|
|
[pscustomobject]@{
|
|
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
|
|
queuedCount = if ($rawEvidence.PSObject.Properties["queuedCount"]) { [int]$rawEvidence.queuedCount } else { 0 }
|
|
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
|
|
duplicateSuppressedCount = if ($rawEvidence.PSObject.Properties["duplicateSuppressedCount"]) { [int]$rawEvidence.duplicateSuppressedCount } else { 0 }
|
|
failedCount = if ($rawEvidence.PSObject.Properties["failedCount"]) { [int]$rawEvidence.failedCount } else { 0 }
|
|
runNow = if ($rawEvidence.PSObject.Properties["runNow"]) { [bool]$rawEvidence.runNow } else { $false }
|
|
controlExitCode = if ($rawEvidence.PSObject.Properties["controlExitCode"]) { [int]$rawEvidence.controlExitCode } else { $null }
|
|
}
|
|
} elseif ($Name -eq "telegram_inbox") {
|
|
[pscustomobject]@{
|
|
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
|
|
reason = if ($rawEvidence.PSObject.Properties["reason"]) { [string]$rawEvidence.reason } else { "" }
|
|
synthetic = if ($rawEvidence.PSObject.Properties["synthetic"]) { [bool]$rawEvidence.synthetic } else { $false }
|
|
updatesSeen = if ($rawEvidence.PSObject.Properties["updatesSeen"]) { [int]$rawEvidence.updatesSeen } else { 0 }
|
|
processedCount = if ($rawEvidence.PSObject.Properties["processedCount"]) { [int]$rawEvidence.processedCount } else { 0 }
|
|
alertedCount = if ($rawEvidence.PSObject.Properties["alertedCount"]) { [int]$rawEvidence.alertedCount } else { 0 }
|
|
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
|
|
}
|
|
} else {
|
|
[pscustomobject]@{
|
|
mode = if ($rawEvidence.PSObject.Properties["mode"]) { [string]$rawEvidence.mode } else { "" }
|
|
controlledApply = if ($rawEvidence.PSObject.Properties["controlledApply"]) { [bool]$rawEvidence.controlledApply } else { $false }
|
|
hostCount = $hostRows.Count
|
|
hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count
|
|
hosts = $hostRows
|
|
vmCount = $vmRows.Count
|
|
vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count
|
|
vms = $vmRows
|
|
harborReady = [bool](
|
|
$rawEvidence.PSObject.Properties["harbor"] -and
|
|
$rawEvidence.harbor -and
|
|
$rawEvidence.harbor.coreHealthy -and
|
|
$rawEvidence.harbor.redisUp -and
|
|
$rawEvidence.harbor.registryHealthy
|
|
)
|
|
awoooiReady = [bool](
|
|
$rawEvidence.PSObject.Properties["awoooi"] -and
|
|
$rawEvidence.awoooi -and
|
|
$rawEvidence.awoooi.deployReady -and
|
|
-not $rawEvidence.awoooi.imagePullFailure -and
|
|
-not $rawEvidence.awoooi.crashFailure
|
|
)
|
|
publicCount = $publicRows.Count
|
|
publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count
|
|
performanceCount = $performanceRows.Count
|
|
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
|
|
selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
|
|
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
|
|
selfHealthPerformanceIssueCount = [int]@($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" }).Count
|
|
selfHealthPerformanceIssues = @($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" })
|
|
backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
|
|
controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
|
|
controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
|
|
telegramAttemptCount = $telegramRows.Count
|
|
telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count
|
|
recovery = if ($rawEvidence.PSObject.Properties["recoverySlo"] -and $rawEvidence.recoverySlo) {
|
|
[pscustomobject]@{
|
|
targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes
|
|
elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds
|
|
completed = [bool]$rawEvidence.recoverySlo.completed
|
|
withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo
|
|
phases = if ($rawEvidence.recoverySlo.PSObject.Properties["phases"]) { @($rawEvidence.recoverySlo.phases | ForEach-Object {
|
|
[pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds }
|
|
}) } else { @() }
|
|
}
|
|
} else { $null }
|
|
recoveryCoordinator = if ($rawEvidence.PSObject.Properties["recoveryCoordinator"] -and $rawEvidence.recoveryCoordinator) {
|
|
[pscustomobject]@{
|
|
verified = [bool]$rawEvidence.recoveryCoordinator.verified
|
|
requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow
|
|
rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed
|
|
scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus
|
|
scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount
|
|
readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent
|
|
failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks)
|
|
}
|
|
} else { $null }
|
|
}
|
|
}
|
|
if ($Name -in @("sre_alert_inbox", "telegram_inbox")) {
|
|
$contentHealthy = [bool]($safeSummary -and $safeSummary.ok)
|
|
}
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
$contentHealthy = $false
|
|
}
|
|
}
|
|
|
|
$parsed = [bool]($file -and -not $parseError)
|
|
$usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and $contentHealthy)
|
|
return [pscustomobject]@{
|
|
name = $Name
|
|
pattern = $Pattern
|
|
required = $Required
|
|
file = if ($file) { $file.Name } else { "" }
|
|
lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" }
|
|
ageMinutes = $ageMinutes
|
|
maxAgeMinutes = $MaxAgeMinutes
|
|
fresh = $fresh
|
|
minimumFreshnessReserveMinutes = $MinimumFreshnessReserveMinutes
|
|
freshnessHeadroomMinutes = $freshnessHeadroomMinutes
|
|
freshnessReserveReady = $freshnessReserveReady
|
|
parsed = $parsed
|
|
contentHealthy = $contentHealthy
|
|
usable = $usable
|
|
healthy = [bool](-not $Required -or $usable)
|
|
parseError = $parseError
|
|
deferredCandidateCount = [int]$deferredCandidates.Count
|
|
latestDeferredTime = if ($deferredCandidates.Count -gt 0) { $deferredCandidates[0].LastWriteTime.ToString("o") } else { "" }
|
|
enumeratedCandidateCount = [int]$candidateSelection.enumeratedCount
|
|
candidateWindowLimit = [int]$candidateSelection.candidateWindow
|
|
candidateWindowTruncated = [bool]$candidateSelection.windowTruncated
|
|
candidateWindowExhausted = $candidateWindowExhausted
|
|
safeSummary = $safeSummary
|
|
}
|
|
}
|
|
|
|
$rootTaskPath = "\"
|
|
$requiredTasks = @(
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Startup-Recovery"; markers = @("agent99-submit-request.ps1", "-Mode Recover", "-Source agent99-startup-recovery") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Heartbeat"; markers = @("agent99-run.ps1", "-Mode Status") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Performance-Guard"; markers = @("agent99-run.ps1", "-Mode Perf", "-ControlledApply") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Control-Loop"; markers = @("agent99-run.ps1", "-Mode ControlTick") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Telegram-Inbox"; markers = @("agent99-telegram-inbox.ps1", "-RunNow") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Inbox"; markers = @("agent99-sre-alert-inbox.ps1", "-RunNow") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-SRE-Alert-Relay"; markers = @("agent99-sre-alert-relay.ps1") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Self-Health"; markers = @("agent99-run.ps1", "-Mode SelfCheck") },
|
|
[pscustomobject]@{ taskPath = $rootTaskPath; name = "Wooo-Agent99-Backup-Health"; markers = @("agent99-run.ps1", "-Mode BackupCheck") }
|
|
)
|
|
$tasks = @($requiredTasks | ForEach-Object {
|
|
Get-TaskReadback `
|
|
-TaskPath $_.taskPath `
|
|
-TaskName $_.name `
|
|
-RequiredArgumentMarkers $_.markers
|
|
})
|
|
$controlLoopMaintenance = Get-AgentControlLoopMaintenanceReadback
|
|
if ($ReadinessScope -eq "PromotionReserve") {
|
|
foreach ($task in $tasks) {
|
|
$runtimeHealthy = [bool]$task.healthy
|
|
$maintenanceFreezeAccepted = [bool](
|
|
$task.name -eq "Wooo-Agent99-Control-Loop" -and
|
|
-not $task.enabled -and
|
|
$controlLoopMaintenance.valid
|
|
)
|
|
$promotionReady = [bool](
|
|
$task.exists -and
|
|
$task.definitionReady -and
|
|
($task.enabled -or $maintenanceFreezeAccepted)
|
|
)
|
|
$task | Add-Member -NotePropertyName runtimeHealthy -NotePropertyValue $runtimeHealthy -Force
|
|
$task | Add-Member -NotePropertyName promotionReady -NotePropertyValue $promotionReady -Force
|
|
$task | Add-Member -NotePropertyName maintenanceFreezeAccepted -NotePropertyValue $maintenanceFreezeAccepted -Force
|
|
$task.healthy = $promotionReady
|
|
}
|
|
}
|
|
|
|
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
|
$manifest = [pscustomobject]@{
|
|
exists = $false
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = ""
|
|
}
|
|
if (Test-Path $manifestPath) {
|
|
try {
|
|
$rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = [string]$rawManifest.sourceRevision
|
|
runtimeMatched = [bool]$rawManifest.runtimeMatched
|
|
fileCount = [int]$rawManifest.fileCount
|
|
mismatchCount = [int]$rawManifest.mismatchCount
|
|
parseError = ""
|
|
}
|
|
} catch {
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
}
|
|
|
|
$listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue)
|
|
$relayListeners = @($listeners | ForEach-Object {
|
|
$process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue
|
|
[pscustomobject]@{
|
|
address = [string]$_.LocalAddress
|
|
port = [int]$_.LocalPort
|
|
processId = [int]$_.OwningProcess
|
|
processName = if ($process) { [string]$process.ProcessName } else { "" }
|
|
}
|
|
})
|
|
|
|
$queues = @(
|
|
(Get-DirectoryReadback "alerts\incoming"),
|
|
(Get-DirectoryReadback "alerts\running"),
|
|
(Get-DirectoryReadback "alerts\failed"),
|
|
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
|
|
(Get-DirectoryReadback "queue\failed")
|
|
)
|
|
$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")
|
|
$evidence = @(
|
|
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $promotionEvidenceRequired $MinimumEvidenceFreshnessReserveMinutes),
|
|
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false 0)
|
|
)
|
|
|
|
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
|
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
|
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy })
|
|
$requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
|
$requiredEvidenceReserveFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.freshnessReserveReady })
|
|
$requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed })
|
|
$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
|
|
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
|
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
|
|
$selfHealthPerformanceSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthPerformanceSeverity } else { "unknown" }
|
|
$selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount } else { 0 }
|
|
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
|
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
|
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
|
$pendingCommandPromotion = Get-AgentPendingCommandPromotionReadback `
|
|
-MaintenanceReadback $controlLoopMaintenance `
|
|
-Scope $ReadinessScope
|
|
$decision = Get-AgentLivePreflightDecision `
|
|
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
|
-Manifest $manifest `
|
|
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
|
|
-ExpectedRuntimeFileCount 20 `
|
|
-TaskFailureCount $taskFailures.Count `
|
|
-RelayListenerCount $relayListeners.Count `
|
|
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
|
-RequiredEvidenceReserveFailureCount $requiredEvidenceReserveFailures.Count `
|
|
-RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count `
|
|
-RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count `
|
|
-SelfHealthSeverity $selfHealthSeverity `
|
|
-SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount `
|
|
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
|
|
-AlertIncomingCount $alertIncoming.count `
|
|
-StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes `
|
|
-PendingCommandBlockingCount $pendingCommandPromotion.blockingCount `
|
|
-PromotionDeferredCommandCount $pendingCommandPromotion.deferredCount
|
|
$blockingReasons = @($decision.blockingReasons)
|
|
$warningReasons = @($decision.warningReasons)
|
|
|
|
$result = [pscustomobject]@{
|
|
schemaVersion = "agent99_live_preflight_v1"
|
|
mode = $Mode
|
|
readinessScope = $ReadinessScope
|
|
minimumEvidenceFreshnessReserveMinutes = $MinimumEvidenceFreshnessReserveMinutes
|
|
observedAt = (Get-Date).ToString("o")
|
|
host = $env:COMPUTERNAME
|
|
lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o")
|
|
secretValueRead = $false
|
|
remoteWritePerformed = $false
|
|
agentRoot = $AgentRoot
|
|
controlLoopMaintenance = $controlLoopMaintenance
|
|
manifest = $manifest
|
|
tasks = $tasks
|
|
relayListeners = $relayListeners
|
|
queues = $queues
|
|
pendingCommandPromotion = $pendingCommandPromotion
|
|
evidence = $evidence
|
|
summary = [pscustomobject]@{
|
|
requiredTaskCount = $requiredTasks.Count
|
|
healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count
|
|
taskFailureCount = $taskFailures.Count
|
|
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
|
usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count
|
|
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
|
requiredEvidenceStaleCount = $requiredEvidenceStale.Count
|
|
requiredEvidenceReserveFailureCount = $requiredEvidenceReserveFailures.Count
|
|
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
|
|
requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
|
|
selfHealthSeverity = $selfHealthSeverity
|
|
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
|
|
selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount
|
|
alertIncomingCount = [int]$alertIncoming.count
|
|
alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes
|
|
alertRunningCount = [int]$alertRunning.count
|
|
pendingCommandCount = [int]$commandQueue.count
|
|
pendingCommandBlockingCount = [int]$pendingCommandPromotion.blockingCount
|
|
promotionDeferredCommandCount = [int]$pendingCommandPromotion.deferredCount
|
|
relayListenerCount = $relayListeners.Count
|
|
preflightGreen = [bool]$decision.deploymentEligible
|
|
deploymentEligible = [bool]$decision.deploymentEligible
|
|
manifestCheckRequired = [bool]$decision.manifestCheckRequired
|
|
manifestIdentityReady = [bool]$decision.manifestIdentityReady
|
|
manifestContentReady = [bool]$decision.manifestContentReady
|
|
blockingReasons = $blockingReasons
|
|
warningCount = [int]$decision.warningCount
|
|
warningReasons = $warningReasons
|
|
}
|
|
}
|
|
|
|
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
|
Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"
|
|
Write-Output "MODE=$Mode"
|
|
Write-Output "READINESS_SCOPE=$ReadinessScope"
|
|
Write-Output "MINIMUM_EVIDENCE_FRESHNESS_RESERVE_MINUTES=$MinimumEvidenceFreshnessReserveMinutes"
|
|
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
|
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
|
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
|
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
|
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
|
Write-Output "ALERT_INCOMING_STALE_COUNT=$($result.summary.alertIncomingStaleCount)"
|
|
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
|
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
|
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
|
Write-Output "WARNING_REASONS=$($warningReasons -join ',')"
|
|
Write-Output "SECRET_VALUE_READ=0"
|
|
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
|
Write-Output "JSON_BEGIN"
|
|
$result | ConvertTo-Json -Depth 8 -Compress
|
|
Write-Output "JSON_END"
|
|
|
|
if ($result.summary.deploymentEligible) {
|
|
exit 0
|
|
}
|
|
exit 2
|