364 lines
17 KiB
PowerShell
364 lines
17 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Verify")]
|
|
[string]$Mode = "Verify",
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Get-TaskReadback {
|
|
param([string]$TaskName)
|
|
|
|
try {
|
|
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
|
$info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop
|
|
$lastResult = [int64]$info.LastTaskResult
|
|
$state = [string]$task.State
|
|
$healthy = [bool](
|
|
$task.Settings.Enabled -and
|
|
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
|
|
)
|
|
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
exists = $true
|
|
enabled = [bool]$task.Settings.Enabled
|
|
state = $state
|
|
lastTaskResult = $lastResult
|
|
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
|
|
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
|
|
healthy = $healthy
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
exists = $false
|
|
enabled = $false
|
|
state = "Missing"
|
|
lastTaskResult = $null
|
|
lastRunTime = ""
|
|
nextRunTime = ""
|
|
healthy = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-DirectoryReadback {
|
|
param(
|
|
[string]$RelativePath,
|
|
[switch]$RootQueueFilesOnly
|
|
)
|
|
|
|
$path = Join-Path $AgentRoot $RelativePath
|
|
$files = @()
|
|
if (Test-Path $path) {
|
|
$files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue)
|
|
if ($RootQueueFilesOnly) {
|
|
$files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
|
|
}
|
|
$files = @($files | Sort-Object LastWriteTime)
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
relativePath = $RelativePath
|
|
exists = [bool](Test-Path $path)
|
|
count = [int]$files.Count
|
|
oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" }
|
|
newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" }
|
|
staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count
|
|
}
|
|
}
|
|
|
|
function Get-EvidenceReadback {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Pattern,
|
|
[int]$MaxAgeMinutes,
|
|
[bool]$Required
|
|
)
|
|
|
|
$evidenceDir = Join-Path $AgentRoot "evidence"
|
|
$file = Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Select-Object -First 1
|
|
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
|
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
|
|
$safeSummary = $null
|
|
$parseError = ""
|
|
$contentHealthy = $true
|
|
if ($file) {
|
|
try {
|
|
$rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
|
|
$hostRows = if ($rawEvidence.PSObject.Properties["hosts"]) { @($rawEvidence.hosts | ForEach-Object {
|
|
[pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping }
|
|
}) } else { @() }
|
|
$vmRows = if ($rawEvidence.PSObject.Properties["vmResults"]) { @($rawEvidence.vmResults | ForEach-Object {
|
|
[pscustomobject]@{
|
|
alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" }
|
|
ok = [bool]$_.ok
|
|
skipped = [bool]$_.skipped
|
|
}
|
|
}) } else { @() }
|
|
$publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() }
|
|
$performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() }
|
|
$telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() }
|
|
$safeSummary = if ($Name -eq "sre_alert_inbox") {
|
|
[pscustomobject]@{
|
|
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
|
|
queuedCount = if ($rawEvidence.PSObject.Properties["queuedCount"]) { [int]$rawEvidence.queuedCount } else { 0 }
|
|
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
|
|
duplicateSuppressedCount = if ($rawEvidence.PSObject.Properties["duplicateSuppressedCount"]) { [int]$rawEvidence.duplicateSuppressedCount } else { 0 }
|
|
failedCount = if ($rawEvidence.PSObject.Properties["failedCount"]) { [int]$rawEvidence.failedCount } else { 0 }
|
|
runNow = if ($rawEvidence.PSObject.Properties["runNow"]) { [bool]$rawEvidence.runNow } else { $false }
|
|
controlExitCode = if ($rawEvidence.PSObject.Properties["controlExitCode"]) { [int]$rawEvidence.controlExitCode } else { $null }
|
|
}
|
|
} elseif ($Name -eq "telegram_inbox") {
|
|
[pscustomobject]@{
|
|
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
|
|
reason = if ($rawEvidence.PSObject.Properties["reason"]) { [string]$rawEvidence.reason } else { "" }
|
|
synthetic = if ($rawEvidence.PSObject.Properties["synthetic"]) { [bool]$rawEvidence.synthetic } else { $false }
|
|
updatesSeen = if ($rawEvidence.PSObject.Properties["updatesSeen"]) { [int]$rawEvidence.updatesSeen } else { 0 }
|
|
processedCount = if ($rawEvidence.PSObject.Properties["processedCount"]) { [int]$rawEvidence.processedCount } else { 0 }
|
|
alertedCount = if ($rawEvidence.PSObject.Properties["alertedCount"]) { [int]$rawEvidence.alertedCount } else { 0 }
|
|
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
|
|
}
|
|
} else {
|
|
[pscustomobject]@{
|
|
mode = if ($rawEvidence.PSObject.Properties["mode"]) { [string]$rawEvidence.mode } else { "" }
|
|
controlledApply = if ($rawEvidence.PSObject.Properties["controlledApply"]) { [bool]$rawEvidence.controlledApply } else { $false }
|
|
hostCount = $hostRows.Count
|
|
hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count
|
|
hosts = $hostRows
|
|
vmCount = $vmRows.Count
|
|
vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count
|
|
vms = $vmRows
|
|
harborReady = [bool](
|
|
$rawEvidence.PSObject.Properties["harbor"] -and
|
|
$rawEvidence.harbor -and
|
|
$rawEvidence.harbor.coreHealthy -and
|
|
$rawEvidence.harbor.redisUp -and
|
|
$rawEvidence.harbor.registryHealthy
|
|
)
|
|
awoooiReady = [bool](
|
|
$rawEvidence.PSObject.Properties["awoooi"] -and
|
|
$rawEvidence.awoooi -and
|
|
$rawEvidence.awoooi.deployReady -and
|
|
-not $rawEvidence.awoooi.imagePullFailure -and
|
|
-not $rawEvidence.awoooi.crashFailure
|
|
)
|
|
publicCount = $publicRows.Count
|
|
publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count
|
|
performanceCount = $performanceRows.Count
|
|
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
|
|
selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
|
|
backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
|
|
controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
|
|
controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
|
|
telegramAttemptCount = $telegramRows.Count
|
|
telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count
|
|
recovery = if ($rawEvidence.PSObject.Properties["recoverySlo"] -and $rawEvidence.recoverySlo) {
|
|
[pscustomobject]@{
|
|
targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes
|
|
elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds
|
|
completed = [bool]$rawEvidence.recoverySlo.completed
|
|
withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo
|
|
phases = if ($rawEvidence.recoverySlo.PSObject.Properties["phases"]) { @($rawEvidence.recoverySlo.phases | ForEach-Object {
|
|
[pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds }
|
|
}) } else { @() }
|
|
}
|
|
} else { $null }
|
|
recoveryCoordinator = if ($rawEvidence.PSObject.Properties["recoveryCoordinator"] -and $rawEvidence.recoveryCoordinator) {
|
|
[pscustomobject]@{
|
|
verified = [bool]$rawEvidence.recoveryCoordinator.verified
|
|
requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow
|
|
rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed
|
|
scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus
|
|
scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount
|
|
readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent
|
|
failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks)
|
|
}
|
|
} else { $null }
|
|
}
|
|
}
|
|
if ($Name -in @("sre_alert_inbox", "telegram_inbox")) {
|
|
$contentHealthy = [bool]($safeSummary -and $safeSummary.ok)
|
|
}
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
$contentHealthy = $false
|
|
}
|
|
}
|
|
|
|
$parsed = [bool]($file -and -not $parseError)
|
|
$usable = [bool]($fresh -and $parsed -and $contentHealthy)
|
|
return [pscustomobject]@{
|
|
name = $Name
|
|
pattern = $Pattern
|
|
required = $Required
|
|
file = if ($file) { $file.Name } else { "" }
|
|
lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" }
|
|
ageMinutes = $ageMinutes
|
|
maxAgeMinutes = $MaxAgeMinutes
|
|
fresh = $fresh
|
|
parsed = $parsed
|
|
contentHealthy = $contentHealthy
|
|
usable = $usable
|
|
healthy = [bool](-not $Required -or $usable)
|
|
parseError = $parseError
|
|
safeSummary = $safeSummary
|
|
}
|
|
}
|
|
|
|
$requiredTasks = @(
|
|
"Wooo-Agent99-Startup-Recovery",
|
|
"Wooo-Agent99-Heartbeat",
|
|
"Wooo-Agent99-Performance-Guard",
|
|
"Wooo-Agent99-Control-Loop",
|
|
"Wooo-Agent99-Telegram-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Relay",
|
|
"Wooo-Agent99-Self-Health",
|
|
"Wooo-Agent99-Backup-Health"
|
|
)
|
|
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ })
|
|
|
|
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
|
$manifest = [pscustomobject]@{
|
|
exists = $false
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = ""
|
|
}
|
|
if (Test-Path $manifestPath) {
|
|
try {
|
|
$rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = [string]$rawManifest.sourceRevision
|
|
runtimeMatched = [bool]$rawManifest.runtimeMatched
|
|
fileCount = [int]$rawManifest.fileCount
|
|
mismatchCount = [int]$rawManifest.mismatchCount
|
|
parseError = ""
|
|
}
|
|
} catch {
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
}
|
|
|
|
$listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue)
|
|
$relayListeners = @($listeners | ForEach-Object {
|
|
$process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue
|
|
[pscustomobject]@{
|
|
address = [string]$_.LocalAddress
|
|
port = [int]$_.LocalPort
|
|
processId = [int]$_.OwningProcess
|
|
processName = if ($process) { [string]$process.ProcessName } else { "" }
|
|
}
|
|
})
|
|
|
|
$queues = @(
|
|
(Get-DirectoryReadback "alerts\incoming"),
|
|
(Get-DirectoryReadback "alerts\running"),
|
|
(Get-DirectoryReadback "alerts\failed"),
|
|
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
|
|
(Get-DirectoryReadback "queue\failed")
|
|
)
|
|
$evidence = @(
|
|
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true),
|
|
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true),
|
|
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true),
|
|
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true),
|
|
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true),
|
|
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true),
|
|
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true),
|
|
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false)
|
|
)
|
|
|
|
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
|
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
|
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy })
|
|
$requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
|
$requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed })
|
|
$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
|
|
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
|
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
|
|
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
|
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
|
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
|
$blockingReasons = @()
|
|
if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" }
|
|
if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
|
|
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
|
|
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
|
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
|
|
if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
|
|
if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
|
|
if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
|
|
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
|
|
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
|
|
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
|
|
|
|
$result = [pscustomobject]@{
|
|
schemaVersion = "agent99_live_preflight_v1"
|
|
mode = $Mode
|
|
observedAt = (Get-Date).ToString("o")
|
|
host = $env:COMPUTERNAME
|
|
lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o")
|
|
secretValueRead = $false
|
|
remoteWritePerformed = $false
|
|
agentRoot = $AgentRoot
|
|
manifest = $manifest
|
|
tasks = $tasks
|
|
relayListeners = $relayListeners
|
|
queues = $queues
|
|
evidence = $evidence
|
|
summary = [pscustomobject]@{
|
|
requiredTaskCount = $requiredTasks.Count
|
|
healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count
|
|
taskFailureCount = $taskFailures.Count
|
|
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
|
usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count
|
|
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
|
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
|
|
requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
|
|
selfHealthSeverity = $selfHealthSeverity
|
|
alertIncomingCount = [int]$alertIncoming.count
|
|
alertRunningCount = [int]$alertRunning.count
|
|
pendingCommandCount = [int]$commandQueue.count
|
|
relayListenerCount = $relayListeners.Count
|
|
preflightGreen = [bool]($blockingReasons.Count -eq 0)
|
|
blockingReasons = $blockingReasons
|
|
}
|
|
}
|
|
|
|
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
|
Write-Output "MODE=$Mode"
|
|
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
|
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
|
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
|
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
|
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
|
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
|
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
|
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
|
Write-Output "SECRET_VALUE_READ=0"
|
|
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
|
Write-Output "JSON_BEGIN"
|
|
$result | ConvertTo-Json -Depth 8 -Compress
|
|
Write-Output "JSON_END"
|
|
|
|
if ($result.summary.preflightGreen) {
|
|
exit 0
|
|
}
|
|
exit 2
|