322 lines
13 KiB
PowerShell
322 lines
13 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[ValidateSet("Verify")]
|
|
[string]$Mode = "Verify",
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
|
|
function Get-TaskReadback {
|
|
param([string]$TaskName)
|
|
|
|
try {
|
|
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
|
$info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop
|
|
$lastResult = [int64]$info.LastTaskResult
|
|
$state = [string]$task.State
|
|
$healthy = [bool](
|
|
$task.Settings.Enabled -and
|
|
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
|
|
)
|
|
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
exists = $true
|
|
enabled = [bool]$task.Settings.Enabled
|
|
state = $state
|
|
lastTaskResult = $lastResult
|
|
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
|
|
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
|
|
healthy = $healthy
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
name = $TaskName
|
|
exists = $false
|
|
enabled = $false
|
|
state = "Missing"
|
|
lastTaskResult = $null
|
|
lastRunTime = ""
|
|
nextRunTime = ""
|
|
healthy = $false
|
|
}
|
|
}
|
|
}
|
|
|
|
function Get-DirectoryReadback {
|
|
param(
|
|
[string]$RelativePath,
|
|
[switch]$RootQueueFilesOnly
|
|
)
|
|
|
|
$path = Join-Path $AgentRoot $RelativePath
|
|
$files = @()
|
|
if (Test-Path $path) {
|
|
$files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue)
|
|
if ($RootQueueFilesOnly) {
|
|
$files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
|
|
}
|
|
$files = @($files | Sort-Object LastWriteTime)
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
relativePath = $RelativePath
|
|
exists = [bool](Test-Path $path)
|
|
count = [int]$files.Count
|
|
oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" }
|
|
newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" }
|
|
staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count
|
|
}
|
|
}
|
|
|
|
function Get-EvidenceReadback {
|
|
param(
|
|
[string]$Name,
|
|
[string]$Pattern,
|
|
[int]$MaxAgeMinutes,
|
|
[bool]$Required
|
|
)
|
|
|
|
$evidenceDir = Join-Path $AgentRoot "evidence"
|
|
$file = Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
|
|
Sort-Object LastWriteTime -Descending |
|
|
Select-Object -First 1
|
|
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
|
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
|
|
$safeSummary = $null
|
|
$parseError = ""
|
|
if ($file) {
|
|
try {
|
|
$rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
|
|
$hostRows = @($rawEvidence.hosts | ForEach-Object {
|
|
[pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping }
|
|
})
|
|
$vmRows = @($rawEvidence.vmResults | ForEach-Object {
|
|
[pscustomobject]@{
|
|
alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" }
|
|
ok = [bool]$_.ok
|
|
skipped = [bool]$_.skipped
|
|
}
|
|
})
|
|
$publicRows = @($rawEvidence.public)
|
|
$performanceRows = @($rawEvidence.performance)
|
|
$telegramRows = @($rawEvidence.telegram)
|
|
$safeSummary = [pscustomobject]@{
|
|
mode = [string]$rawEvidence.mode
|
|
controlledApply = [bool]$rawEvidence.controlledApply
|
|
hostCount = $hostRows.Count
|
|
hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count
|
|
hosts = $hostRows
|
|
vmCount = $vmRows.Count
|
|
vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count
|
|
vms = $vmRows
|
|
harborReady = [bool](
|
|
$rawEvidence.harbor -and
|
|
$rawEvidence.harbor.coreHealthy -and
|
|
$rawEvidence.harbor.redisUp -and
|
|
$rawEvidence.harbor.registryHealthy
|
|
)
|
|
awoooiReady = [bool](
|
|
$rawEvidence.awoooi -and
|
|
$rawEvidence.awoooi.deployReady -and
|
|
-not $rawEvidence.awoooi.imagePullFailure -and
|
|
-not $rawEvidence.awoooi.crashFailure
|
|
)
|
|
publicCount = $publicRows.Count
|
|
publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count
|
|
performanceCount = $performanceRows.Count
|
|
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
|
|
selfHealthSeverity = if ($rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
|
|
backupHealthSeverity = if ($rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
|
|
controlProcessedCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
|
|
controlPendingCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
|
|
telegramAttemptCount = $telegramRows.Count
|
|
telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count
|
|
recovery = if ($rawEvidence.recoverySlo) {
|
|
[pscustomobject]@{
|
|
targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes
|
|
elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds
|
|
completed = [bool]$rawEvidence.recoverySlo.completed
|
|
withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo
|
|
phases = @($rawEvidence.recoverySlo.phases | ForEach-Object {
|
|
[pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds }
|
|
})
|
|
}
|
|
} else { $null }
|
|
recoveryCoordinator = if ($rawEvidence.recoveryCoordinator) {
|
|
[pscustomobject]@{
|
|
verified = [bool]$rawEvidence.recoveryCoordinator.verified
|
|
requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow
|
|
rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed
|
|
scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus
|
|
scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount
|
|
readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent
|
|
failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks)
|
|
}
|
|
} else { $null }
|
|
}
|
|
} catch {
|
|
$parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
|
|
return [pscustomobject]@{
|
|
name = $Name
|
|
pattern = $Pattern
|
|
required = $Required
|
|
file = if ($file) { $file.Name } else { "" }
|
|
lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" }
|
|
ageMinutes = $ageMinutes
|
|
maxAgeMinutes = $MaxAgeMinutes
|
|
fresh = $fresh
|
|
healthy = [bool](-not $Required -or $fresh)
|
|
parseError = $parseError
|
|
safeSummary = $safeSummary
|
|
}
|
|
}
|
|
|
|
$requiredTasks = @(
|
|
"Wooo-Agent99-Startup-Recovery",
|
|
"Wooo-Agent99-Heartbeat",
|
|
"Wooo-Agent99-Performance-Guard",
|
|
"Wooo-Agent99-Control-Loop",
|
|
"Wooo-Agent99-Telegram-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Inbox",
|
|
"Wooo-Agent99-SRE-Alert-Relay",
|
|
"Wooo-Agent99-Self-Health",
|
|
"Wooo-Agent99-Backup-Health"
|
|
)
|
|
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ })
|
|
|
|
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
|
$manifest = [pscustomobject]@{
|
|
exists = $false
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = ""
|
|
}
|
|
if (Test-Path $manifestPath) {
|
|
try {
|
|
$rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = [string]$rawManifest.sourceRevision
|
|
runtimeMatched = [bool]$rawManifest.runtimeMatched
|
|
fileCount = [int]$rawManifest.fileCount
|
|
mismatchCount = [int]$rawManifest.mismatchCount
|
|
parseError = ""
|
|
}
|
|
} catch {
|
|
$manifest = [pscustomobject]@{
|
|
exists = $true
|
|
sourceRevision = ""
|
|
runtimeMatched = $false
|
|
fileCount = 0
|
|
mismatchCount = -1
|
|
parseError = $_.Exception.GetType().Name
|
|
}
|
|
}
|
|
}
|
|
|
|
$listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue)
|
|
$relayListeners = @($listeners | ForEach-Object {
|
|
$process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue
|
|
[pscustomobject]@{
|
|
address = [string]$_.LocalAddress
|
|
port = [int]$_.LocalPort
|
|
processId = [int]$_.OwningProcess
|
|
processName = if ($process) { [string]$process.ProcessName } else { "" }
|
|
}
|
|
})
|
|
|
|
$queues = @(
|
|
(Get-DirectoryReadback "alerts\incoming"),
|
|
(Get-DirectoryReadback "alerts\running"),
|
|
(Get-DirectoryReadback "alerts\failed"),
|
|
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
|
|
(Get-DirectoryReadback "queue\failed")
|
|
)
|
|
$evidence = @(
|
|
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true),
|
|
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true),
|
|
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true),
|
|
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true),
|
|
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true),
|
|
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true),
|
|
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true),
|
|
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false)
|
|
)
|
|
|
|
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
|
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
|
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
|
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
|
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
|
|
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
|
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
|
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
|
$blockingReasons = @()
|
|
if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" }
|
|
if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
|
|
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
|
|
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
|
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
|
|
if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
|
|
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
|
|
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
|
|
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
|
|
|
|
$result = [pscustomobject]@{
|
|
schemaVersion = "agent99_live_preflight_v1"
|
|
mode = $Mode
|
|
observedAt = (Get-Date).ToString("o")
|
|
host = $env:COMPUTERNAME
|
|
lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o")
|
|
secretValueRead = $false
|
|
remoteWritePerformed = $false
|
|
agentRoot = $AgentRoot
|
|
manifest = $manifest
|
|
tasks = $tasks
|
|
relayListeners = $relayListeners
|
|
queues = $queues
|
|
evidence = $evidence
|
|
summary = [pscustomobject]@{
|
|
requiredTaskCount = $requiredTasks.Count
|
|
healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count
|
|
taskFailureCount = $taskFailures.Count
|
|
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
|
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
|
selfHealthSeverity = $selfHealthSeverity
|
|
alertIncomingCount = [int]$alertIncoming.count
|
|
alertRunningCount = [int]$alertRunning.count
|
|
pendingCommandCount = [int]$commandQueue.count
|
|
relayListenerCount = $relayListeners.Count
|
|
preflightGreen = [bool]($blockingReasons.Count -eq 0)
|
|
blockingReasons = $blockingReasons
|
|
}
|
|
}
|
|
|
|
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
|
Write-Output "MODE=$Mode"
|
|
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
|
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
|
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
|
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
|
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
|
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
|
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
|
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
|
Write-Output "SECRET_VALUE_READ=0"
|
|
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
|
Write-Output "JSON_BEGIN"
|
|
$result | ConvertTo-Json -Depth 8 -Compress
|
|
Write-Output "JSON_END"
|
|
|
|
if ($result.summary.preflightGreen) {
|
|
exit 0
|
|
}
|
|
exit 2
|