[CmdletBinding()] param( [ValidateSet("Verify")] [string]$Mode = "Verify", [string]$AgentRoot = "C:\Wooo\Agent99" ) $ErrorActionPreference = "Stop" function Get-TaskReadback { param([string]$TaskName) try { $task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop $info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop $lastResult = [int64]$info.LastTaskResult $state = [string]$task.State $healthy = [bool]( $task.Settings.Enabled -and ($lastResult -in @(0, 267009, 267011) -or $state -eq "Running") ) return [pscustomobject]@{ name = $TaskName exists = $true enabled = [bool]$task.Settings.Enabled state = $state lastTaskResult = $lastResult lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" } nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" } healthy = $healthy } } catch { return [pscustomobject]@{ name = $TaskName exists = $false enabled = $false state = "Missing" lastTaskResult = $null lastRunTime = "" nextRunTime = "" healthy = $false } } } function Get-DirectoryReadback { param( [string]$RelativePath, [switch]$RootQueueFilesOnly ) $path = Join-Path $AgentRoot $RelativePath $files = @() if (Test-Path $path) { $files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue) if ($RootQueueFilesOnly) { $files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" }) } $files = @($files | Sort-Object LastWriteTime) } return [pscustomobject]@{ relativePath = $RelativePath exists = [bool](Test-Path $path) count = [int]$files.Count oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" } newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" } staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count } } function Get-EvidenceReadback { param( [string]$Name, [string]$Pattern, [int]$MaxAgeMinutes, [bool]$Required ) $evidenceDir = Join-Path $AgentRoot "evidence" $file = Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending | Select-Object -First 1 $ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null } $fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes) $safeSummary = $null $parseError = "" $contentHealthy = $true if ($file) { try { $rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json $hostRows = if ($rawEvidence.PSObject.Properties["hosts"]) { @($rawEvidence.hosts | ForEach-Object { [pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping } }) } else { @() } $vmRows = if ($rawEvidence.PSObject.Properties["vmResults"]) { @($rawEvidence.vmResults | ForEach-Object { [pscustomobject]@{ alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" } ok = [bool]$_.ok skipped = [bool]$_.skipped } }) } else { @() } $publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() } $performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() } $telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() } $safeSummary = if ($Name -eq "sre_alert_inbox") { [pscustomobject]@{ ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false } queuedCount = if ($rawEvidence.PSObject.Properties["queuedCount"]) { [int]$rawEvidence.queuedCount } else { 0 } ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 } duplicateSuppressedCount = if ($rawEvidence.PSObject.Properties["duplicateSuppressedCount"]) { [int]$rawEvidence.duplicateSuppressedCount } else { 0 } failedCount = if ($rawEvidence.PSObject.Properties["failedCount"]) { [int]$rawEvidence.failedCount } else { 0 } runNow = if ($rawEvidence.PSObject.Properties["runNow"]) { [bool]$rawEvidence.runNow } else { $false } controlExitCode = if ($rawEvidence.PSObject.Properties["controlExitCode"]) { [int]$rawEvidence.controlExitCode } else { $null } } } elseif ($Name -eq "telegram_inbox") { [pscustomobject]@{ ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false } reason = if ($rawEvidence.PSObject.Properties["reason"]) { [string]$rawEvidence.reason } else { "" } synthetic = if ($rawEvidence.PSObject.Properties["synthetic"]) { [bool]$rawEvidence.synthetic } else { $false } updatesSeen = if ($rawEvidence.PSObject.Properties["updatesSeen"]) { [int]$rawEvidence.updatesSeen } else { 0 } processedCount = if ($rawEvidence.PSObject.Properties["processedCount"]) { [int]$rawEvidence.processedCount } else { 0 } alertedCount = if ($rawEvidence.PSObject.Properties["alertedCount"]) { [int]$rawEvidence.alertedCount } else { 0 } ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 } } } else { [pscustomobject]@{ mode = if ($rawEvidence.PSObject.Properties["mode"]) { [string]$rawEvidence.mode } else { "" } controlledApply = if ($rawEvidence.PSObject.Properties["controlledApply"]) { [bool]$rawEvidence.controlledApply } else { $false } hostCount = $hostRows.Count hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count hosts = $hostRows vmCount = $vmRows.Count vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count vms = $vmRows harborReady = [bool]( $rawEvidence.PSObject.Properties["harbor"] -and $rawEvidence.harbor -and $rawEvidence.harbor.coreHealthy -and $rawEvidence.harbor.redisUp -and $rawEvidence.harbor.registryHealthy ) awoooiReady = [bool]( $rawEvidence.PSObject.Properties["awoooi"] -and $rawEvidence.awoooi -and $rawEvidence.awoooi.deployReady -and -not $rawEvidence.awoooi.imagePullFailure -and -not $rawEvidence.awoooi.crashFailure ) publicCount = $publicRows.Count publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count performanceCount = $performanceRows.Count performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" } backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" } controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 } controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 } telegramAttemptCount = $telegramRows.Count telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count recovery = if ($rawEvidence.PSObject.Properties["recoverySlo"] -and $rawEvidence.recoverySlo) { [pscustomobject]@{ targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds completed = [bool]$rawEvidence.recoverySlo.completed withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo phases = if ($rawEvidence.recoverySlo.PSObject.Properties["phases"]) { @($rawEvidence.recoverySlo.phases | ForEach-Object { [pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds } }) } else { @() } } } else { $null } recoveryCoordinator = if ($rawEvidence.PSObject.Properties["recoveryCoordinator"] -and $rawEvidence.recoveryCoordinator) { [pscustomobject]@{ verified = [bool]$rawEvidence.recoveryCoordinator.verified requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks) } } else { $null } } } if ($Name -in @("sre_alert_inbox", "telegram_inbox")) { $contentHealthy = [bool]($safeSummary -and $safeSummary.ok) } } catch { $parseError = $_.Exception.GetType().Name $contentHealthy = $false } } $parsed = [bool]($file -and -not $parseError) $usable = [bool]($fresh -and $parsed -and $contentHealthy) return [pscustomobject]@{ name = $Name pattern = $Pattern required = $Required file = if ($file) { $file.Name } else { "" } lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" } ageMinutes = $ageMinutes maxAgeMinutes = $MaxAgeMinutes fresh = $fresh parsed = $parsed contentHealthy = $contentHealthy usable = $usable healthy = [bool](-not $Required -or $usable) parseError = $parseError safeSummary = $safeSummary } } $requiredTasks = @( "Wooo-Agent99-Startup-Recovery", "Wooo-Agent99-Heartbeat", "Wooo-Agent99-Performance-Guard", "Wooo-Agent99-Control-Loop", "Wooo-Agent99-Telegram-Inbox", "Wooo-Agent99-SRE-Alert-Inbox", "Wooo-Agent99-SRE-Alert-Relay", "Wooo-Agent99-Self-Health", "Wooo-Agent99-Backup-Health" ) $tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ }) $manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json" $manifest = [pscustomobject]@{ exists = $false sourceRevision = "" runtimeMatched = $false fileCount = 0 mismatchCount = -1 parseError = "" } if (Test-Path $manifestPath) { try { $rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json $manifest = [pscustomobject]@{ exists = $true sourceRevision = [string]$rawManifest.sourceRevision runtimeMatched = [bool]$rawManifest.runtimeMatched fileCount = [int]$rawManifest.fileCount mismatchCount = [int]$rawManifest.mismatchCount parseError = "" } } catch { $manifest = [pscustomobject]@{ exists = $true sourceRevision = "" runtimeMatched = $false fileCount = 0 mismatchCount = -1 parseError = $_.Exception.GetType().Name } } } $listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue) $relayListeners = @($listeners | ForEach-Object { $process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue [pscustomobject]@{ address = [string]$_.LocalAddress port = [int]$_.LocalPort processId = [int]$_.OwningProcess processName = if ($process) { [string]$process.ProcessName } else { "" } } }) $queues = @( (Get-DirectoryReadback "alerts\incoming"), (Get-DirectoryReadback "alerts\running"), (Get-DirectoryReadback "alerts\failed"), (Get-DirectoryReadback "queue" -RootQueueFilesOnly), (Get-DirectoryReadback "queue\failed") ) $evidence = @( (Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true), (Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true), (Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true), (Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true), (Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true), (Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true), (Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true), (Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false) ) $operatingSystem = Get-CimInstance Win32_OperatingSystem $taskFailures = @($tasks | Where-Object { -not $_.healthy }) $requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy }) $requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh }) $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed }) $requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy }) $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 $selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" } $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 $alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 $blockingReasons = @() if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" } if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" } if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" } if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" } if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" } if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" } if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" } if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } $result = [pscustomobject]@{ schemaVersion = "agent99_live_preflight_v1" mode = $Mode observedAt = (Get-Date).ToString("o") host = $env:COMPUTERNAME lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o") secretValueRead = $false remoteWritePerformed = $false agentRoot = $AgentRoot manifest = $manifest tasks = $tasks relayListeners = $relayListeners queues = $queues evidence = $evidence summary = [pscustomobject]@{ requiredTaskCount = $requiredTasks.Count healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count taskFailureCount = $taskFailures.Count freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count requiredEvidenceFailureCount = $requiredEvidenceFailures.Count requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count selfHealthSeverity = $selfHealthSeverity alertIncomingCount = [int]$alertIncoming.count alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count relayListenerCount = $relayListeners.Count preflightGreen = [bool]($blockingReasons.Count -eq 0) blockingReasons = $blockingReasons } } Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)" Write-Output "MODE=$Mode" Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)" Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)" Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)" Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)" Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" Write-Output "SECRET_VALUE_READ=0" Write-Output "REMOTE_WRITE_PERFORMED=0" Write-Output "JSON_BEGIN" $result | ConvertTo-Json -Depth 8 -Compress Write-Output "JSON_END" if ($result.summary.preflightGreen) { exit 0 } exit 2