fix(agent99): gate reboot recovery on full SOP
This commit is contained in:
ogt
305
scripts/reboot-recovery/agent99-live-preflight.ps1
Normal file
305
scripts/reboot-recovery/agent99-live-preflight.ps1
Normal file
@@ -0,0 +1,305 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[ValidateSet("Verify")]
|
||||
[string]$Mode = "Verify",
|
||||
[string]$AgentRoot = "C:\Wooo\Agent99"
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Get-TaskReadback {
|
||||
param([string]$TaskName)
|
||||
|
||||
try {
|
||||
$task = Get-ScheduledTask -TaskName $TaskName -ErrorAction Stop
|
||||
$info = Get-ScheduledTaskInfo -TaskName $TaskName -ErrorAction Stop
|
||||
$lastResult = [int64]$info.LastTaskResult
|
||||
$state = [string]$task.State
|
||||
$healthy = [bool](
|
||||
$task.Settings.Enabled -and
|
||||
($lastResult -in @(0, 267009, 267011) -or $state -eq "Running")
|
||||
)
|
||||
|
||||
return [pscustomobject]@{
|
||||
name = $TaskName
|
||||
exists = $true
|
||||
enabled = [bool]$task.Settings.Enabled
|
||||
state = $state
|
||||
lastTaskResult = $lastResult
|
||||
lastRunTime = if ($info.LastRunTime.Year -gt 2000) { $info.LastRunTime.ToString("o") } else { "" }
|
||||
nextRunTime = if ($info.NextRunTime.Year -gt 2000) { $info.NextRunTime.ToString("o") } else { "" }
|
||||
healthy = $healthy
|
||||
}
|
||||
} catch {
|
||||
return [pscustomobject]@{
|
||||
name = $TaskName
|
||||
exists = $false
|
||||
enabled = $false
|
||||
state = "Missing"
|
||||
lastTaskResult = $null
|
||||
lastRunTime = ""
|
||||
nextRunTime = ""
|
||||
healthy = $false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Get-DirectoryReadback {
|
||||
param(
|
||||
[string]$RelativePath,
|
||||
[switch]$RootQueueFilesOnly
|
||||
)
|
||||
|
||||
$path = Join-Path $AgentRoot $RelativePath
|
||||
$files = @()
|
||||
if (Test-Path $path) {
|
||||
$files = @(Get-ChildItem -Path $path -Filter "*.json" -File -ErrorAction SilentlyContinue)
|
||||
if ($RootQueueFilesOnly) {
|
||||
$files = @($files | Where-Object { $_.Name -notmatch "^(processed|failed|running)-" })
|
||||
}
|
||||
$files = @($files | Sort-Object LastWriteTime)
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
relativePath = $RelativePath
|
||||
exists = [bool](Test-Path $path)
|
||||
count = [int]$files.Count
|
||||
oldest = if ($files.Count -gt 0) { $files[0].LastWriteTime.ToString("o") } else { "" }
|
||||
newest = if ($files.Count -gt 0) { $files[-1].LastWriteTime.ToString("o") } else { "" }
|
||||
staleOverFiveMinutes = [int]@($files | Where-Object { ((Get-Date) - $_.LastWriteTime).TotalMinutes -gt 5 }).Count
|
||||
}
|
||||
}
|
||||
|
||||
function Get-EvidenceReadback {
|
||||
param(
|
||||
[string]$Name,
|
||||
[string]$Pattern,
|
||||
[int]$MaxAgeMinutes,
|
||||
[bool]$Required
|
||||
)
|
||||
|
||||
$evidenceDir = Join-Path $AgentRoot "evidence"
|
||||
$file = Get-ChildItem -Path $evidenceDir -Filter $Pattern -File -ErrorAction SilentlyContinue |
|
||||
Sort-Object LastWriteTime -Descending |
|
||||
Select-Object -First 1
|
||||
$ageMinutes = if ($file) { [math]::Round(((Get-Date) - $file.LastWriteTime).TotalMinutes, 1) } else { $null }
|
||||
$fresh = [bool]($file -and $ageMinutes -le $MaxAgeMinutes)
|
||||
$safeSummary = $null
|
||||
$parseError = ""
|
||||
if ($file) {
|
||||
try {
|
||||
$rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
|
||||
$hostRows = @($rawEvidence.hosts | ForEach-Object {
|
||||
[pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping }
|
||||
})
|
||||
$vmRows = @($rawEvidence.vmResults | ForEach-Object {
|
||||
[pscustomobject]@{
|
||||
alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" }
|
||||
ok = [bool]$_.ok
|
||||
skipped = [bool]$_.skipped
|
||||
}
|
||||
})
|
||||
$publicRows = @($rawEvidence.public)
|
||||
$performanceRows = @($rawEvidence.performance)
|
||||
$telegramRows = @($rawEvidence.telegram)
|
||||
$safeSummary = [pscustomobject]@{
|
||||
mode = [string]$rawEvidence.mode
|
||||
controlledApply = [bool]$rawEvidence.controlledApply
|
||||
hostCount = $hostRows.Count
|
||||
hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count
|
||||
hosts = $hostRows
|
||||
vmCount = $vmRows.Count
|
||||
vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count
|
||||
vms = $vmRows
|
||||
harborReady = [bool](
|
||||
$rawEvidence.harbor -and
|
||||
$rawEvidence.harbor.coreHealthy -and
|
||||
$rawEvidence.harbor.redisUp -and
|
||||
$rawEvidence.harbor.registryHealthy
|
||||
)
|
||||
awoooiReady = [bool](
|
||||
$rawEvidence.awoooi -and
|
||||
$rawEvidence.awoooi.deployReady -and
|
||||
-not $rawEvidence.awoooi.imagePullFailure -and
|
||||
-not $rawEvidence.awoooi.crashFailure
|
||||
)
|
||||
publicCount = $publicRows.Count
|
||||
publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count
|
||||
performanceCount = $performanceRows.Count
|
||||
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
|
||||
selfHealthSeverity = if ($rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
|
||||
backupHealthSeverity = if ($rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
|
||||
controlProcessedCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
|
||||
controlPendingCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
|
||||
telegramAttemptCount = $telegramRows.Count
|
||||
telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count
|
||||
recovery = if ($rawEvidence.recoverySlo) {
|
||||
[pscustomobject]@{
|
||||
targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes
|
||||
elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds
|
||||
completed = [bool]$rawEvidence.recoverySlo.completed
|
||||
withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo
|
||||
phases = @($rawEvidence.recoverySlo.phases | ForEach-Object {
|
||||
[pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds }
|
||||
})
|
||||
}
|
||||
} else { $null }
|
||||
}
|
||||
} catch {
|
||||
$parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
name = $Name
|
||||
pattern = $Pattern
|
||||
required = $Required
|
||||
file = if ($file) { $file.Name } else { "" }
|
||||
lastWriteTime = if ($file) { $file.LastWriteTime.ToString("o") } else { "" }
|
||||
ageMinutes = $ageMinutes
|
||||
maxAgeMinutes = $MaxAgeMinutes
|
||||
fresh = $fresh
|
||||
healthy = [bool](-not $Required -or $fresh)
|
||||
parseError = $parseError
|
||||
safeSummary = $safeSummary
|
||||
}
|
||||
}
|
||||
|
||||
$requiredTasks = @(
|
||||
"Wooo-Agent99-Startup-Recovery",
|
||||
"Wooo-Agent99-Heartbeat",
|
||||
"Wooo-Agent99-Performance-Guard",
|
||||
"Wooo-Agent99-Control-Loop",
|
||||
"Wooo-Agent99-Telegram-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Inbox",
|
||||
"Wooo-Agent99-SRE-Alert-Relay",
|
||||
"Wooo-Agent99-Self-Health",
|
||||
"Wooo-Agent99-Backup-Health"
|
||||
)
|
||||
$tasks = @($requiredTasks | ForEach-Object { Get-TaskReadback $_ })
|
||||
|
||||
$manifestPath = Join-Path $AgentRoot "state\runtime-manifest.json"
|
||||
$manifest = [pscustomobject]@{
|
||||
exists = $false
|
||||
sourceRevision = ""
|
||||
runtimeMatched = $false
|
||||
fileCount = 0
|
||||
mismatchCount = -1
|
||||
parseError = ""
|
||||
}
|
||||
if (Test-Path $manifestPath) {
|
||||
try {
|
||||
$rawManifest = Get-Content -Path $manifestPath -Raw | ConvertFrom-Json
|
||||
$manifest = [pscustomobject]@{
|
||||
exists = $true
|
||||
sourceRevision = [string]$rawManifest.sourceRevision
|
||||
runtimeMatched = [bool]$rawManifest.runtimeMatched
|
||||
fileCount = [int]$rawManifest.fileCount
|
||||
mismatchCount = [int]$rawManifest.mismatchCount
|
||||
parseError = ""
|
||||
}
|
||||
} catch {
|
||||
$manifest = [pscustomobject]@{
|
||||
exists = $true
|
||||
sourceRevision = ""
|
||||
runtimeMatched = $false
|
||||
fileCount = 0
|
||||
mismatchCount = -1
|
||||
parseError = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$listeners = @(Get-NetTCPConnection -LocalPort 8787 -State Listen -ErrorAction SilentlyContinue)
|
||||
$relayListeners = @($listeners | ForEach-Object {
|
||||
$process = Get-Process -Id $_.OwningProcess -ErrorAction SilentlyContinue
|
||||
[pscustomobject]@{
|
||||
address = [string]$_.LocalAddress
|
||||
port = [int]$_.LocalPort
|
||||
processId = [int]$_.OwningProcess
|
||||
processName = if ($process) { [string]$process.ProcessName } else { "" }
|
||||
}
|
||||
})
|
||||
|
||||
$queues = @(
|
||||
(Get-DirectoryReadback "alerts\incoming"),
|
||||
(Get-DirectoryReadback "alerts\running"),
|
||||
(Get-DirectoryReadback "alerts\failed"),
|
||||
(Get-DirectoryReadback "queue" -RootQueueFilesOnly),
|
||||
(Get-DirectoryReadback "queue\failed")
|
||||
)
|
||||
$evidence = @(
|
||||
(Get-EvidenceReadback "self_check" "agent99-SelfCheck-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "status" "agent99-Status-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "control_tick" "agent99-ControlTick-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "sre_alert_inbox" "agent99-SreAlertInbox-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "telegram_inbox" "agent99-TelegramInbox-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "performance" "agent99-Perf-*.json" 20 $true),
|
||||
(Get-EvidenceReadback "backup" "agent99-BackupCheck-*.json" 1440 $true),
|
||||
(Get-EvidenceReadback "recovery" "agent99-Recover-*.json" 10080 $false)
|
||||
)
|
||||
|
||||
$operatingSystem = Get-CimInstance Win32_OperatingSystem
|
||||
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
|
||||
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh })
|
||||
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
||||
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
||||
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
||||
$blockingReasons = @()
|
||||
if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" }
|
||||
if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
|
||||
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
|
||||
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
||||
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
|
||||
if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
|
||||
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_live_preflight_v1"
|
||||
mode = $Mode
|
||||
observedAt = (Get-Date).ToString("o")
|
||||
host = $env:COMPUTERNAME
|
||||
lastBootUpTime = $operatingSystem.LastBootUpTime.ToString("o")
|
||||
secretValueRead = $false
|
||||
remoteWritePerformed = $false
|
||||
agentRoot = $AgentRoot
|
||||
manifest = $manifest
|
||||
tasks = $tasks
|
||||
relayListeners = $relayListeners
|
||||
queues = $queues
|
||||
evidence = $evidence
|
||||
summary = [pscustomobject]@{
|
||||
requiredTaskCount = $requiredTasks.Count
|
||||
healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count
|
||||
taskFailureCount = $taskFailures.Count
|
||||
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
|
||||
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
|
||||
alertIncomingCount = [int]$alertIncoming.count
|
||||
alertRunningCount = [int]$alertRunning.count
|
||||
pendingCommandCount = [int]$commandQueue.count
|
||||
relayListenerCount = $relayListeners.Count
|
||||
preflightGreen = [bool]($blockingReasons.Count -eq 0)
|
||||
blockingReasons = $blockingReasons
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
||||
Write-Output "MODE=$Mode"
|
||||
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
||||
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
||||
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
||||
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
||||
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
||||
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
||||
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
||||
Write-Output "SECRET_VALUE_READ=0"
|
||||
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
||||
Write-Output "JSON_BEGIN"
|
||||
$result | ConvertTo-Json -Depth 8 -Compress
|
||||
Write-Output "JSON_END"
|
||||
|
||||
if ($result.summary.preflightGreen) {
|
||||
exit 0
|
||||
}
|
||||
exit 2
|
||||
@@ -0,0 +1,96 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
|
||||
|
||||
def read(relative: str) -> str:
|
||||
return (ROOT / relative).read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_recover_requires_full_sop_coordinator_before_verified_resolution() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
|
||||
assert "Invoke-AgentRecoveryCoordinatorReadback" in control
|
||||
assert 'New-AgentOutcomeCheck "full_sop_coordinator_verified"' in control
|
||||
assert "-and $coordinatorOk" in control
|
||||
assert 'New-AgentRecoveryPhase "full_sop_scorecard"' in control
|
||||
assert 'schemaVersion = "agent99_recovery_slo_v2"' in control
|
||||
assert 'scope = $recoveryScope' in control
|
||||
assert 'rebootSloClaimed = $rebootSloClaimed' in control
|
||||
|
||||
|
||||
def test_coordinator_covers_all_hosts_and_full_recovery_evidence() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
|
||||
for alias in ("99", "110", "111", "112", "120", "121", "188"):
|
||||
assert f'"{alias}"' in control
|
||||
|
||||
for field in (
|
||||
"SERVICE_GREEN",
|
||||
"PRODUCT_DATA_GREEN",
|
||||
"STOCK_FRESHNESS_STATUS",
|
||||
"BACKUP_CORE_GREEN",
|
||||
"HOST_188_SERVICE_GREEN",
|
||||
"HOST_188_HYGIENE_BLOCKED",
|
||||
"EDGE_FALLBACK_READY",
|
||||
"VMWARE_AUTOSTART_VERIFY_READY",
|
||||
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY",
|
||||
"SCORECARD_FRESH_REBOOT_WINDOW",
|
||||
"SCORECARD_CAN_CLAIM",
|
||||
"SCORECARD_BLOCKER_COUNT",
|
||||
):
|
||||
assert field in control
|
||||
|
||||
|
||||
def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
|
||||
assert 'name = "artifact_fresh_for_recovery"' in control
|
||||
assert 'name = "fresh_reboot_window_observed"' in control
|
||||
assert 'name = "scorecard_can_claim_slo"' in control
|
||||
assert 'name = "scorecard_has_no_blockers"' in control
|
||||
assert "$artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control
|
||||
assert "-not $requiresFreshWindow -or $rebootSloClaimed" in control
|
||||
|
||||
|
||||
def test_runtime_deployer_migrates_recovery_coordinator_defaults() -> None:
|
||||
deployer = read("agent99-deploy.ps1")
|
||||
config = read("agent99.config.99.example.json")
|
||||
|
||||
assert 'Add-DefaultProperty $config "recoveryCoordinator"' in deployer
|
||||
assert 'Add-DefaultProperty $config.recoveryCoordinator "controllerHost" "192.168.0.110"' in deployer
|
||||
assert 'Add-DefaultProperty $config.recoveryCoordinator "requiredHostAliases"' in deployer
|
||||
assert '"recoveryCoordinator"' in config
|
||||
assert '"freshWaitTimeoutSeconds": 420' in config
|
||||
|
||||
|
||||
def test_coordinator_reads_only_bounded_summary_artifacts() -> None:
|
||||
control = read("agent99-control-plane.ps1")
|
||||
coordinator = control.split("function Invoke-AgentRecoveryCoordinatorReadback", 1)[1]
|
||||
coordinator = coordinator.split("function Convert-AgentDouble", 1)[0]
|
||||
|
||||
assert "summary.txt" in coordinator
|
||||
assert "host-probe.txt" in coordinator
|
||||
assert "public-maintenance-edge-fallback.txt" in coordinator
|
||||
assert "windows99-vmware-verify.txt" in coordinator
|
||||
assert "scorecard.json" in coordinator
|
||||
assert "*.log" not in coordinator
|
||||
assert ".env" not in coordinator
|
||||
assert "auth.json" not in coordinator
|
||||
assert "session" not in coordinator.lower()
|
||||
|
||||
|
||||
def test_live_preflight_is_no_secret_and_no_remote_write() -> None:
|
||||
preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")
|
||||
|
||||
assert 'secretValueRead = $false' in preflight
|
||||
assert 'remoteWritePerformed = $false' in preflight
|
||||
assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight
|
||||
assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight
|
||||
assert "runtime-manifest.json" in preflight
|
||||
assert "Get-ScheduledTask" in preflight
|
||||
assert "Get-NetTCPConnection -LocalPort 8787" in preflight
|
||||
assert ".env" not in preflight
|
||||
assert "auth.json" not in preflight
|
||||
assert "agent99.config.json" not in preflight
|
||||
Reference in New Issue
Block a user