wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

docs(recovery): record host112 agent closure

Browse Source
This commit is contained in:
ogt
2026-07-11 20:17:24 +08:00
parent 55f075f176
commit 244bad7259
6 changed files with 161 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files

75
scripts/reboot-recovery/agent99-live-preflight.ps1
View File

@@ -89,22 +89,43 @@ function Get-EvidenceReadback {
if ($file) {
try {
$rawEvidence = Get-Content -Path $file.FullName -Raw | ConvertFrom-Json
$hostRows = @($rawEvidence.hosts | ForEach-Object {
$hostRows = if ($rawEvidence.PSObject.Properties["hosts"]) { @($rawEvidence.hosts | ForEach-Object {
[pscustomobject]@{ host = [string]$_.host; reachable = [bool]$_.ping }
})
$vmRows = @($rawEvidence.vmResults | ForEach-Object {
}) } else { @() }
$vmRows = if ($rawEvidence.PSObject.Properties["vmResults"]) { @($rawEvidence.vmResults | ForEach-Object {
[pscustomobject]@{
alias = if ($_.PSObject.Properties["alias"]) { [string]$_.alias } else { "" }
ok = [bool]$_.ok
skipped = [bool]$_.skipped
}
})
$publicRows = @($rawEvidence.public)
$performanceRows = @($rawEvidence.performance)
$telegramRows = @($rawEvidence.telegram)
$safeSummary = [pscustomobject]@{
mode = [string]$rawEvidence.mode
controlledApply = [bool]$rawEvidence.controlledApply
}) } else { @() }
$publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() }
$performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() }
$telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() }
$safeSummary = if ($Name -eq "sre_alert_inbox") {
[pscustomobject]@{
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
queuedCount = if ($rawEvidence.PSObject.Properties["queuedCount"]) { [int]$rawEvidence.queuedCount } else { 0 }
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
duplicateSuppressedCount = if ($rawEvidence.PSObject.Properties["duplicateSuppressedCount"]) { [int]$rawEvidence.duplicateSuppressedCount } else { 0 }
failedCount = if ($rawEvidence.PSObject.Properties["failedCount"]) { [int]$rawEvidence.failedCount } else { 0 }
runNow = if ($rawEvidence.PSObject.Properties["runNow"]) { [bool]$rawEvidence.runNow } else { $false }
controlExitCode = if ($rawEvidence.PSObject.Properties["controlExitCode"]) { [int]$rawEvidence.controlExitCode } else { $null }
}
} elseif ($Name -eq "telegram_inbox") {
[pscustomobject]@{
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
reason = if ($rawEvidence.PSObject.Properties["reason"]) { [string]$rawEvidence.reason } else { "" }
synthetic = if ($rawEvidence.PSObject.Properties["synthetic"]) { [bool]$rawEvidence.synthetic } else { $false }
updatesSeen = if ($rawEvidence.PSObject.Properties["updatesSeen"]) { [int]$rawEvidence.updatesSeen } else { 0 }
processedCount = if ($rawEvidence.PSObject.Properties["processedCount"]) { [int]$rawEvidence.processedCount } else { 0 }
alertedCount = if ($rawEvidence.PSObject.Properties["alertedCount"]) { [int]$rawEvidence.alertedCount } else { 0 }
ignoredCount = if ($rawEvidence.PSObject.Properties["ignoredCount"]) { [int]$rawEvidence.ignoredCount } else { 0 }
}
} else {
[pscustomobject]@{
mode = if ($rawEvidence.PSObject.Properties["mode"]) { [string]$rawEvidence.mode } else { "" }
controlledApply = if ($rawEvidence.PSObject.Properties["controlledApply"]) { [bool]$rawEvidence.controlledApply } else { $false }
hostCount = $hostRows.Count
hostReachableCount = [int]@($hostRows | Where-Object { $_.reachable }).Count
hosts = $hostRows
@@ -112,12 +133,14 @@ function Get-EvidenceReadback {
vmOkCount = [int]@($vmRows | Where-Object { $_.ok -or $_.skipped }).Count
vms = $vmRows
harborReady = [bool](
$rawEvidence.PSObject.Properties["harbor"] -and
$rawEvidence.harbor -and
$rawEvidence.harbor.coreHealthy -and
$rawEvidence.harbor.redisUp -and
$rawEvidence.harbor.registryHealthy
)
awoooiReady = [bool](
$rawEvidence.PSObject.Properties["awoooi"] -and
$rawEvidence.awoooi -and
$rawEvidence.awoooi.deployReady -and
-not $rawEvidence.awoooi.imagePullFailure -and
@@ -127,24 +150,24 @@ function Get-EvidenceReadback {
publicOkCount = [int]@($publicRows | Where-Object { $_.ok }).Count
performanceCount = $performanceRows.Count
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
selfHealthSeverity = if ($rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
backupHealthSeverity = if ($rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
controlProcessedCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
controlPendingCount = if ($rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
telegramAttemptCount = $telegramRows.Count
telegramSentCount = [int]@($telegramRows | Where-Object { $_.sent }).Count
recovery = if ($rawEvidence.recoverySlo) {
recovery = if ($rawEvidence.PSObject.Properties["recoverySlo"] -and $rawEvidence.recoverySlo) {
[pscustomobject]@{
targetMinutes = [int]$rawEvidence.recoverySlo.targetMinutes
elapsedSeconds = [double]$rawEvidence.recoverySlo.elapsedSeconds
completed = [bool]$rawEvidence.recoverySlo.completed
withinSlo = [bool]$rawEvidence.recoverySlo.withinSlo
phases = @($rawEvidence.recoverySlo.phases | ForEach-Object {
phases = if ($rawEvidence.recoverySlo.PSObject.Properties["phases"]) { @($rawEvidence.recoverySlo.phases | ForEach-Object {
[pscustomobject]@{ name = [string]$_.name; status = [string]$_.status; elapsedSeconds = [double]$_.elapsedSeconds }
})
}) } else { @() }
}
} else { $null }
recoveryCoordinator = if ($rawEvidence.recoveryCoordinator) {
recoveryCoordinator = if ($rawEvidence.PSObject.Properties["recoveryCoordinator"] -and $rawEvidence.recoveryCoordinator) {
[pscustomobject]@{
verified = [bool]$rawEvidence.recoveryCoordinator.verified
requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow
@@ -155,12 +178,15 @@ function Get-EvidenceReadback {
failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks)
}
} else { $null }
}
}
} catch {
$parseError = $_.Exception.GetType().Name
}
}
$parsed = [bool]($file -and -not $parseError)
$usable = [bool]($fresh -and $parsed)
return [pscustomobject]@{
name = $Name
pattern = $Pattern
@@ -170,7 +196,9 @@ function Get-EvidenceReadback {
ageMinutes = $ageMinutes
maxAgeMinutes = $MaxAgeMinutes
fresh = $fresh
healthy = [bool](-not $Required -or $fresh)
parsed = $parsed
usable = $usable
healthy = [bool](-not $Required -or $usable)
parseError = $parseError
safeSummary = $safeSummary
}
@@ -252,7 +280,9 @@ $evidence = @(
$operatingSystem = Get-CimInstance Win32_OperatingSystem
$taskFailures = @($tasks | Where-Object { -not $_.healthy })
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh })
$requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.healthy })
$requiredEvidenceStale = @($evidence | Where-Object { $_.required -and -not $_.fresh })
$requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and -not $_.parsed })
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
@@ -264,7 +294,8 @@ if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
@@ -288,7 +319,9 @@ $result = [pscustomobject]@{
healthyTaskCount = [int]@($tasks | Where-Object { $_.healthy }).Count
taskFailureCount = $taskFailures.Count
freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count
usableRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.usable }).Count
requiredEvidenceFailureCount = $requiredEvidenceFailures.Count
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
selfHealthSeverity = $selfHealthSeverity
alertIncomingCount = [int]$alertIncoming.count
alertRunningCount = [int]$alertRunning.count

29
scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
View File

@@ -116,3 +116,32 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None:
assert ".env" not in preflight
assert "auth.json" not in preflight
assert "agent99.config.json" not in preflight
def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> None:
preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")
assert '$parsed = [bool]($file -and -not $parseError)' in preflight
assert '$usable = [bool]($fresh -and $parsed)' in preflight
assert 'healthy = [bool](-not $Required -or $usable)' in preflight
assert '$_.required -and -not $_.healthy' in preflight
assert 'required_evidence_parse_failed' in preflight
assert 'requiredEvidenceParseFailureCount' in preflight
def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> None:
preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")
assert '$Name -eq "sre_alert_inbox"' in preflight
assert '$Name -eq "telegram_inbox"' in preflight
for safe_field in (
"queuedCount",
"duplicateSuppressedCount",
"failedCount",
"updatesSeen",
"processedCount",
"alertedCount",
):
assert safe_field in preflight
for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"):
assert forbidden not in preflight