feat(automation): harden D037 runtime closure receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s

This commit is contained in:
ogt
2026-07-14 13:43:38 +08:00
parent 3047b8d045
commit a6307e785d
31 changed files with 2616 additions and 369 deletions

View File

@@ -77,6 +77,14 @@ workload_required_connection_budget() {
esac
}
workload_representative_probe_count() {
# Capacity is proven by the PostgreSQL role limit and production HTTP
# concurrency. The isolated pod opens one representative connection for
# role, privilege, and query verification without consuming the live
# workload's entire role budget.
printf '1'
}
workload_deployment() {
case "$1" in
api) printf 'awoooi-api|api' ;;
@@ -222,6 +230,7 @@ preflight_workload_once() {
secret="$2"
connection_limit="$3"
required_connection_budget="$4"
representative_probe_count="$5"
deployment_spec="$(workload_deployment "$workload")"
deployment="${deployment_spec%%|*}"
container="${deployment_spec#*|}"
@@ -262,7 +271,9 @@ spec:
key: DATABASE_URL
- name: EXPECTED_CONNECTION_LIMIT
value: "${connection_limit}"
- name: REQUIRED_CONCURRENT_CONNECTIONS
- name: REPRESENTATIVE_CONNECTION_PROBE_COUNT
value: "${representative_probe_count}"
- name: REQUIRED_CONNECTION_BUDGET
value: "${required_connection_budget}"
- name: WORKLOAD_ID
value: "${workload}"
@@ -287,7 +298,15 @@ spec:
async with engine.connect() as conn:
identity = (
await conn.execute(text("""
SELECT current_user AS role_name, rolconnlimit
SELECT
current_user AS role_name,
rolconnlimit,
(
SELECT count(*)
FROM pg_stat_activity
WHERE usename = current_user
AND pid <> pg_backend_pid()
) AS active_role_connection_count
FROM pg_roles WHERE rolname = current_user
"""))
).mappings().one()
@@ -333,39 +352,38 @@ spec:
AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE'))
)
"""))).scalar_one())
required_concurrency = int(
os.environ["REQUIRED_CONCURRENT_CONNECTIONS"]
)
existing_role_connection_count = int((
await conn.execute(text("""
SELECT count(*)
FROM pg_stat_activity
WHERE usename = current_user
"""))
).scalar_one())
additional_probe_count = max(
0,
required_concurrency
- existing_role_connection_count,
)
representative_probe_count = int(
os.environ["REPRESENTATIVE_CONNECTION_PROBE_COUNT"]
)
if representative_probe_count < 1:
raise RuntimeError("representative_probe_count_invalid")
async def probe_connection() -> None:
async with engine.connect() as probe:
await probe.execute(
text("SELECT pg_sleep(0.25)")
)
async def probe_connection() -> None:
async with engine.connect() as probe:
await probe.execute(text("SELECT pg_sleep(0.25)"))
await asyncio.gather(*(
probe_connection()
for _ in range(additional_probe_count)
))
await asyncio.gather(*(
probe_connection()
for _ in range(representative_probe_count)
))
finally:
await engine.dispose()
expected_limit = int(os.environ["EXPECTED_CONNECTION_LIMIT"])
required_budget = int(os.environ["REQUIRED_CONNECTION_BUDGET"])
observed_limit = int(identity["rolconnlimit"])
active_role_connection_count = int(
identity["active_role_connection_count"]
)
available_connection_headroom = (
observed_limit - active_role_connection_count
)
if observed_limit != expected_limit:
raise RuntimeError("connection_limit_mismatch")
if observed_limit < required_budget:
raise RuntimeError("connection_budget_below_required")
if available_connection_headroom < required_budget:
raise RuntimeError("connection_headroom_below_required")
if table_gap_count or sequence_gap_count:
raise RuntimeError("database_privilege_gap")
print(json.dumps({
@@ -375,12 +393,11 @@ spec:
str(identity["role_name"]).encode("utf-8")
).hexdigest(),
"connection_limit": observed_limit,
"concurrent_connection_probe_count": required_concurrency,
"existing_role_connection_count": (
existing_role_connection_count
),
"additional_connection_probe_count": (
additional_probe_count
"required_connection_budget": required_budget,
"active_role_connection_count": active_role_connection_count,
"available_connection_headroom": available_connection_headroom,
"representative_connection_probe_count": (
representative_probe_count
),
"representative_read_count": 3,
"table_privilege_gap_count": table_gap_count,
@@ -474,6 +491,7 @@ for workload in "${selected_workloads[@]}"; do
secret="$(workload_secret "$workload")"
connection_limit="$(workload_connection_limit "$workload")"
required_connection_budget="$(workload_required_connection_budget "$workload")"
representative_probe_count="$(workload_representative_probe_count "$workload")"
secret_present=false
if kube get secret "$secret" -n "$NAMESPACE" >/dev/null 2>&1; then
secret_present=true
@@ -507,11 +525,11 @@ for workload in "${selected_workloads[@]}"; do
else
ready=false
fi
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget"
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget representative_probe_count=$representative_probe_count"
if [ "$MODE" = "apply" ] || [ "$MODE" = "verify" ]; then
preflight_workload "$workload" "$secret" "$connection_limit" \
"$required_connection_budget"
"$required_connection_budget" "$representative_probe_count"
fi
done

View File

@@ -7,6 +7,89 @@ param(
$ErrorActionPreference = "Stop"
function Get-AgentLivePreflightDecision {
param(
[bool]$AgentRootPresent,
[object]$Manifest,
[int]$ExpectedRuntimeFileCount,
[int]$TaskFailureCount,
[int]$RelayListenerCount,
[int]$RequiredEvidenceStaleCount,
[int]$RequiredEvidenceParseFailureCount,
[int]$RequiredEvidenceContentFailureCount,
[string]$SelfHealthSeverity,
[int]$SelfHealthPerformanceIssueCount,
[int]$StaleAlertExecutionCount,
[int]$AlertIncomingCount,
[int]$PendingCommandCount
)
$blockingReasons = @()
$warningReasons = @()
$manifestExists = [bool]($Manifest -and $Manifest.exists)
$manifestParseReady = [bool](
$manifestExists -and
-not ([string]$Manifest.parseError)
)
$manifestIdentityReady = [bool](
$manifestParseReady -and
[string]$Manifest.sourceRevision -match "^[a-fA-F0-9]{40}$" -and
[int]$Manifest.fileCount -eq $ExpectedRuntimeFileCount
)
$manifestContentReady = [bool](
$manifestParseReady -and
$Manifest.runtimeMatched -eq $true -and
[int]$Manifest.mismatchCount -eq 0
)
if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" }
if (-not $manifestExists) {
$blockingReasons += "runtime_manifest_missing"
} elseif (-not $manifestParseReady) {
$blockingReasons += "runtime_manifest_parse_failed"
} else {
if (-not $manifestIdentityReady) {
$blockingReasons += "runtime_manifest_identity_invalid"
}
if (-not $manifestContentReady) {
$blockingReasons += "runtime_manifest_mismatch"
}
}
if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" }
if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" }
if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
$normalizedSelfHealth = ([string]$SelfHealthSeverity).Trim().ToLowerInvariant()
switch ($normalizedSelfHealth) {
"ok" {}
"warning" {
$warningReasons += "self_health_warning"
if ($SelfHealthPerformanceIssueCount -gt 0) {
$warningReasons += "performance_warning"
}
}
"critical" { $blockingReasons += "self_health_critical" }
default { $blockingReasons += "self_health_unclassified" }
}
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
$blockingReasons += "pending_work_before_reboot"
}
return [pscustomobject]@{
deploymentEligible = [bool]($blockingReasons.Count -eq 0)
blockingReasons = @($blockingReasons)
warningReasons = @($warningReasons)
warningCount = [int]$warningReasons.Count
manifestIdentityReady = $manifestIdentityReady
manifestContentReady = $manifestContentReady
selfHealthClassification = $normalizedSelfHealth
}
}
function Get-TaskReadback {
param([string]$TaskName)
@@ -103,6 +186,38 @@ function Get-EvidenceReadback {
$publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() }
$performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() }
$telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() }
$selfHealthPerformance = if (
$rawEvidence.PSObject.Properties["selfHealth"] -and
$rawEvidence.selfHealth -and
$rawEvidence.selfHealth.PSObject.Properties["latestPerformance"]
) { $rawEvidence.selfHealth.latestPerformance } else { $null }
$selfHealthPerformanceSeverity = if ($selfHealthPerformance) {
([string]$selfHealthPerformance.severity).Trim().ToLowerInvariant()
} else { "" }
if ($selfHealthPerformanceSeverity -notin @("ok", "warning", "critical")) {
$selfHealthPerformanceSeverity = "unknown"
}
$selfHealthPerformanceRows = if ($selfHealthPerformance) {
@($selfHealthPerformance.hosts | ForEach-Object {
$safeHost = ([string]$_.host).Trim()
if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$") {
$safeHost = "unknown"
}
$safeSeverity = ([string]$_.severity).Trim().ToLowerInvariant()
if ($safeSeverity -notin @("ok", "warning", "critical", "missing")) {
$safeSeverity = "unknown"
}
$safeReasons = @($_.reasons | ForEach-Object {
$reason = ([string]$_).Trim().ToLowerInvariant()
if ($reason -match "^[a-z0-9_]{1,80}$") { $reason }
})
[pscustomobject]@{
host = $safeHost
severity = $safeSeverity
reasons = $safeReasons
}
})
} else { @() }
$safeSummary = if ($Name -eq "sre_alert_inbox") {
[pscustomobject]@{
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
@@ -152,6 +267,9 @@ function Get-EvidenceReadback {
performanceCount = $performanceRows.Count
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
selfHealthPerformanceIssueCount = [int]@($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" }).Count
selfHealthPerformanceIssues = @($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" })
backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
@@ -292,21 +410,27 @@ $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $
$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
$selfHealthPerformanceSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthPerformanceSeverity } else { "unknown" }
$selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount } else { 0 }
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
$blockingReasons = @()
if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" }
if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
$decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ExpectedRuntimeFileCount 14 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
-RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count `
-RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count `
-SelfHealthSeverity $selfHealthSeverity `
-SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount `
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
-AlertIncomingCount $alertIncoming.count `
-PendingCommandCount $commandQueue.count
$blockingReasons = @($decision.blockingReasons)
$warningReasons = @($decision.warningReasons)
$result = [pscustomobject]@{
schemaVersion = "agent99_live_preflight_v1"
@@ -332,16 +456,24 @@ $result = [pscustomobject]@{
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
selfHealthSeverity = $selfHealthSeverity
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount
alertIncomingCount = [int]$alertIncoming.count
alertRunningCount = [int]$alertRunning.count
pendingCommandCount = [int]$commandQueue.count
relayListenerCount = $relayListeners.Count
preflightGreen = [bool]($blockingReasons.Count -eq 0)
preflightGreen = [bool]$decision.deploymentEligible
deploymentEligible = [bool]$decision.deploymentEligible
manifestIdentityReady = [bool]$decision.manifestIdentityReady
manifestContentReady = [bool]$decision.manifestContentReady
blockingReasons = $blockingReasons
warningCount = [int]$decision.warningCount
warningReasons = $warningReasons
}
}
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"
Write-Output "MODE=$Mode"
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
@@ -351,13 +483,14 @@ Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
Write-Output "WARNING_REASONS=$($warningReasons -join ',')"
Write-Output "SECRET_VALUE_READ=0"
Write-Output "REMOTE_WRITE_PERFORMED=0"
Write-Output "JSON_BEGIN"
$result | ConvertTo-Json -Depth 8 -Compress
Write-Output "JSON_END"
if ($result.summary.preflightGreen) {
if ($result.summary.deploymentEligible) {
exit 0
}
exit 2

View File

@@ -33,6 +33,21 @@ $FixedRuntimeFiles = @(
"agent99.config.99.example.json"
)
$script:DeployLock = $null
$script:ReceiverRemoteWritePerformed = $false
$script:ReceiverLiveRuntimeWritePerformed = $false
$script:ReceiverLivePromotionAttempted = $false
$script:ReceiverLivePromotionPerformed = $false
$script:ReceiverScheduledTaskRestartPerformed = $false
$script:ReceiverRollbackAttempted = $false
$script:ReceiverRollbackVerified = $false
$script:ReceiverOperationPhase = "envelope_validation"
$script:ReceiverAttemptReceiptPath = ""
$script:ReceiverCanonicalReceiptPath = ""
$script:ReceiverTraceId = ""
$script:ReceiverRunId = ""
$script:ReceiverWorkItemId = ""
$script:ReceiverSourceRevision = ""
$script:ReceiverManifestDigest = ""
function Get-AgentSha256Bytes {
param([byte[]]$Bytes)
@@ -503,9 +518,26 @@ function Restore-AgentBundle {
function Write-AgentRemoteDeployReceipt {
param([string]$Path, [object]$Receipt)
$temporary = "$Path.tmp.$PID"
$Receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $temporary -Encoding UTF8
Move-Item -LiteralPath $temporary -Destination $Path -Force
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
try {
$json = $Receipt | ConvertTo-Json -Depth 12
$null = $json | ConvertFrom-Json
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
$script:ReceiverRemoteWritePerformed = $true
[IO.File]::WriteAllText($temporary, $json, $utf8NoBom)
$storedJson = [IO.File]::ReadAllText($temporary, [Text.Encoding]::UTF8)
$null = $storedJson | ConvertFrom-Json
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
# File.Move is the final operation: on Windows PowerShell/.NET it is atomic
# within this volume and refuses to replace an existing destination.
[IO.File]::Move($temporary, $Path)
} catch {
if (Test-Path -LiteralPath $temporary) {
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue
}
throw
}
}
function Write-AgentResultAndExit {
@@ -576,6 +608,19 @@ try {
$stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20)
$stagePath = Join-Path $DeployRoot "remote-source-$stageToken"
$receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"
$attemptStamp = [DateTime]::UtcNow.ToString(
"yyyyMMdd'T'HHmmssfffffff'Z'",
[Globalization.CultureInfo]::InvariantCulture
)
$attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString('N'))"
$attemptReceiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken-attempt-$attemptToken.json"
$script:ReceiverAttemptReceiptPath = $attemptReceiptPath
$script:ReceiverCanonicalReceiptPath = $receiptPath
$script:ReceiverTraceId = $traceId
$script:ReceiverRunId = $runId
$script:ReceiverWorkItemId = $workItemId
$script:ReceiverSourceRevision = $sourceRevision
$script:ReceiverManifestDigest = $manifestDigest
$currentManifest = Get-AgentSafeRuntimeManifest
if ($mode -eq "check") {
@@ -591,6 +636,9 @@ try {
livePreflight = [pscustomobject]@{ executed = $false; reason = "apply_only_after_validate_only" }
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $false
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
@@ -606,6 +654,8 @@ try {
}) 0
}
$script:ReceiverOperationPhase = "apply_state_initialization"
$script:ReceiverRemoteWritePerformed = $true
New-Item -ItemType Directory -Force -Path $StateDir, $EvidenceDir, $DeployRoot | Out-Null
$lockPath = Join-Path $StateDir "remote-atomic-deploy.lock"
try {
@@ -619,18 +669,29 @@ try {
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
remoteWritePerformed = $false
operationPhase = $script:ReceiverOperationPhase
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
}) 75
}
$canonicalReceiptExists = $false
$prior = $null
$priorParseErrorType = ""
if (Test-Path -LiteralPath $receiptPath -PathType Leaf) {
try { $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json } catch { $prior = $null }
$canonicalReceiptExists = $true
try {
$prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json
if (-not $prior) { $priorParseErrorType = "NullCanonicalReceipt" }
} catch {
$prior = $null
$priorParseErrorType = $_.Exception.GetType().Name
}
}
$stageWrittenThisRun = $false
$script:ReceiverOperationPhase = "staging"
if (-not (Test-Path -LiteralPath $stagePath -PathType Container)) {
$stageBuildPath = "$stagePath.partial.$PID.$([Guid]::NewGuid().ToString('N'))"
New-Item -ItemType Directory -Path $stageBuildPath | Out-Null
@@ -640,7 +701,7 @@ try {
[IO.File]::WriteAllBytes((Join-Path $stageBuildPath "agent99-live-preflight.ps1"), $preflightBytes)
$sourceManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $stageBuildPath "source-manifest.json") -Encoding UTF8
Move-Item -LiteralPath $stageBuildPath -Destination $stagePath
$stageWrittenThisRun = $true
$script:ReceiverRemoteWritePerformed = $true
} else {
foreach ($name in $FixedRuntimeFiles) {
if ((Get-AgentSha256File (Join-Path $stagePath $name)) -ne ([string]($filesByName[$name].sha256)).ToLowerInvariant()) {
@@ -678,6 +739,7 @@ try {
$live.fileCount -eq 14 -and
$live.mismatchCount -eq 0
) {
$script:ReceiverOperationPhase = "idempotent_replay_post_verifier"
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched) {
$originalOperationBoundaries = $prior.operationBoundaries
@@ -686,8 +748,9 @@ try {
$prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force
$prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force
$prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{
remoteWritePerformed = $stageWrittenThisRun
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
@@ -702,7 +765,7 @@ try {
}) -Force
Write-AgentResultAndExit $prior 0
}
Write-AgentResultAndExit ([pscustomobject]@{
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "idempotent_replay_post_verifier_failed_no_promotion"
mode = "apply"
@@ -712,15 +775,66 @@ try {
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
runtimeManifest = $live
livePreflight = $replayPreflight
remoteWritePerformed = $stageWrittenThisRun
durableReceiptWritten = $true
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
nextSafeAction = "repair_reported_live_preflight_blocker_then_retry_same_identity"
}) 2
}
$script:ReceiverOperationPhase = "idempotent_replay_failure_receipt_write"
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
if ($canonicalReceiptExists) {
$canonicalStatus = if (-not $prior) {
"blocked_canonical_receipt_invalid_no_promotion"
} elseif ([string]$prior.status -eq "deployed_verified") {
"blocked_canonical_success_receipt_runtime_drift_no_promotion"
} else {
"blocked_canonical_receipt_reserved_no_promotion"
}
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = $canonicalStatus
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
canonicalReceiptStatus = if ($prior) { [string]$prior.status } else { "unparseable" }
canonicalReceiptParseErrorType = $priorParseErrorType
runtimeManifest = $live
durableReceiptWritten = $true
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_trace_run_work_item_identity"
}
$script:ReceiverOperationPhase = "canonical_receipt_reserved_failure_receipt_write"
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
$script:ReceiverOperationPhase = "validate_only"
$validate = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision -ValidateOnly
if (-not $validate.ok -or -not $validate.payload -or [string]$validate.payload.status -ne "validated") {
$receipt = [pscustomobject]@{
@@ -733,12 +847,22 @@ try {
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
remoteWritePerformed = $true
durableReceiptWritten = $true
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
nextSafeAction = "repair_validate_only_blocker_then_retry_same_identity"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
$script:ReceiverOperationPhase = "validate_only_failure_receipt_write"
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
@@ -773,10 +897,14 @@ try {
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
relayRuntime = $beforeRelayRuntime
remoteWritePerformed = $true
durableReceiptWritten = $true
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $false
livePromotionAttempted = $false
livePromotionPerformed = $false
secretValueRead = $false
privateKeyValueRead = $false
@@ -790,7 +918,8 @@ try {
scheduledTaskModification = $false
nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
$script:ReceiverOperationPhase = "relay_precondition_failure_receipt_write"
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
Write-AgentResultAndExit $receipt 2
}
@@ -803,6 +932,9 @@ try {
$failureType = ""
$backupPath = ""
try {
$script:ReceiverOperationPhase = "live_deploy"
$script:ReceiverLiveRuntimeWritePerformed = $true
$script:ReceiverLivePromotionAttempted = $true
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
$backupPath = [string]$deploy.payload.backupDir
@@ -811,11 +943,15 @@ try {
$failurePhase = "deploy"
$failureType = "deploy_child_failed_or_invalid"
} else {
$script:ReceiverLivePromotionPerformed = $true
$script:ReceiverOperationPhase = "relay_reload"
$script:ReceiverScheduledTaskRestartPerformed = $true
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
if (-not $relayReload.ok) {
$failurePhase = "relay_reload"
$failureType = "relay_reload_failed"
} else {
$script:ReceiverOperationPhase = "independent_post_verifier"
$runtimeManifest = Get-AgentSafeRuntimeManifest
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
$postVerified = [bool](
@@ -845,6 +981,55 @@ try {
$failureType = $_.Exception.GetType().Name
}
$successReceipt = $null
if (-not $failurePhase) {
try {
$script:ReceiverOperationPhase = "success_receipt_construction"
$successReceipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "deployed_verified"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
canonicalReceiptPath = $receiptPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
relayReload = $relayReload
runtimeManifest = $runtimeManifest
livePreflight = $livePreflight
durableReceiptWritten = $true
idempotentReplay = $false
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $true
liveRuntimeWritePerformed = $true
livePromotionAttempted = $true
livePromotionPerformed = $true
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $true
scheduledTaskModification = $false
}
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
}
$script:ReceiverOperationPhase = "success_receipt_write"
Write-AgentRemoteDeployReceipt $receiptPath $successReceipt
} catch {
$failurePhase = $script:ReceiverOperationPhase
$failureType = $_.Exception.GetType().Name
}
}
if ($failurePhase) {
if (-not $backupPath) {
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
@@ -853,6 +1038,8 @@ try {
Select-Object -First 1
if ($newBackup) { $backupPath = $newBackup.FullName }
}
$script:ReceiverOperationPhase = "runtime_rollback"
$script:ReceiverRollbackAttempted = $true
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
if ($relayReload) {
@@ -902,6 +1089,7 @@ try {
$rollbackRelayRestart.after.taskRunning -and
$rollbackRelayRestart.after.listenerCount -gt 0
)
$script:ReceiverRollbackVerified = $rollbackVerified
} catch {
$rollbackVerified = $false
$rollbackVerifierErrorType = $_.Exception.GetType().Name
@@ -911,6 +1099,8 @@ try {
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = if ($failurePhase -eq "deploy") {
if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
} elseif ($failurePhase -in @("success_receipt_construction", "success_receipt_write")) {
if ($rollbackVerified) { "rolled_back_success_receipt_write_failed" } else { "rollback_failed_after_success_receipt_write_failure" }
} else {
if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
}
@@ -926,6 +1116,8 @@ try {
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
failurePhase = $failurePhase
failureType = $failureType
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
relayReload = $relayReload
failedRuntimeManifest = $runtimeManifest
failedLivePreflight = $livePreflight
@@ -935,6 +1127,24 @@ try {
rollbackVerifierErrorType = $rollbackVerifierErrorType
runtimeManifest = $afterManifest
durableReceiptWritten = $true
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
rollbackAttempted = $script:ReceiverRollbackAttempted
rollbackVerified = $rollbackVerified
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted)
scheduledTaskModification = $false
}
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
@@ -957,10 +1167,22 @@ try {
sourceRevision = $sourceRevision
failurePhase = $failurePhase
failureType = $failureType
canonicalReceiptPath = $receiptPath
attemptReceiptPath = $attemptReceiptPath
rollbackVerified = $false
runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified
receiptConstructionErrorType = $_.Exception.GetType().Name
durableReceiptWritten = $true
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
rollbackAttempted = $script:ReceiverRollbackAttempted
rollbackVerified = $false
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
scheduledTaskModification = $false
}
secretValueRead = $false
environmentSecretRead = $false
scheduledTaskModification = $false
@@ -968,7 +1190,7 @@ try {
$rollbackVerified = $false
}
try {
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
} catch {
$receipt.status = "rollback_receipt_write_failed"
$receipt.durableReceiptWritten = $false
@@ -978,45 +1200,29 @@ try {
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
}
$receipt = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "deployed_verified"
mode = "apply"
traceId = $traceId
runId = $runId
workItemId = $workItemId
sourceRevision = $sourceRevision
manifestSha256 = $manifestDigest
stagingPath = $stagePath
backupPath = $backupPath
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
relayReload = $relayReload
runtimeManifest = $runtimeManifest
livePreflight = $livePreflight
idempotentReplay = $false
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $true
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $true
scheduledTaskModification = $false
}
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
}
Write-AgentRemoteDeployReceipt $receiptPath $receipt
Write-AgentResultAndExit $receipt 0
$script:ReceiverOperationPhase = "success_result_output"
Write-AgentResultAndExit $successReceipt 0
} catch {
Write-AgentResultAndExit ([pscustomobject]@{
$outerErrorType = $_.Exception.GetType().Name
$outerFailure = [pscustomobject]@{
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
status = "transport_receiver_failed"
errorType = $_.Exception.GetType().Name
errorType = $outerErrorType
operationPhase = $script:ReceiverOperationPhase
traceId = $script:ReceiverTraceId
runId = $script:ReceiverRunId
workItemId = $script:ReceiverWorkItemId
sourceRevision = $script:ReceiverSourceRevision
manifestSha256 = $script:ReceiverManifestDigest
canonicalReceiptPath = if ($script:ReceiverCanonicalReceiptPath) { $script:ReceiverCanonicalReceiptPath } else { "unavailable" }
attemptReceiptPath = if ($script:ReceiverAttemptReceiptPath) { $script:ReceiverAttemptReceiptPath } else { "unavailable" }
durableReceiptWritten = $false
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
rollbackAttempted = $script:ReceiverRollbackAttempted
rollbackVerified = $script:ReceiverRollbackVerified
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
@@ -1026,9 +1232,40 @@ try {
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $false
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
scheduledTaskModification = $false
}) 70
operationBoundaries = [pscustomobject]@{
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
rollbackAttempted = $script:ReceiverRollbackAttempted
rollbackVerified = $script:ReceiverRollbackVerified
secretValueRead = $false
privateKeyValueRead = $false
tokenValueRead = $false
environmentSecretRead = $false
uiInteraction = $false
vmPowerChange = $false
hostReboot = $false
serviceRestart = $false
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
scheduledTaskModification = $false
}
}
if ($script:ReceiverAttemptReceiptPath) {
try {
$script:ReceiverRemoteWritePerformed = $true
$outerFailure.remoteWritePerformed = $true
$outerFailure.operationBoundaries.remoteWritePerformed = $true
$outerFailure.durableReceiptWritten = $true
Write-AgentRemoteDeployReceipt $script:ReceiverAttemptReceiptPath $outerFailure
} catch {
$outerFailure.durableReceiptWritten = $false
$outerFailure | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force
}
}
Write-AgentResultAndExit $outerFailure 70
} finally {
if ($script:DeployLock) {
try { $script:DeployLock.Dispose() } catch {}

View File

@@ -1,11 +1,10 @@
#!/usr/bin/env bash
# Host 112 guest readiness and bounded recovery executor.
# Version: 2.0
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
# Version: 2.1
# Last modified: 2026-07-14 13:15 (Asia/Taipei)
# Last modified by: Codex
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
# independent process/port verifier without reboot, VM power, firewall, or
# secret access.
# Change: guard every runtime mutation with a fixed durable artifact WAL and
# require independently verified receipt/readback persistence before success.
set -uo pipefail
@@ -174,6 +173,18 @@ signal_readback_status="not_requested"
signal_readback_path="none"
signal_readback_sha256="none"
action_failure_observed=0
ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt"
artifact_transaction_status="not_requested"
artifact_transaction_phase="not_started"
artifact_transaction_started=0
artifact_transaction_blocked=0
artifact_transaction_write_performed=0
artifact_transaction_pair_verified=0
artifact_transaction_prior_pair_verified=0
artifact_expected_receipt_path="none"
artifact_expected_readback_path="none"
artifact_chain_failed=0
artifact_receipt_ready=0
is_uint() {
[[ "$1" =~ ^[0-9]+$ ]]
@@ -274,6 +285,12 @@ start_if_inactive() {
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
return 0
fi
if ! authorize_runtime_mutation "start_${unit//[^A-Za-z0-9._-]/_}"; then
record_action "${action}_artifact_transaction_blocked_failed"
# Keep the executor on its fail-closed evidence path. The recorded
# failure drives a degraded terminal without allowing this mutation.
return 0
fi
# A mutating systemctl invocation may partially change runtime state before
# returning non-zero or timing out. Record the attempt before execution so
# those paths can never be classified as no-write timer observations.
@@ -291,6 +308,10 @@ enable_if_disabled() {
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
return 0
fi
if ! authorize_runtime_mutation "enable_${unit//[^A-Za-z0-9._-]/_}"; then
record_action "${action}_artifact_transaction_blocked_failed"
return 0
fi
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
@@ -507,8 +528,268 @@ timer_slot_for_identity() {
printf '%03d' "$((checksum % timer_ledger_capacity))"
}
reset_durable_receipt_verdict() {
durable_receipt_present=0
durable_receipt_identity_match=0
durable_receipt_boot_match=0
durable_receipt_terminal="none"
}
reset_durable_readback_verdict() {
durable_readback_present=0
durable_readback_identity_match=0
durable_readback_boot_match=0
durable_readback_receipt_link_match=0
durable_readback_terminal_match=0
}
artifact_paths_for_current_identity() {
local receipt_name timer_slot
if [ "$MODE" = "timer_apply" ]; then
timer_slot="$(timer_slot_for_identity)" || return 1
timer_ledger_slot="$timer_slot"
receipt_name="timer-slot-${timer_slot}"
artifact_expected_receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt"
artifact_expected_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt"
else
receipt_name="$(receipt_name_for_identity)"
artifact_expected_receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
artifact_expected_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
fi
case "$artifact_expected_receipt_path" in *[[:space:]]*) return 1 ;; esac
case "$artifact_expected_readback_path" in *[[:space:]]*) return 1 ;; esac
}
controlled_artifact_paths() {
local candidate_receipt="$1" candidate_readback="$2"
case "$candidate_receipt" in
"$STATE_DIR"/receipts/*.txt|"$STATE_DIR"/timer-ledger/*.receipt.txt) ;;
*) return 1 ;;
esac
case "$candidate_readback" in
"$STATE_DIR"/readbacks/*.txt|"$STATE_DIR"/timer-ledger/*.readback.txt) ;;
*) return 1 ;;
esac
}
verify_artifact_pair_paths() {
local expected_trace="$1" expected_run="$2" expected_work_item="$3"
local expected_boot="$4" candidate_receipt="$5" candidate_readback="$6"
local receipt_schema receipt_trace receipt_run receipt_work_item receipt_boot receipt_terminal receipt_digest
local readback_schema
local readback_trace readback_run readback_work_item readback_boot
local readback_receipt_path readback_receipt_sha readback_terminal
controlled_artifact_paths "$candidate_receipt" "$candidate_readback" || return 1
[ -f "$candidate_receipt" ] && [ -f "$candidate_readback" ] || return 1
receipt_schema="$(receipt_token "$candidate_receipt" schema_version)"
receipt_trace="$(receipt_token "$candidate_receipt" trace_id)"
receipt_run="$(receipt_token "$candidate_receipt" run_id)"
receipt_work_item="$(receipt_token "$candidate_receipt" work_item_id)"
receipt_boot="$(receipt_token "$candidate_receipt" boot_id)"
receipt_terminal="$(receipt_token "$candidate_receipt" terminal)"
[ "$receipt_schema" = "host112_wazuh_manager_recovery_receipt_v2" ] \
&& [ -n "$receipt_terminal" ] && [ "$receipt_terminal" != "none" ] \
&& [ "$receipt_trace" = "$expected_trace" ] \
&& [ "$receipt_run" = "$expected_run" ] \
&& [ "$receipt_work_item" = "$expected_work_item" ] \
&& [ "$receipt_boot" = "$expected_boot" ] || return 1
receipt_digest="$(sha256sum "$candidate_receipt" | awk '{print $1}')"
[ -n "$receipt_digest" ] || return 1
readback_schema="$(receipt_token "$candidate_readback" schema_version)"
readback_trace="$(receipt_token "$candidate_readback" trace_id)"
readback_run="$(receipt_token "$candidate_readback" run_id)"
readback_work_item="$(receipt_token "$candidate_readback" work_item_id)"
readback_boot="$(receipt_token "$candidate_readback" boot_id)"
readback_receipt_path="$(receipt_token "$candidate_readback" receipt_path)"
readback_receipt_sha="$(receipt_token "$candidate_readback" receipt_sha256)"
readback_terminal="$(receipt_token "$candidate_readback" durable_receipt_terminal)"
[ "$readback_schema" = "host112_guest_recovery_v2" ] \
&& [ "$readback_trace" = "$expected_trace" ] \
&& [ "$readback_run" = "$expected_run" ] \
&& [ "$readback_work_item" = "$expected_work_item" ] \
&& [ "$readback_boot" = "$expected_boot" ] \
&& [ "$readback_receipt_path" = "$candidate_receipt" ] \
&& [ "$readback_receipt_sha" = "$receipt_digest" ] \
&& [ "$readback_terminal" = "$receipt_terminal" ]
}
write_artifact_transaction_record() {
local requested_status="$1" requested_phase="$2" pair_verified="$3"
local tmp stored_schema stored_status stored_phase stored_trace stored_run
local stored_work_item stored_boot stored_receipt stored_readback stored_pair
artifact_paths_for_current_identity || return 1
install -d -m 0750 "$STATE_DIR" || return 1
tmp="${ARTIFACT_TRANSACTION_PATH}.tmp.$$"
if ! printf '%s\n' \
"schema_version=host112_active_artifact_transaction_v1 updated_at=$(date --iso-8601=seconds) status=$requested_status phase=$requested_phase trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id receipt_path=$artifact_expected_receipt_path readback_path=$artifact_expected_readback_path pair_verified=$pair_verified" \
>"$tmp" \
|| ! chmod 0640 "$tmp" \
|| ! mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"; then
rm -f "$tmp" >/dev/null 2>&1 || true
return 1
fi
stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"
stored_status="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" status)"
stored_phase="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" phase)"
stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)"
stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)"
stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)"
stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)"
stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)"
stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)"
stored_pair="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" pair_verified)"
if [ "$stored_schema" != "host112_active_artifact_transaction_v1" ] \
|| [ "$stored_status" != "$requested_status" ] \
|| [ "$stored_phase" != "$requested_phase" ] \
|| [ "$stored_trace" != "$TRACE_ID" ] \
|| [ "$stored_run" != "$RUN_ID" ] \
|| [ "$stored_work_item" != "$WORK_ITEM_ID" ] \
|| [ "$stored_boot" != "$boot_id" ] \
|| [ "$stored_receipt" != "$artifact_expected_receipt_path" ] \
|| [ "$stored_readback" != "$artifact_expected_readback_path" ] \
|| [ "$stored_pair" != "$pair_verified" ]; then
return 1
fi
artifact_transaction_status="$requested_status"
artifact_transaction_phase="$requested_phase"
artifact_transaction_write_performed=1
state_write_performed=1
return 0
}
inspect_active_artifact_transaction() {
local stored_schema stored_trace stored_run stored_work_item stored_boot
local stored_receipt stored_readback
[ -f "$ARTIFACT_TRANSACTION_PATH" ] || {
artifact_transaction_status="none"
artifact_transaction_phase="no_prior_transaction"
return 0
}
stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"
stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)"
stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)"
stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)"
stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)"
stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)"
stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)"
if [ "$stored_schema" = "host112_active_artifact_transaction_v1" ] \
&& safe_identity "$stored_trace" \
&& safe_identity "$stored_run" \
&& safe_identity "$stored_work_item" \
&& [ -n "$stored_boot" ] \
&& verify_artifact_pair_paths "$stored_trace" "$stored_run" "$stored_work_item" \
"$stored_boot" "$stored_receipt" "$stored_readback"; then
artifact_transaction_prior_pair_verified=1
artifact_transaction_status="prior_pair_verified"
artifact_transaction_phase="prior_transaction_resolved"
return 0
fi
artifact_transaction_blocked=1
artifact_transaction_status="blocked_unresolved_prior_transaction"
artifact_transaction_phase="mutation_denied"
return 1
}
authorize_runtime_mutation() {
local phase="${1:-runtime_mutation}"
[ "$artifact_transaction_started" -eq 0 ] || return 0
[ "$artifact_transaction_blocked" -eq 0 ] || return 1
if write_artifact_transaction_record "pending_artifact_commit" "$phase" 0; then
artifact_transaction_started=1
return 0
fi
artifact_transaction_blocked=1
artifact_transaction_status="wal_write_or_readback_failed"
artifact_transaction_phase="mutation_denied"
return 1
}
mark_artifact_transaction_partial() {
local phase="$1"
[ "$artifact_transaction_started" -eq 1 ] || return 0
if ! write_artifact_transaction_record "partial_pending_public_artifact_commit" "$phase" 0; then
artifact_transaction_status="pending_artifact_commit_partial_update_failed"
artifact_transaction_phase="$phase"
fi
}
verify_current_receipt_from_disk() {
local expected_terminal="${1:-}" stored_trace stored_run stored_work_item stored_boot stored_terminal
reset_durable_receipt_verdict
[ -f "$receipt_path" ] || return 1
durable_receipt_present=1
stored_trace="$(receipt_token "$receipt_path" trace_id)"
stored_run="$(receipt_token "$receipt_path" run_id)"
stored_work_item="$(receipt_token "$receipt_path" work_item_id)"
stored_boot="$(receipt_token "$receipt_path" boot_id)"
stored_terminal="$(receipt_token "$receipt_path" terminal)"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
durable_receipt_terminal="$stored_terminal"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_receipt_identity_match=1
fi
[ "$stored_boot" = "$boot_id" ] && durable_receipt_boot_match=1
[ "$durable_receipt_identity_match" -eq 1 ] \
&& [ "$durable_receipt_boot_match" -eq 1 ] \
&& [ -n "$stored_terminal" ] \
&& { [ -z "$expected_terminal" ] || [ "$stored_terminal" = "$expected_terminal" ]; }
}
verify_current_readback_from_disk() {
local stored_trace stored_run stored_work_item stored_boot
local stored_receipt_path stored_receipt_sha stored_terminal
reset_durable_readback_verdict
[ -f "$durable_readback_path" ] || return 1
durable_readback_present=1
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
stored_run="$(receipt_token "$durable_readback_path" run_id)"
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)"
stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)"
stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
if [ "$stored_trace" = "$TRACE_ID" ] \
&& [ "$stored_run" = "$RUN_ID" ] \
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
durable_readback_identity_match=1
fi
[ "$stored_boot" = "$boot_id" ] && durable_readback_boot_match=1
if [ "$stored_receipt_path" = "$receipt_path" ] \
&& [ "$stored_receipt_sha" = "$receipt_sha256" ]; then
durable_readback_receipt_link_match=1
fi
[ "$stored_terminal" = "$durable_receipt_terminal" ] \
&& durable_readback_terminal_match=1
[ "$durable_readback_identity_match" -eq 1 ] \
&& [ "$durable_readback_boot_match" -eq 1 ] \
&& [ "$durable_readback_receipt_link_match" -eq 1 ] \
&& [ "$durable_readback_terminal_match" -eq 1 ]
}
commit_current_artifact_transaction() {
[ "$artifact_transaction_blocked" -eq 0 ] || return 1
artifact_paths_for_current_identity || return 1
if ! verify_artifact_pair_paths "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$boot_id" \
"$artifact_expected_receipt_path" "$artifact_expected_readback_path"; then
mark_artifact_transaction_partial "independent_pair_verifier_failed"
return 1
fi
artifact_transaction_pair_verified=1
if [ "$artifact_transaction_started" -eq 1 ]; then
write_artifact_transaction_record "committed_verified_pair" \
"independent_pair_verifier_passed" 1 || return 1
else
artifact_transaction_status="existing_pair_verified"
artifact_transaction_phase="independent_pair_verifier_passed"
fi
}
load_durable_receipt() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
reset_durable_receipt_verdict
receipt_name="$(receipt_name_for_identity)"
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
[ -f "$receipt_path" ] || return 1
@@ -539,6 +820,7 @@ load_durable_receipt() {
load_durable_readback() {
local receipt_name stored_trace stored_run stored_work_item stored_boot
local stored_receipt_path stored_receipt_sha stored_terminal
reset_durable_readback_verdict
receipt_name="$(receipt_name_for_identity)"
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
[ -f "$durable_readback_path" ] || {
@@ -618,13 +900,10 @@ write_public_receipt() {
&& mv "$tmp" "$receipt_path"; then
receipt_write_performed=1
receipt_status="written_bounded_timer_slot"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-timer-ledger:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$run_terminal"
return 0
if verify_current_receipt_from_disk "$run_terminal"; then
return 0
fi
elif [ "$MODE" != "timer_apply" ] \
&& printf '%s\n' "$receipt_line" >"$tmp" \
&& chmod 0640 "$tmp" \
@@ -632,13 +911,10 @@ write_public_receipt() {
rm -f "$tmp"
receipt_write_performed=1
receipt_status="written_immutable"
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
receipt_ref="host112-manager-recovery:${receipt_name}"
durable_receipt_present=1
durable_receipt_identity_match=1
durable_receipt_boot_match=1
durable_receipt_terminal="$run_terminal"
return 0
if verify_current_receipt_from_disk "$run_terminal"; then
return 0
fi
fi
rm -f "$tmp" >/dev/null 2>&1 || true
if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then
@@ -720,19 +996,15 @@ write_bounded_timer_mutation_readback() {
return 1
}
tmp="${durable_readback_path}.tmp"
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite stored_readback_kind=bounded_timer_slot durable_readback_path=%s\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \
"$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \
&& chmod 0640 "$tmp" \
&& mv "$tmp" "$durable_readback_path"; then
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
durable_readback_status="written_bounded_timer_slot"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
return 0
if verify_current_readback_from_disk; then
durable_readback_status="written_bounded_timer_slot"
return 0
fi
fi
rm -f "$tmp" >/dev/null 2>&1 || true
durable_readback_status="write_failed"
@@ -806,7 +1078,7 @@ write_immutable_readback() {
fi
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
tmp="${durable_readback_path}.tmp"
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s stored_readback_kind=immutable durable_readback_path=%s\n' \
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \
|| ! chmod 0640 "$tmp" \
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
@@ -815,13 +1087,11 @@ write_immutable_readback() {
return 1
fi
rm -f "$tmp"
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
if ! verify_current_readback_from_disk; then
durable_readback_status="written_immutable_disk_verification_failed"
return 1
fi
durable_readback_status="written_immutable"
durable_readback_present=1
durable_readback_identity_match=1
durable_readback_boot_match=1
durable_readback_receipt_link_match=1
durable_readback_terminal_match=1
# Preserve the historical convenience readback as an explicitly mutable
# pointer. It is never used as execution proof; the immutable path/hash is.
@@ -834,8 +1104,11 @@ write_immutable_readback() {
return 0
fi
rm -f "$latest_tmp" >/dev/null 2>&1 || true
durable_readback_status="immutable_written_latest_pointer_failed"
return 1
# The immutable artifact remains verified even when this auxiliary mutable
# convenience pointer cannot be refreshed. Keep the stable consumer status;
# latest_readback_pointer_written=0 carries the narrower pointer failure.
durable_readback_status="written_immutable"
return 0
}
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
@@ -875,6 +1148,41 @@ if [ "$durable_receipt_present" -eq 1 ]; then
fi
fi
# Every dry-run first proves that no unresolved fixed WAL makes the planned
# apply impossible. When a WAL exists, hold a shared lock on the already-
# existing lock file so an apply cannot race this read-only verifier. Only a
# post-apply dry-run with a current immutable pair performs the extra in-memory
# pair verification; neither path creates or mutates state.
if [ "$MODE" = "dry_run" ]; then
if [ -f "$ARTIFACT_TRANSACTION_PATH" ]; then
if ! { [ -f "$MANAGER_LOCK_FILE" ] \
&& exec 8<"$MANAGER_LOCK_FILE" \
&& flock -s -n 8 \
&& inspect_active_artifact_transaction; }; then
artifact_chain_failed=1
artifact_transaction_blocked=1
if [ "$artifact_transaction_status" = "not_requested" ]; then
artifact_transaction_status="blocked_artifact_transaction_verifier_lock_unavailable"
artifact_transaction_phase="read_only_verifier_denied"
fi
fi
else
inspect_active_artifact_transaction
fi
if [ "$artifact_chain_failed" -eq 0 ] \
&& [ "$durable_receipt_present" -eq 1 ]; then
if [ "$artifact_transaction_prior_pair_verified" -ne 1 ]; then
artifact_chain_failed=1
artifact_transaction_blocked=1
artifact_transaction_status="blocked_fixed_wal_not_verified_for_durable_pair"
artifact_transaction_phase="read_only_pair_verifier_denied"
elif ! commit_current_artifact_transaction; then
artifact_chain_failed=1
artifact_transaction_blocked=1
fi
fi
fi
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
# unset state, even when interruption lands during the general guest repair.
capture_manager_state
@@ -901,16 +1209,18 @@ handle_executor_signal() {
fi
capture_manager_state || true
if [ "$runtime_write_performed" -eq 1 ]; then
if write_public_receipt && write_signal_readback; then
exit 75
if write_public_receipt; then
mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}"
write_signal_readback || true
else
mark_artifact_transaction_partial "receipt_write_failed_after_signal_${signal_name}"
fi
exit 74
exit 1
fi
exit 75
}
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 0 ]; then
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
echo "BLOCKER host112_apply_requires_root" >&2
exit 77
@@ -920,18 +1230,30 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
echo "BLOCKER host112_manager_single_flight_busy" >&2
exit 75
fi
if ! inspect_active_artifact_transaction; then
record_action "artifact_transaction_unresolved_prior_run_failed"
artifact_chain_failed=1
fi
trap 'handle_executor_signal TERM' TERM
trap 'handle_executor_signal INT' INT
trap 'handle_executor_signal HUP' HUP
fi
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 0 ] \
&& [ "$artifact_transaction_blocked" -eq 0 ]; then
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
if [ "$current_default" != "graphical.target" ]; then
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
if authorize_runtime_mutation "set_default_graphical_target"; then
runtime_write_performed=1
state_write_performed=1
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
record_action "set_default_graphical_target"
else
record_action "set_default_graphical_target_failed"
fi
else
record_action "set_default_graphical_target_failed"
record_action "set_default_graphical_target_artifact_transaction_blocked_failed"
fi
fi
@@ -964,18 +1286,23 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
fi
is_uint "$last_retry_epoch" || last_retry_epoch=0
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then
# A marker write can partially mutate state before a non-zero return.
runtime_write_performed=1
state_write_performed=1
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
record_action "retry_wazuh_indexer"
else
record_action "retry_wazuh_indexer_failed"
fi
else
record_action "retry_wazuh_indexer_failed"
record_action "wazuh_indexer_marker_write_failed"
fi
else
record_action "wazuh_indexer_marker_write_failed_no_write"
record_action "wazuh_indexer_artifact_transaction_blocked_failed"
fi
else
record_action "wazuh_indexer_retry_cooldown"
@@ -988,11 +1315,15 @@ now_epoch="$(date +%s)"
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
if [ "$MODE" = "dry_run" ]; then
case "$manager_preflight_status" in
ready) manager_terminal="dry_run_ready" ;;
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
esac
if [ "$artifact_chain_failed" -eq 1 ]; then
manager_terminal="dry_run_artifact_transaction_blocked"
else
case "$manager_preflight_status" in
ready) manager_terminal="dry_run_ready" ;;
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
esac
fi
elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
&& [ "$immutable_replay" -eq 1 ]; then
manager_terminal="immutable_receipt_replay"
@@ -1002,48 +1333,52 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
manager_terminal="idempotent_already_verified_healthy"
;;
ready)
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then
# The cooldown marker is a durable runtime state mutation even if the
# following systemctl operation fails. Account for it immediately so
# no failure path can emit a false no-write receipt.
# marker writer returns non-zero. Account for the attempt immediately.
runtime_write_performed=1
state_write_performed=1
manager_marker_write_performed=1
manager_attempt_count=1
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
record_action "reset_failed_wazuh_manager"
runtime_write_performed=1
manager_runtime_write_performed=1
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
manager_start_exit=0
else
manager_start_exit=$?
fi
if wait_for_manager_verifier; then
if [ "$manager_start_exit" = "0" ]; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
manager_marker_write_performed=1
manager_attempt_count=1
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
record_action "reset_failed_wazuh_manager"
runtime_write_performed=1
manager_runtime_write_performed=1
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
manager_start_exit=0
else
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
manager_start_exit=$?
fi
if wait_for_manager_verifier; then
if [ "$manager_start_exit" = "0" ]; then
record_action "start_wazuh_manager_verified"
manager_terminal="verified_healthy"
else
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
fi
else
record_action "start_wazuh_manager_verifier_failed"
rollback_manager_to_inactive
if [ "$rollback_verified" -eq 1 ]; then
manager_terminal="rolled_back_verifier_failed"
else
manager_terminal="rollback_failed_degraded"
fi
fi
else
record_action "start_wazuh_manager_verifier_failed"
rollback_manager_to_inactive
if [ "$rollback_verified" -eq 1 ]; then
manager_terminal="rolled_back_verifier_failed"
else
manager_terminal="rollback_failed_degraded"
fi
manager_start_exit="reset_failed_no_start"
record_action "reset_failed_wazuh_manager_failed_no_start"
manager_terminal="no_start_reset_failed"
fi
else
manager_start_exit="reset_failed_no_start"
record_action "reset_failed_wazuh_manager_failed_no_start"
manager_terminal="no_start_reset_failed"
record_action "wazuh_manager_marker_write_failed"
manager_terminal="cooldown_marker_write_failed_degraded"
fi
else
record_action "wazuh_manager_marker_write_failed_no_write"
manager_terminal="no_write_cooldown_marker_write_failed"
record_action "wazuh_manager_artifact_transaction_blocked_failed"
manager_terminal="blocked_unresolved_artifact_transaction"
fi
;;
*)
@@ -1159,13 +1494,22 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "$MODE" = "timer_apply" ]; then
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
fi
if ! write_public_receipt; then
if ! authorize_runtime_mutation "public_artifact_commit"; then
artifact_chain_failed=1
run_terminal="${run_terminal}_artifact_transaction_blocked"
elif write_public_receipt; then
artifact_receipt_ready=1
else
artifact_chain_failed=1
mark_artifact_transaction_partial "receipt_write_or_disk_verification_failed"
run_terminal="${run_terminal}_receipt_failed"
fi
else
receipt_status="timer_no_runtime_mutation_aggregated"
timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"
fi
elif [ "$durable_receipt_present" -eq 1 ]; then
artifact_receipt_ready=1
fi
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if ! write_bounded_timer_observation; then
@@ -1178,7 +1522,7 @@ fi
trace_output="${TRACE_ID:-none}"
run_output="${RUN_ID:-none}"
work_item_output="${WORK_ITEM_ID:-none}"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed artifact_transaction_status=$artifact_transaction_status artifact_transaction_phase=$artifact_transaction_phase artifact_transaction_path=$ARTIFACT_TRANSACTION_PATH artifact_transaction_started=$artifact_transaction_started artifact_transaction_blocked=$artifact_transaction_blocked artifact_transaction_write_performed=$artifact_transaction_write_performed artifact_transaction_pair_verified=$artifact_transaction_pair_verified artifact_transaction_prior_pair_verified=$artifact_transaction_prior_pair_verified receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
canonical_readback_base() {
printf '%s\n' "$readback" | awk '
@@ -1188,6 +1532,9 @@ canonical_readback_base() {
if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) {
continue
}
if ($i ~ /^artifact_transaction_/) {
continue
}
printf "%s%s", separator, $i
separator = " "
}
@@ -1197,8 +1544,12 @@ canonical_readback_base() {
}
emit_readback() {
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
"$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \
printf '%s artifact_transaction_status=%s artifact_transaction_phase=%s artifact_transaction_path=%s artifact_transaction_started=%s artifact_transaction_blocked=%s artifact_transaction_write_performed=%s artifact_transaction_pair_verified=%s artifact_transaction_prior_pair_verified=%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
"$(canonical_readback_base)" "$artifact_transaction_status" "$artifact_transaction_phase" \
"$ARTIFACT_TRANSACTION_PATH" "$artifact_transaction_started" \
"$artifact_transaction_blocked" "$artifact_transaction_write_performed" \
"$artifact_transaction_pair_verified" "$artifact_transaction_prior_pair_verified" \
"$durable_readback_status" "$durable_readback_path" \
"$durable_readback_sha256" "$durable_readback_present" \
"$durable_readback_identity_match" "$durable_readback_boot_match" \
"$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \
@@ -1206,24 +1557,39 @@ emit_readback() {
}
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if [ "$artifact_chain_failed" -eq 1 ]; then
emit_readback
exit 1
elif [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
if [ "$timer_observation_failed" -eq 1 ]; then
emit_readback
exit 74
exit 1
fi
elif [ "$MODE" = "timer_apply" ]; then
if ! write_bounded_timer_mutation_readback; then
mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed"
emit_readback
exit 74
exit 1
fi
elif ! write_immutable_readback; then
mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed"
emit_readback
exit 74
exit 1
fi
if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then
if ! commit_current_artifact_transaction; then
mark_artifact_transaction_partial "artifact_pair_commit_failed"
emit_readback
exit 1
fi
fi
fi
emit_readback
if [ "$MODE" = "dry_run" ]; then
if [ "$artifact_chain_failed" -eq 1 ]; then
exit 1
fi
case "$manager_preflight_status" in
ready|already_verified_healthy) exit 0 ;;
*) exit 1 ;;
@@ -1237,6 +1603,8 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
fi
[ "$guest_ready" -eq 1 ] \
&& [ "$receipt_status" != "write_failed" ] \
&& { [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ] \
|| [ "$artifact_transaction_pair_verified" -eq 1 ]; } \
&& { [ "$successful_terminal" = "verified_healthy" ] \
|| [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; }
exit $?

View File

@@ -1,10 +1,10 @@
#!/usr/bin/env bash
# Host 112 guest recovery installer.
# Version: 2.1
# Last modified: 2026-07-14 12:10 (Asia/Taipei)
# Last modified: 2026-07-14 13:15 (Asia/Taipei)
# Last modified by: Codex
# Change: make installation transactional, verify the Windows99 direct route
# and exact sudo policy, and keep installation separate from recovery runs.
# Change: make timer shutdown state-aware on clean installs while preserving
# transactional rollback, exact sudo policy, and recovery-run separation.
set -Eeuo pipefail
@@ -66,6 +66,9 @@ exact correlated apply/dry-run sudo rules, the host-110 forced readback key,
and the existing Windows99 direct Agent99 authorization. It starts only the
timer after all static gates pass; it never synchronously starts the recovery
oneshot, reboots the host, or changes VM power.
HOST112_CONTROL_PUBLIC_KEY_PATH must point to an explicitly pre-staged public
key file. The installer never discovers, reads, copies, or derives a private key.
USAGE
}
@@ -487,7 +490,9 @@ static_source_preflight() {
for tool in bash cksum cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do
command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; }
done
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do
[ -f "$CONTROL_PUBLIC_KEY_PATH" ] \
|| { echo "BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH" >&2; return 1; }
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do
[ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; return 1; }
done
bash -n "$READINESS_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"
@@ -565,6 +570,25 @@ wait_for_recovery_service_idle() {
done
}
stop_recovery_timer_if_running() {
local state
state="$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
case "${state:-unknown}" in
active|activating|reloading|deactivating)
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1
;;
inactive|failed|not-found|unknown)
# A clean install has no timer unit yet. Treat that as an idempotent
# no-op instead of asking systemd to stop a unit which does not exist.
return 0
;;
*)
echo "BLOCKER unexpected_recovery_timer_state=${state}" >&2
return 1
;;
esac
}
resolve_target_user
if [ "$MODE" = "check" ]; then
@@ -612,7 +636,7 @@ TRANSACTION_ACTIVE=1
trap 'handle_apply_failure $?' ERR
trap 'handle_apply_failure 130' INT TERM
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1
stop_recovery_timer_if_running
wait_for_recovery_service_idle
install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET"

View File

@@ -0,0 +1,203 @@
from __future__ import annotations
import json
import shutil
import subprocess
from copy import deepcopy
from pathlib import Path
from typing import Any
import pytest
ROOT = Path(__file__).resolve().parents[3]
PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
def _function_source() -> str:
source = PREFLIGHT.read_text(encoding="utf-8")
start = source.index("function Get-AgentLivePreflightDecision")
end = source.index("function Get-TaskReadback", start)
return source[start:end]
def _base_case() -> dict[str, Any]:
return {
"agentRootPresent": True,
"manifest": {
"exists": True,
"sourceRevision": "a" * 40,
"runtimeMatched": True,
"fileCount": 14,
"mismatchCount": 0,
"parseError": "",
},
"taskFailureCount": 0,
"relayListenerCount": 1,
"requiredEvidenceStaleCount": 0,
"requiredEvidenceParseFailureCount": 0,
"requiredEvidenceContentFailureCount": 0,
"selfHealthSeverity": "ok",
"selfHealthPerformanceIssueCount": 0,
"staleAlertExecutionCount": 0,
"alertIncomingCount": 0,
"pendingCommandCount": 0,
}
def _with_overrides(**overrides: Any) -> dict[str, Any]:
case = deepcopy(_base_case())
manifest_overrides = overrides.pop("manifest", None)
case.update(overrides)
if manifest_overrides:
case["manifest"].update(manifest_overrides)
return case
def _as_list(value: Any) -> list[str]:
if value is None:
return []
if isinstance(value, list):
return [str(item) for item in value]
return [str(value)]
def _run_decision(powershell: str, case: dict[str, Any]) -> dict[str, Any]:
fixture = json.dumps(case, separators=(",", ":"))
command = f"""
{_function_source()}
$case = @'
{fixture}
'@ | ConvertFrom-Json
$parameters = @{{
AgentRootPresent = [bool]$case.agentRootPresent
Manifest = $case.manifest
ExpectedRuntimeFileCount = 14
TaskFailureCount = [int]$case.taskFailureCount
RelayListenerCount = [int]$case.relayListenerCount
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount
RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount
SelfHealthSeverity = [string]$case.selfHealthSeverity
SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
AlertIncomingCount = [int]$case.alertIncomingCount
PendingCommandCount = [int]$case.pendingCommandCount
}}
Get-AgentLivePreflightDecision @parameters |
ConvertTo-Json -Depth 6 -Compress
"""
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = next(
line for line in reversed(result.stdout.splitlines()) if line.startswith("{")
)
return json.loads(payload)
def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> None:
source = PREFLIGHT.read_text(encoding="utf-8")
decision = _function_source()
assert '$warningReasons += "self_health_warning"' in decision
assert '$warningReasons += "performance_warning"' in decision
assert '"critical" { $blockingReasons += "self_health_critical" }' in decision
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
assert '$blockingReasons += "runtime_manifest_mismatch"' in decision
assert '$blockingReasons += "scheduled_task_unhealthy"' in decision
assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
assert "warningReasons = $warningReasons" in source
assert "selfHealthPerformanceIssues = @(" in source
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
assert 'if ($reason -match "^[a-z0-9_]{1,80}$")' in source
assert (
"[int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount"
in source
)
assert 'if ($result.summary.deploymentEligible)' in source
assert 'if ($selfHealthSeverity -ne "ok")' not in source
def test_decision_behavior_when_powershell_is_available() -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
cases = (
(_base_case(), True, (), ()),
(
_with_overrides(
selfHealthSeverity="warning",
selfHealthPerformanceIssueCount=1,
),
True,
(),
("self_health_warning", "performance_warning"),
),
(
_with_overrides(selfHealthSeverity="critical"),
False,
("self_health_critical",),
(),
),
(
_with_overrides(taskFailureCount=1),
False,
("scheduled_task_unhealthy",),
(),
),
(
_with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}),
False,
("runtime_manifest_mismatch",),
(),
),
(
_with_overrides(manifest={"sourceRevision": "not-a-full-sha"}),
False,
("runtime_manifest_identity_invalid",),
(),
),
(
_with_overrides(relayListenerCount=0),
False,
("sre_alert_relay_not_listening",),
(),
),
)
for case, expected_eligible, expected_blockers, expected_warnings in cases:
result = _run_decision(powershell, case)
assert result["deploymentEligible"] is expected_eligible
assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers)
assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings)
preflight_path = str(PREFLIGHT).replace("'", "''")
parser_command = (
"$tokens=$null; $errors=$null; "
"[System.Management.Automation.Language.Parser]::ParseFile("
f"'{preflight_path}', [ref]$tokens, [ref]$errors) | Out-Null; "
"if ($errors.Count -gt 0) { "
"$errors | ForEach-Object { $_.Message }; exit 1 }"
)
parsed = subprocess.run(
[
powershell,
"-NoProfile",
"-NonInteractive",
"-Command",
parser_command,
],
check=False,
capture_output=True,
text=True,
)
assert parsed.returncode == 0, parsed.stdout + parsed.stderr

View File

@@ -109,7 +109,15 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None:
assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight
assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight
assert 'Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"' in preflight
assert 'if ($selfHealthSeverity -ne "ok")' in preflight
assert "function Get-AgentLivePreflightDecision" in preflight
assert '"warning" {' in preflight
assert '$warningReasons += "self_health_warning"' in preflight
assert '$warningReasons += "performance_warning"' in preflight
assert '"critical" { $blockingReasons += "self_health_critical" }' in preflight
assert 'default { $blockingReasons += "self_health_unclassified" }' in preflight
assert 'if ($selfHealthSeverity -ne "ok")' not in preflight
assert 'Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"' in preflight
assert 'Write-Output "WARNING_REASONS=$($warningReasons -join \',\')"' in preflight
assert "runtime-manifest.json" in preflight
assert "Get-ScheduledTask" in preflight
assert "Get-NetTCPConnection -LocalPort 8787" in preflight
@@ -143,6 +151,9 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() ->
"updatesSeen",
"processedCount",
"alertedCount",
"selfHealthPerformanceSeverity",
"selfHealthPerformanceIssueCount",
"selfHealthPerformanceIssues",
):
assert safe_field in preflight
for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"):

View File

@@ -7,6 +7,8 @@ import shutil
import subprocess
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
SENDER = ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
RECEIVER = (
@@ -259,6 +261,178 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
assert "[bool]$prior.relayReload.newGenerationVerified" in source
def test_receiver_creates_canonical_success_receipt_once_with_atomic_no_replace() -> None:
source = RECEIVER.read_text(encoding="utf-8")
writer = source[
source.index("function Write-AgentRemoteDeployReceipt") : source.index(
"function Write-AgentResultAndExit"
)
]
assert (
source.count(
"Write-AgentRemoteDeployReceipt $receiptPath $successReceipt"
)
== 1
)
assert 'throw "immutable_receipt_path_already_exists"' in writer
assert writer.count("[IO.File]::Move($temporary, $Path)") == 1
assert "Move-Item -LiteralPath $temporary -Destination $Path -Force" not in writer
assert "$json = $Receipt | ConvertTo-Json -Depth 12" in writer
assert "$null = $storedJson | ConvertFrom-Json" in writer
atomic_move = writer.index("[IO.File]::Move($temporary, $Path)")
catch_boundary = writer.index("} catch {", atomic_move)
assert writer[atomic_move:catch_boundary].strip().endswith(
"[IO.File]::Move($temporary, $Path)"
)
def test_receiver_uses_unique_filesystem_safe_attempt_failure_receipts() -> None:
source = RECEIVER.read_text(encoding="utf-8")
assert '"yyyyMMdd\'T\'HHmmssfffffff\'Z\'"' in source
assert "[Globalization.CultureInfo]::InvariantCulture" in source
assert (
'$attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString(\'N\'))"'
in source
)
assert (
'"agent99-remote-deploy-$stageToken-attempt-$attemptToken.json"'
in source
)
assert source.count(
"Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt"
) >= 5
assert "yyyyMMddTHHmmssfffffffK" not in source
assert 'ToString("K")' not in source
def test_success_receipt_failure_rolls_back_before_durable_attempt_receipt() -> None:
source = RECEIVER.read_text(encoding="utf-8")
success_construction = source.index(
'$script:ReceiverOperationPhase = "success_receipt_construction"'
)
canonical_write = source.index(
"Write-AgentRemoteDeployReceipt $receiptPath $successReceipt"
)
failure_branch = source.index("if ($failurePhase)", canonical_write)
rollback = source.index(
"$rollback = Restore-AgentBundle $backupPath $beforeFilePresence",
failure_branch,
)
rollback_restart = source.index(
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
"$rollbackForbiddenGenerations",
rollback,
)
failure_receipt = source.index(
"Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt",
rollback_restart,
)
assert success_construction < canonical_write < failure_branch
assert failure_branch < rollback < rollback_restart < failure_receipt
assert (
'$failurePhase = $script:ReceiverOperationPhase' in source
)
assert (
'$failurePhase -in @("success_receipt_construction", '
'"success_receipt_write")'
in source
)
assert '"rolled_back_success_receipt_write_failed"' in source
assert '"rollback_failed_after_success_receipt_write_failure"' in source
assert "canonicalReceiptPath = $receiptPath" in source
assert "attemptReceiptPath = $attemptReceiptPath" in source
assert "durableReceiptWritten = $true" in source
def test_existing_canonical_receipt_blocks_live_promotion_without_overwrite() -> None:
source = RECEIVER.read_text(encoding="utf-8")
canonical_guard = source.index("if ($canonicalReceiptExists)")
validate_only = source.index(
"$validate = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision -ValidateOnly"
)
live_deploy = source.index(
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision"
)
assert canonical_guard < validate_only < live_deploy
assert '"blocked_canonical_receipt_invalid_no_promotion"' in source
assert (
'"blocked_canonical_success_receipt_runtime_drift_no_promotion"'
in source
)
assert '"blocked_canonical_receipt_reserved_no_promotion"' in source
assert (
'nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_'
'trace_run_work_item_identity"'
in source
)
def test_outer_catch_reports_conservative_operation_boundaries() -> None:
source = RECEIVER.read_text(encoding="utf-8")
transport_failure = source.index('status = "transport_receiver_failed"')
outer_catch = source[
source.rfind("} catch {", 0, transport_failure) : source.index(
"} finally {", transport_failure
)
]
for tracker in (
"$script:ReceiverRemoteWritePerformed",
"$script:ReceiverLiveRuntimeWritePerformed",
"$script:ReceiverLivePromotionAttempted",
"$script:ReceiverLivePromotionPerformed",
"$script:ReceiverRollbackAttempted",
"$script:ReceiverRollbackVerified",
"$script:ReceiverScheduledTaskRestartPerformed",
):
assert tracker in outer_catch
assert "operationBoundaries = [pscustomobject]@{" in outer_catch
assert "secretValueRead = $false" in outer_catch
assert "privateKeyValueRead = $false" in outer_catch
assert "tokenValueRead = $false" in outer_catch
assert "environmentSecretRead = $false" in outer_catch
assert "rawEnvelopeStored = $false" in outer_catch
assert "$outerErrorType = $_.Exception.GetType().Name" in outer_catch
assert "$script:ReceiverAttemptReceiptPath" in outer_catch
assert (
"Write-AgentRemoteDeployReceipt "
"$script:ReceiverAttemptReceiptPath $outerFailure"
) in outer_catch
assert "durableReceiptWritten = $false" in outer_catch
assert "$outerFailure.durableReceiptWritten = $true" in outer_catch
assert "$outerFailure.durableReceiptWritten = $false" in outer_catch
def test_receiver_parses_with_powershell_ast_when_runtime_is_available() -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
receiver_path = str(RECEIVER).replace("'", "''")
parser_command = (
"$tokens=$null; $errors=$null; "
"[System.Management.Automation.Language.Parser]::ParseFile("
f"'{receiver_path}', [ref]$tokens, [ref]$errors) | Out-Null; "
"if ($errors.Count -gt 0) { "
"$errors | ForEach-Object { $_.Message }; exit 1 }"
)
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", parser_command],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None:
source = RECEIVER.read_text(encoding="utf-8")

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import hashlib
import json
import os
import shutil
import subprocess
@@ -180,37 +181,41 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
assert "return 1" in marker_function
manager_start = source.index(
' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then'
' if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then'
)
manager_end = source.index(" ;;", manager_start)
manager_branch = source[manager_start:manager_end]
marker_failure_at = manager_branch.index(
' else\n record_action "wazuh_manager_marker_write_failed_no_write"'
' else\n record_action "wazuh_manager_marker_write_failed"'
)
success = manager_branch[:marker_failure_at]
failure = manager_branch[marker_failure_at:]
authorization_at = success.index("authorize_runtime_mutation")
marker_write_at = success.index(
'write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"'
)
marker_success_at = success.index(
' manager_marker_write_performed=1'
' manager_marker_write_performed=1'
)
reset_at = success.index("systemctl reset-failed wazuh-manager.service")
assert marker_success_at < reset_at
assert "runtime_write_performed=1" in success[:reset_at]
assert "state_write_performed=1" in success[:reset_at]
assert authorization_at < marker_write_at < marker_success_at < reset_at
assert "runtime_write_performed=1" in success[:marker_write_at]
assert "state_write_performed=1" in success[:marker_write_at]
assert "systemctl reset-failed wazuh-manager.service" in success
assert "systemctl start wazuh-manager.service" in success
assert 'bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
assert "manager_attempt_count=1" in success
assert "systemctl reset-failed wazuh-manager.service" not in failure
assert "systemctl start wazuh-manager.service" not in failure
assert "wazuh_manager_marker_write_failed_no_write" in failure
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
assert "wazuh_manager_marker_write_failed" in failure
assert 'manager_terminal="cooldown_marker_write_failed_degraded"' in failure
assert "manager_marker_write_performed=1" not in failure
assert "state_write_performed=$state_write_performed" in source
assert "manager_marker_write_performed=$manager_marker_write_performed" in source
indexer_start = source.index(
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
' if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then'
)
indexer_end = source.index(" else\n record_action", indexer_start)
indexer_branch = source[indexer_start:indexer_end]
@@ -219,7 +224,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
assert "systemctl start wazuh-indexer.service" in indexer_success
assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure
assert "systemctl start wazuh-indexer.service" not in indexer_failure
assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure
assert "wazuh_indexer_marker_write_failed" in indexer_failure
def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
@@ -271,6 +276,46 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No
assert "latest_readback_pointer_written=%s" in source
def test_runtime_mutations_are_guarded_by_a_fixed_durable_artifact_wal() -> None:
source = read(READINESS)
transaction = source[
source.index("artifact_paths_for_current_identity() {") : source.index(
"load_durable_receipt() {"
)
]
assert 'ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt"' in source
assert "schema_version=host112_active_artifact_transaction_v1" in transaction
assert 'mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"' in transaction
assert 'stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"' in transaction
assert 'artifact_transaction_status="blocked_unresolved_prior_transaction"' in transaction
assert 'write_artifact_transaction_record "pending_artifact_commit"' in transaction
assert 'write_artifact_transaction_record "partial_pending_public_artifact_commit"' in transaction
assert 'write_artifact_transaction_record "committed_verified_pair"' in transaction
assert "verify_artifact_pair_paths" in transaction
apply = source[source.index(' exec 9>"$MANAGER_LOCK_FILE"') :]
inspect_at = apply.index("inspect_active_artifact_transaction")
first_mutation_at = apply.index("systemctl set-default graphical.target")
assert inspect_at < first_mutation_at
for phase in (
"set_default_graphical_target",
"wazuh_indexer_cooldown_marker",
"wazuh_manager_cooldown_marker",
"public_artifact_commit",
):
assert f'authorize_runtime_mutation "{phase}"' in source
immutable_writer = source[
source.index("write_immutable_readback() {") : source.index(
'boot_id="$(cat /proc/sys/kernel/random/boot_id'
)
]
assert "durable_readback_present=1" not in immutable_writer
assert "durable_readback_identity_match=1" not in immutable_writer
assert "verify_current_readback_from_disk" in immutable_writer
def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
tmp_path: Path,
) -> None:
@@ -352,7 +397,9 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No
assert "trap 'handle_executor_signal TERM' TERM" in source
assert "trap 'handle_executor_signal INT' INT" in source
assert "trap 'handle_executor_signal HUP' HUP" in source
assert "if write_public_receipt && write_signal_readback; then" in source
assert "if write_public_receipt; then" in source
assert 'mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}"' in source
assert "write_signal_readback || true" in source
assert "action_count=${#actions[@]} actions=$receipt_action_text" in source
assert "schema_version=host112_guest_recovery_signal_readback_v1" in source
assert 'ln "$tmp" "$signal_readback_path"' in source
@@ -394,8 +441,9 @@ def test_apply_success_is_bound_to_same_run_durable_terminal() -> None:
assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate
assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate
guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]')
receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier)
receipt_commit = source.index("elif write_public_receipt; then", guest_verifier)
assert guest_verifier < receipt_commit
assert "commit_current_artifact_transaction" in source[receipt_commit:]
assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source
assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source
assert 'action_failure_observed=1' in source
@@ -471,7 +519,13 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
assert "-not $dryRun.artifactTransactionBlocked" in function
assert '$dryRun.artifactTransactionStatus -eq "prior_pair_verified"' in function
assert 'receiptStatus -eq "written_immutable"' in function
assert '$applyReceipt.artifactTransactionStatus -eq "committed_verified_pair"' in function
assert "$applyReceipt.artifactTransactionPairVerified" in function
assert "$durableReceiptReadback.artifactTransactionPriorPairVerified" in function
assert '$durableReceiptReadback.artifactTransactionStatus -eq "existing_pair_verified"' in function
assert '$after.managerVerified' in function
assert 'name = "wazuh_manager_service_active"' in function
assert 'name = "wazuh_analysisd_process_present"' in function
@@ -570,11 +624,56 @@ def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None:
assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy
assert 'name = "host112_canonical_direct_route"' in deploy
assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy
assert 'Host112 canonical SSH and VMX config post-verifier failed.' in deploy
assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap
assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap
assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap
assert 'Host112 canonical SSH and VMX config bootstrap verifier failed.' in bootstrap
for config in (generic_config, host_config):
assert '"192.168.0.112": "kali"' in config
assert '"directHosts"' in config
assert '"192.168.0.112"' in config
def test_deploy_and_bootstrap_canonicalize_host112_ssd_vmx_identity() -> None:
deploy = read(ROOT / "agent99-deploy.ps1")
bootstrap = read(BOOTSTRAP)
contract = read(ROOT / "agent99-contract-check.ps1")
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
host_config = read(ROOT / "agent99.config.99.example.json")
canonical_ps = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
canonical_json = r"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"
policy = json.loads(host_config)
host112_rows = [
row
for row in policy["vms"]
if row.get("host") == "192.168.0.112" or row.get("name") == "host112-kali"
]
assert canonical_json in host_config
assert host112_rows == [
{
"name": "host112-kali",
"host": "192.168.0.112",
"vmx": canonical_ps,
"priority": 30,
}
]
assert canonical_ps not in deploy
assert canonical_ps not in bootstrap
assert "Get-AgentHost112CanonicalVm $policyConfigPath" in deploy
assert "Get-AgentHost112CanonicalVm $PolicyPath" in bootstrap
bootstrap_preflight = (
'$null = Get-AgentHost112CanonicalVm $host112BootstrapPolicyPath'
)
bootstrap_first_write = "foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir"
assert bootstrap.index(bootstrap_preflight) < bootstrap.index(bootstrap_first_write)
assert 'name = "host112_canonical_vmx_path"' in deploy
assert "Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf" in deploy
assert "Host112 canonical SSH and VMX config post-verifier failed." in deploy
assert "Host112 canonical SSH and VMX config bootstrap verifier failed." in bootstrap
assert '$persistedHost112Vm.Count -ne 1' in deploy
assert '$persistedHost112Vm.Count -ne 1' in bootstrap
assert 'Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in deploy
assert 'Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in bootstrap
assert 'recovery:host112_canonical_vmx' in contract
assert 'recovery:host112_canonical_vmx' in synthetic

View File

@@ -81,6 +81,21 @@ if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
fi
/bin/mv "$@"
""",
)
_write_executable(
fake_bin / "ln",
"""#!/usr/bin/env bash
set -u
destination=""
for argument in "$@"; do destination="$argument"; done
if [ "${FAKE_RECEIPT_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */receipts/*.txt ]]; then
exit 1
fi
if [ "${FAKE_READBACK_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */readbacks/*.txt ]] && [[ "$destination" != *.signal.txt ]]; then
exit 1
fi
/bin/ln "$@"
""",
)
_write_executable(
@@ -118,6 +133,11 @@ printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
"""#!/usr/bin/env bash
set -u
state="${FAKE_HOST_STATE_DIR:?}"
require_wal() {
if [ ! -f "${HOST112_RECOVERY_STATE_DIR:?}/active-artifact-transaction.txt" ]; then
: >"$state/mutation_without_wal"
fi
}
cmd="${1:-}"
[ "$#" -gt 0 ] && shift
case "$cmd" in
@@ -154,10 +174,12 @@ case "$cmd" in
;;
get-default) printf '%s\n' 'graphical.target' ;;
set-default)
require_wal
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
exit "${FAKE_SET_DEFAULT_RC:-0}"
;;
start)
require_wal
unit="${1:-}"
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
if [ "$unit" = "lightdm.service" ]; then
@@ -168,9 +190,10 @@ case "$cmd" in
fi
;;
enable)
require_wal
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
;;
reset-failed|stop) ;;
reset-failed|stop) require_wal ;;
*) exit 1 ;;
esac
""",
@@ -211,6 +234,81 @@ def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str
)
def _dry_run(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
"bash",
str(script),
"--dry-run",
"--trace-id",
"trace:runtime-harness",
"--run-id",
"run:runtime-harness",
"--work-item-id",
"HOST112-RUNTIME-HARNESS",
],
text=True,
capture_output=True,
env=env,
check=False,
timeout=15,
)
def test_post_apply_dry_run_read_only_verifies_fixed_wal_and_immutable_pair(
tmp_path: Path,
) -> None:
script, env, _ = _runtime_harness(tmp_path)
applied = _apply(script, env)
assert applied.returncode == 0, applied.stdout + applied.stderr
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
transaction_before = transaction.read_bytes()
verified = _dry_run(script, env)
output = verified.stdout + verified.stderr
assert verified.returncode == 0, output
assert "durable_readback_status=loaded_immutable_verified " in output
assert "artifact_transaction_status=existing_pair_verified" in output
assert "artifact_transaction_started=0" in output
assert "artifact_transaction_blocked=0" in output
assert "artifact_transaction_write_performed=0" in output
assert "artifact_transaction_pair_verified=1" in output
assert "artifact_transaction_prior_pair_verified=1" in output
assert transaction.read_bytes() == transaction_before
def test_pre_apply_dry_run_blocks_unresolved_prior_transaction(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
state = tmp_path / "state"
state.mkdir()
transaction = state / "active-artifact-transaction.txt"
unresolved = (
"schema_version=host112_active_artifact_transaction_v1 "
"updated_at=2026-07-14T12:01:00+08:00 "
"status=pending_artifact_commit phase=prior_runtime_mutation "
"trace_id=trace:prior run_id=run:prior "
"work_item_id=HOST112-PRIOR boot_id=boot:prior "
f"receipt_path={state / 'receipts' / 'prior.txt'} "
f"readback_path={state / 'readbacks' / 'prior.txt'} "
"pair_verified=0\n"
)
transaction.write_text(unresolved, encoding="utf-8")
Path(env["HOST112_MANAGER_LOCK_FILE"]).touch()
dry_run = _dry_run(script, env)
output = dry_run.stdout + dry_run.stderr
assert dry_run.returncode == 1, output
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
assert "artifact_transaction_blocked=1" in output
assert "runtime_write_performed=0" in output
assert "receipt_write_performed=0" in output
assert transaction.read_text(encoding="utf-8") == unresolved
assert not (fake_state / "mutations.log").exists()
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
tmp_path: Path,
) -> None:
@@ -223,16 +321,28 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
assert failed.returncode == 1
assert "runtime_write_performed=1" in failed_output
assert "receipt_status=written_immutable" in failed_output
assert "durable_readback_status=written_immutable" in failed_output
assert "durable_readback_status=written_immutable " in failed_output
assert (
"durable_receipt_terminal=execution_failed_or_unattributed_"
"idempotent_already_verified_healthy"
) in failed_output
assert "guest_ready=1" in failed_output
assert "action_failure_observed=1" in failed_output
assert "artifact_transaction_status=committed_verified_pair" in failed_output
assert "artifact_transaction_pair_verified=1" in failed_output
assert not (fake_state / "mutation_without_wal").exists()
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
stored_readback = readback.read_text(encoding="utf-8")
for derived in (
"durable_readback_present=1",
"durable_readback_identity_match=1",
"durable_readback_boot_match=1",
"durable_readback_receipt_link_match=1",
"durable_readback_terminal_match=1",
):
assert derived not in stored_readback
receipt_before = receipt.read_bytes()
readback_before = readback.read_bytes()
@@ -250,6 +360,114 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
assert readback.read_bytes() == readback_before
def test_immutable_replay_cannot_ignore_newer_unresolved_fixed_transaction(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
first = _apply(script, env)
assert first.returncode == 0, first.stdout + first.stderr
state = tmp_path / "state"
receipt = next((state / "receipts").glob("*.txt"))
readback = next((state / "readbacks").glob("*.txt"))
receipt_before = receipt.read_bytes()
readback_before = readback.read_bytes()
transaction = state / "active-artifact-transaction.txt"
unresolved = (
"schema_version=host112_active_artifact_transaction_v1 "
"updated_at=2026-07-14T12:01:00+08:00 "
"status=pending_artifact_commit phase=newer_runtime_mutation "
"trace_id=trace:newer run_id=run:newer "
"work_item_id=HOST112-NEWER boot_id=boot:newer "
f"receipt_path={state / 'receipts' / 'newer.txt'} "
f"readback_path={state / 'readbacks' / 'newer.txt'} "
"pair_verified=0\n"
)
transaction.write_text(unresolved, encoding="utf-8")
mutations = fake_state / "mutations.log"
mutations_before = mutations.read_bytes()
replay = _apply(script, env)
output = replay.stdout + replay.stderr
assert replay.returncode == 1, output
assert "receipt_status=reused_immutable" in output
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
assert "artifact_transaction_blocked=1" in output
assert "artifact_transaction_pair_verified=0" in output
assert transaction.read_text(encoding="utf-8") == unresolved
assert receipt.read_bytes() == receipt_before
assert readback.read_bytes() == readback_before
assert mutations.read_bytes() == mutations_before
def test_receipt_failure_leaves_durable_partial_without_readback_and_blocks_retry(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
env.update(
{
"FAKE_LIGHTDM_START_RC": "124",
"FAKE_RECEIPT_LINK_FAIL": "1",
}
)
failed = _apply(script, env)
output = failed.stdout + failed.stderr
assert failed.returncode == 1
assert "runtime_write_performed=1" in output
assert "receipt_status=write_failed" in output
assert "artifact_transaction_status=partial_pending_public_artifact_commit" in output
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
stored = transaction.read_text(encoding="utf-8")
assert "status=partial_pending_public_artifact_commit" in stored
assert "phase=receipt_write_or_disk_verification_failed" in stored
assert not (tmp_path / "state" / "readbacks").exists()
assert not (fake_state / "mutation_without_wal").exists()
mutations = fake_state / "mutations.log"
mutations.write_text("", encoding="utf-8")
retry_env = env.copy()
retry_env.pop("FAKE_RECEIPT_LINK_FAIL")
retry = _apply(script, retry_env)
retry_output = retry.stdout + retry.stderr
assert retry.returncode == 1
assert "schema_version=host112_guest_recovery_v2" in retry_output
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
retry_output
)
assert "terminal=execution_failed_or_unattributed_" in retry_output
assert "artifact_transaction_blocked" in retry_output
assert "receipt_write_performed=0" in retry_output
assert mutations.read_text(encoding="utf-8") == ""
def test_readback_failure_keeps_receipt_and_durable_partial_not_false_success(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
env.update(
{
"FAKE_LIGHTDM_START_RC": "124",
"FAKE_READBACK_LINK_FAIL": "1",
}
)
failed = _apply(script, env)
output = failed.stdout + failed.stderr
assert failed.returncode == 1
assert "receipt_status=written_immutable" in output
assert "durable_readback_status=write_failed" in output
assert "artifact_transaction_pair_verified=0" in output
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
assert "terminal=" in receipt.read_text(encoding="utf-8")
assert not list((tmp_path / "state" / "readbacks").glob("*.txt"))
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
stored = transaction.read_text(encoding="utf-8")
assert "status=partial_pending_public_artifact_commit" in stored
assert "phase=durable_readback_write_or_disk_verification_failed" in stored
assert not (fake_state / "mutation_without_wal").exists()
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
tmp_path: Path,
) -> None:
@@ -282,8 +500,12 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
mutation_root.mkdir()
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
mutation_slots: list[str] = []
# These five identities cover every four-slot ring position and then reuse
# one slot. That proves both capacity coverage and rollover without twelve
# expensive full shell-harness invocations.
for index in range(5):
run_env = env | {"INVOCATION_ID": f"slot-{index}"}
result = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
@@ -296,7 +518,14 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
assert result.returncode == 1
assert "runtime_write_performed=1" in output
assert "receipt_status=written_bounded_timer_slot" in output
assert "durable_readback_status=written_bounded_timer_slot" in output
assert "durable_readback_status=written_bounded_timer_slot " in output
mutation_slots.append(
next(
token.split("=", 1)[1]
for token in output.split()
if token.startswith("timer_ledger_slot=")
)
)
for field in (
"durable_readback_present",
"durable_readback_identity_match",
@@ -308,12 +537,15 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
assert f"{field}=0" not in output
assert "timer_no_runtime_mutation_aggregated" not in output
assert len(set(mutation_slots)) == 4
assert len(mutation_slots) > len(set(mutation_slots))
ledger_files = [
path
for path in (mutation_root / "state" / "timer-ledger").iterdir()
if path.is_file()
]
assert len(ledger_files) <= 8
assert len(ledger_files) == 8
assert not (mutation_root / "state" / "receipts").exists()
assert not (mutation_root / "state" / "readbacks").exists()
for readback in (mutation_root / "state" / "timer-ledger").glob(
@@ -327,8 +559,10 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
"durable_readback_receipt_link_match",
"durable_readback_terminal_match",
):
assert stored.count(f"{field}=1") == 1
assert f"{field}=1" not in stored
assert f"{field}=0" not in stored
transaction = mutation_root / "state" / "active-artifact-transaction.txt"
assert "status=committed_verified_pair" in transaction.read_text(encoding="utf-8")
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
@@ -337,33 +571,50 @@ def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
marker = tmp_path / f"receipt-mv-{index}.entered"
run_env = env | {
"INVOCATION_ID": f"sigkill-{index}",
"FAKE_RECEIPT_MV_MARKER": str(marker),
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
}
process = subprocess.Popen(
["bash", str(script), "--apply-timer"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=run_env,
start_new_session=True,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists()
os.killpg(process.pid, signal.SIGKILL)
process.communicate(timeout=5)
assert process.returncode == -signal.SIGKILL
marker = tmp_path / "receipt-mv.entered"
run_env = env | {
"INVOCATION_ID": "sigkill-0",
"FAKE_RECEIPT_MV_MARKER": str(marker),
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
}
process = subprocess.Popen(
["bash", str(script), "--apply-timer"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=run_env,
start_new_session=True,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists()
os.killpg(process.pid, signal.SIGKILL)
process.communicate(timeout=5)
assert process.returncode == -signal.SIGKILL
ledger = tmp_path / "state" / "timer-ledger"
files = [path for path in ledger.iterdir() if path.is_file()]
assert len(files) <= 4
assert len(files) == 1
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
assert "status=pending_artifact_commit" in transaction.read_text(encoding="utf-8")
mutations = tmp_path / "fake-host" / "mutations.log"
before = mutations.read_bytes()
retry = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
capture_output=True,
env=env | {"INVOCATION_ID": "sigkill-retry"},
check=False,
timeout=15,
)
assert retry.returncode == 1
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
retry.stdout + retry.stderr
)
assert mutations.read_bytes() == before
def test_signal_during_final_verifier_writes_correlated_failure_chain(
@@ -400,7 +651,7 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain(
assert marker.exists()
os.kill(process.pid, signal.SIGTERM)
stdout, stderr = process.communicate(timeout=12)
assert process.returncode == 75, stdout + stderr
assert process.returncode == 1, stdout + stderr
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
@@ -411,6 +662,15 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain(
assert f"receipt_path={receipt}" in signal_text
assert "terminal=interrupted_signal_TERM" in signal_text
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
transaction_text = transaction.read_text(encoding="utf-8")
assert "status=partial_pending_public_artifact_commit" in transaction_text
assert "phase=interrupted_before_durable_readback_TERM" in transaction_text
assert not [
path
for path in (tmp_path / "state" / "readbacks").glob("*.txt")
if not path.name.endswith(".signal.txt")
]
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
assert replay.returncode == 65

View File

@@ -1,5 +1,6 @@
from __future__ import annotations
import os
import subprocess
from pathlib import Path
@@ -67,6 +68,8 @@ def test_windows99_direct_route_is_fingerprint_bound_and_not_forced() -> None:
assert 'cat "$AUTH_FILE"' not in text
assert 'from="%s",restrict,command="%s --check"' in text
assert 'CONTROL_SOURCE_ADDRESS:-192.168.0.110' in text
assert 'BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH' in text
assert "never discovers, reads, copies, or derives a private key" in text
def test_apply_is_transactional_and_has_a_verified_rollback() -> None:
@@ -149,3 +152,73 @@ def test_check_gate_requires_policy_route_timer_and_no_write_readback() -> None:
assert 'runtime_write_performed=0' in text
assert 'manager_runtime_write_performed=0' in text
assert 'receipt_write_performed=0' in text
def test_clean_install_timer_stop_is_state_aware_and_idempotent(
tmp_path: Path,
) -> None:
text = source()
function = text[
text.index("stop_recovery_timer_if_running() {") : text.index(
"\n\nresolve_target_user",
text.index("stop_recovery_timer_if_running() {"),
)
]
runner = tmp_path / "run-stop-contract.sh"
runner.write_text(
"#!/usr/bin/env bash\nset -u\n" + function + "\nstop_recovery_timer_if_running\n",
encoding="utf-8",
)
runner.chmod(0o755)
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
log = tmp_path / "systemctl.log"
systemctl = fake_bin / "systemctl"
systemctl.write_text(
"""#!/usr/bin/env bash
printf '%s\n' "$*" >>"${SYSTEMCTL_LOG:?}"
if [ "${1:-}" = "is-active" ]; then
printf '%s\n' "${FAKE_TIMER_STATE:-not-found}"
[ "${FAKE_TIMER_STATE:-not-found}" = "active" ]
fi
""",
encoding="utf-8",
)
systemctl.chmod(0o755)
timeout = fake_bin / "timeout"
timeout.write_text(
"#!/usr/bin/env bash\nshift\n\"$@\"\n",
encoding="utf-8",
)
timeout.chmod(0o755)
env = os.environ | {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"SYSTEMCTL_LOG": str(log),
}
clean = subprocess.run(
["bash", str(runner)],
text=True,
capture_output=True,
env=env | {"FAKE_TIMER_STATE": "not-found"},
check=False,
timeout=10,
)
assert clean.returncode == 0, clean.stdout + clean.stderr
assert "stop awoooi-host112-guest-recovery.timer" not in log.read_text(
encoding="utf-8"
)
log.write_text("", encoding="utf-8")
active = subprocess.run(
["bash", str(runner)],
text=True,
capture_output=True,
env=env | {"FAKE_TIMER_STATE": "active"},
check=False,
timeout=10,
)
assert active.returncode == 0, active.stdout + active.stderr
assert "stop awoooi-host112-guest-recovery.timer" in log.read_text(
encoding="utf-8"
)

View File

@@ -88,6 +88,22 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]:
assert field in host_probe
canonical_host112_vmx = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
assert canonical_host112_vmx in windows99
assert r"D:\Downloads\kali-linux-2025.4-vmware-amd64" not in windows99
assert r'"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"' in read(
"agent99.config.99.example.json"
)
assert canonical_host112_vmx in read(
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py"
)
for locator in (
"scripts/reboot-recovery/windows99-vmx-source-locator.ps1",
"scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1",
"scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1",
):
assert r"S:\VMs" in read(locator)
host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh")
host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh")
host112_service = read(

View File

@@ -1,6 +1,6 @@
param(
[string[]]$VmNameAllowList = @("110", "188", "120", "121", "112"),
[string[]]$SearchRoots = @("C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"),
[string[]]$SearchRoots = @("S:\VMs", "C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"),
[string]$TaskName = "AWOOOI VMware VM AutoStart",
[switch]$WhatIfOnly
)

View File

@@ -12,6 +12,7 @@ param(
[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112"),
[switch]$EnableRecursiveDiscovery,
[string[]]$DiscoveryRoot = @(
"S:\VMs",
"D:\Documents\Virtual Machines",
"D:\Downloads",
"D:\VMs",
@@ -41,7 +42,7 @@ $KnownVmxCandidates = @{
"188" = @("D:\Documents\Virtual Machines\Ollama_Ubuntu_64-bit\Ollama_Ubuntu_64-bit.vmx")
"120" = @("D:\Documents\Virtual Machines\192.168.0.120_Ubuntu_64-bit\192.168.0.120_Ubuntu_64-bit.vmx")
"121" = @("D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx")
"112" = @("D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx")
"112" = @("S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx")
}
function Resolve-VmxPath {

View File

@@ -2,6 +2,7 @@ param(
[string]$Alias = "111",
[string]$ExpectedVmxPath = "D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx",
[string[]]$DiscoveryRoot = @(
"S:\VMs",
"D:\Documents\Virtual Machines",
"D:\Downloads",
"D:\VMs",

View File

@@ -6,6 +6,7 @@ param(
"D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx"
),
[string[]]$DiscoveryRoot = @(
"S:\VMs",
"D:\Documents\Virtual Machines",
"D:\Downloads",
"D:\VMs",