feat(automation): harden D037 runtime closure receipts
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
This commit is contained in:
@@ -77,6 +77,14 @@ workload_required_connection_budget() {
|
||||
esac
|
||||
}
|
||||
|
||||
workload_representative_probe_count() {
|
||||
# Capacity is proven by the PostgreSQL role limit and production HTTP
|
||||
# concurrency. The isolated pod opens one representative connection for
|
||||
# role, privilege, and query verification without consuming the live
|
||||
# workload's entire role budget.
|
||||
printf '1'
|
||||
}
|
||||
|
||||
workload_deployment() {
|
||||
case "$1" in
|
||||
api) printf 'awoooi-api|api' ;;
|
||||
@@ -222,6 +230,7 @@ preflight_workload_once() {
|
||||
secret="$2"
|
||||
connection_limit="$3"
|
||||
required_connection_budget="$4"
|
||||
representative_probe_count="$5"
|
||||
deployment_spec="$(workload_deployment "$workload")"
|
||||
deployment="${deployment_spec%%|*}"
|
||||
container="${deployment_spec#*|}"
|
||||
@@ -262,7 +271,9 @@ spec:
|
||||
key: DATABASE_URL
|
||||
- name: EXPECTED_CONNECTION_LIMIT
|
||||
value: "${connection_limit}"
|
||||
- name: REQUIRED_CONCURRENT_CONNECTIONS
|
||||
- name: REPRESENTATIVE_CONNECTION_PROBE_COUNT
|
||||
value: "${representative_probe_count}"
|
||||
- name: REQUIRED_CONNECTION_BUDGET
|
||||
value: "${required_connection_budget}"
|
||||
- name: WORKLOAD_ID
|
||||
value: "${workload}"
|
||||
@@ -287,7 +298,15 @@ spec:
|
||||
async with engine.connect() as conn:
|
||||
identity = (
|
||||
await conn.execute(text("""
|
||||
SELECT current_user AS role_name, rolconnlimit
|
||||
SELECT
|
||||
current_user AS role_name,
|
||||
rolconnlimit,
|
||||
(
|
||||
SELECT count(*)
|
||||
FROM pg_stat_activity
|
||||
WHERE usename = current_user
|
||||
AND pid <> pg_backend_pid()
|
||||
) AS active_role_connection_count
|
||||
FROM pg_roles WHERE rolname = current_user
|
||||
"""))
|
||||
).mappings().one()
|
||||
@@ -333,39 +352,38 @@ spec:
|
||||
AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE'))
|
||||
)
|
||||
"""))).scalar_one())
|
||||
required_concurrency = int(
|
||||
os.environ["REQUIRED_CONCURRENT_CONNECTIONS"]
|
||||
)
|
||||
existing_role_connection_count = int((
|
||||
await conn.execute(text("""
|
||||
SELECT count(*)
|
||||
FROM pg_stat_activity
|
||||
WHERE usename = current_user
|
||||
"""))
|
||||
).scalar_one())
|
||||
additional_probe_count = max(
|
||||
0,
|
||||
required_concurrency
|
||||
- existing_role_connection_count,
|
||||
)
|
||||
representative_probe_count = int(
|
||||
os.environ["REPRESENTATIVE_CONNECTION_PROBE_COUNT"]
|
||||
)
|
||||
if representative_probe_count < 1:
|
||||
raise RuntimeError("representative_probe_count_invalid")
|
||||
|
||||
async def probe_connection() -> None:
|
||||
async with engine.connect() as probe:
|
||||
await probe.execute(
|
||||
text("SELECT pg_sleep(0.25)")
|
||||
)
|
||||
async def probe_connection() -> None:
|
||||
async with engine.connect() as probe:
|
||||
await probe.execute(text("SELECT pg_sleep(0.25)"))
|
||||
|
||||
await asyncio.gather(*(
|
||||
probe_connection()
|
||||
for _ in range(additional_probe_count)
|
||||
))
|
||||
await asyncio.gather(*(
|
||||
probe_connection()
|
||||
for _ in range(representative_probe_count)
|
||||
))
|
||||
finally:
|
||||
await engine.dispose()
|
||||
|
||||
expected_limit = int(os.environ["EXPECTED_CONNECTION_LIMIT"])
|
||||
required_budget = int(os.environ["REQUIRED_CONNECTION_BUDGET"])
|
||||
observed_limit = int(identity["rolconnlimit"])
|
||||
active_role_connection_count = int(
|
||||
identity["active_role_connection_count"]
|
||||
)
|
||||
available_connection_headroom = (
|
||||
observed_limit - active_role_connection_count
|
||||
)
|
||||
if observed_limit != expected_limit:
|
||||
raise RuntimeError("connection_limit_mismatch")
|
||||
if observed_limit < required_budget:
|
||||
raise RuntimeError("connection_budget_below_required")
|
||||
if available_connection_headroom < required_budget:
|
||||
raise RuntimeError("connection_headroom_below_required")
|
||||
if table_gap_count or sequence_gap_count:
|
||||
raise RuntimeError("database_privilege_gap")
|
||||
print(json.dumps({
|
||||
@@ -375,12 +393,11 @@ spec:
|
||||
str(identity["role_name"]).encode("utf-8")
|
||||
).hexdigest(),
|
||||
"connection_limit": observed_limit,
|
||||
"concurrent_connection_probe_count": required_concurrency,
|
||||
"existing_role_connection_count": (
|
||||
existing_role_connection_count
|
||||
),
|
||||
"additional_connection_probe_count": (
|
||||
additional_probe_count
|
||||
"required_connection_budget": required_budget,
|
||||
"active_role_connection_count": active_role_connection_count,
|
||||
"available_connection_headroom": available_connection_headroom,
|
||||
"representative_connection_probe_count": (
|
||||
representative_probe_count
|
||||
),
|
||||
"representative_read_count": 3,
|
||||
"table_privilege_gap_count": table_gap_count,
|
||||
@@ -474,6 +491,7 @@ for workload in "${selected_workloads[@]}"; do
|
||||
secret="$(workload_secret "$workload")"
|
||||
connection_limit="$(workload_connection_limit "$workload")"
|
||||
required_connection_budget="$(workload_required_connection_budget "$workload")"
|
||||
representative_probe_count="$(workload_representative_probe_count "$workload")"
|
||||
secret_present=false
|
||||
if kube get secret "$secret" -n "$NAMESPACE" >/dev/null 2>&1; then
|
||||
secret_present=true
|
||||
@@ -507,11 +525,11 @@ for workload in "${selected_workloads[@]}"; do
|
||||
else
|
||||
ready=false
|
||||
fi
|
||||
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget"
|
||||
echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget representative_probe_count=$representative_probe_count"
|
||||
|
||||
if [ "$MODE" = "apply" ] || [ "$MODE" = "verify" ]; then
|
||||
preflight_workload "$workload" "$secret" "$connection_limit" \
|
||||
"$required_connection_budget"
|
||||
"$required_connection_budget" "$representative_probe_count"
|
||||
fi
|
||||
done
|
||||
|
||||
|
||||
@@ -7,6 +7,89 @@ param(
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
function Get-AgentLivePreflightDecision {
|
||||
param(
|
||||
[bool]$AgentRootPresent,
|
||||
[object]$Manifest,
|
||||
[int]$ExpectedRuntimeFileCount,
|
||||
[int]$TaskFailureCount,
|
||||
[int]$RelayListenerCount,
|
||||
[int]$RequiredEvidenceStaleCount,
|
||||
[int]$RequiredEvidenceParseFailureCount,
|
||||
[int]$RequiredEvidenceContentFailureCount,
|
||||
[string]$SelfHealthSeverity,
|
||||
[int]$SelfHealthPerformanceIssueCount,
|
||||
[int]$StaleAlertExecutionCount,
|
||||
[int]$AlertIncomingCount,
|
||||
[int]$PendingCommandCount
|
||||
)
|
||||
|
||||
$blockingReasons = @()
|
||||
$warningReasons = @()
|
||||
$manifestExists = [bool]($Manifest -and $Manifest.exists)
|
||||
$manifestParseReady = [bool](
|
||||
$manifestExists -and
|
||||
-not ([string]$Manifest.parseError)
|
||||
)
|
||||
$manifestIdentityReady = [bool](
|
||||
$manifestParseReady -and
|
||||
[string]$Manifest.sourceRevision -match "^[a-fA-F0-9]{40}$" -and
|
||||
[int]$Manifest.fileCount -eq $ExpectedRuntimeFileCount
|
||||
)
|
||||
$manifestContentReady = [bool](
|
||||
$manifestParseReady -and
|
||||
$Manifest.runtimeMatched -eq $true -and
|
||||
[int]$Manifest.mismatchCount -eq 0
|
||||
)
|
||||
|
||||
if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" }
|
||||
if (-not $manifestExists) {
|
||||
$blockingReasons += "runtime_manifest_missing"
|
||||
} elseif (-not $manifestParseReady) {
|
||||
$blockingReasons += "runtime_manifest_parse_failed"
|
||||
} else {
|
||||
if (-not $manifestIdentityReady) {
|
||||
$blockingReasons += "runtime_manifest_identity_invalid"
|
||||
}
|
||||
if (-not $manifestContentReady) {
|
||||
$blockingReasons += "runtime_manifest_mismatch"
|
||||
}
|
||||
}
|
||||
if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
||||
if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" }
|
||||
if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" }
|
||||
if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
|
||||
if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
|
||||
|
||||
$normalizedSelfHealth = ([string]$SelfHealthSeverity).Trim().ToLowerInvariant()
|
||||
switch ($normalizedSelfHealth) {
|
||||
"ok" {}
|
||||
"warning" {
|
||||
$warningReasons += "self_health_warning"
|
||||
if ($SelfHealthPerformanceIssueCount -gt 0) {
|
||||
$warningReasons += "performance_warning"
|
||||
}
|
||||
}
|
||||
"critical" { $blockingReasons += "self_health_critical" }
|
||||
default { $blockingReasons += "self_health_unclassified" }
|
||||
}
|
||||
|
||||
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
|
||||
$blockingReasons += "pending_work_before_reboot"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
deploymentEligible = [bool]($blockingReasons.Count -eq 0)
|
||||
blockingReasons = @($blockingReasons)
|
||||
warningReasons = @($warningReasons)
|
||||
warningCount = [int]$warningReasons.Count
|
||||
manifestIdentityReady = $manifestIdentityReady
|
||||
manifestContentReady = $manifestContentReady
|
||||
selfHealthClassification = $normalizedSelfHealth
|
||||
}
|
||||
}
|
||||
|
||||
function Get-TaskReadback {
|
||||
param([string]$TaskName)
|
||||
|
||||
@@ -103,6 +186,38 @@ function Get-EvidenceReadback {
|
||||
$publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() }
|
||||
$performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() }
|
||||
$telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() }
|
||||
$selfHealthPerformance = if (
|
||||
$rawEvidence.PSObject.Properties["selfHealth"] -and
|
||||
$rawEvidence.selfHealth -and
|
||||
$rawEvidence.selfHealth.PSObject.Properties["latestPerformance"]
|
||||
) { $rawEvidence.selfHealth.latestPerformance } else { $null }
|
||||
$selfHealthPerformanceSeverity = if ($selfHealthPerformance) {
|
||||
([string]$selfHealthPerformance.severity).Trim().ToLowerInvariant()
|
||||
} else { "" }
|
||||
if ($selfHealthPerformanceSeverity -notin @("ok", "warning", "critical")) {
|
||||
$selfHealthPerformanceSeverity = "unknown"
|
||||
}
|
||||
$selfHealthPerformanceRows = if ($selfHealthPerformance) {
|
||||
@($selfHealthPerformance.hosts | ForEach-Object {
|
||||
$safeHost = ([string]$_.host).Trim()
|
||||
if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$") {
|
||||
$safeHost = "unknown"
|
||||
}
|
||||
$safeSeverity = ([string]$_.severity).Trim().ToLowerInvariant()
|
||||
if ($safeSeverity -notin @("ok", "warning", "critical", "missing")) {
|
||||
$safeSeverity = "unknown"
|
||||
}
|
||||
$safeReasons = @($_.reasons | ForEach-Object {
|
||||
$reason = ([string]$_).Trim().ToLowerInvariant()
|
||||
if ($reason -match "^[a-z0-9_]{1,80}$") { $reason }
|
||||
})
|
||||
[pscustomobject]@{
|
||||
host = $safeHost
|
||||
severity = $safeSeverity
|
||||
reasons = $safeReasons
|
||||
}
|
||||
})
|
||||
} else { @() }
|
||||
$safeSummary = if ($Name -eq "sre_alert_inbox") {
|
||||
[pscustomobject]@{
|
||||
ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false }
|
||||
@@ -152,6 +267,9 @@ function Get-EvidenceReadback {
|
||||
performanceCount = $performanceRows.Count
|
||||
performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count
|
||||
selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" }
|
||||
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
|
||||
selfHealthPerformanceIssueCount = [int]@($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" }).Count
|
||||
selfHealthPerformanceIssues = @($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" })
|
||||
backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" }
|
||||
controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 }
|
||||
controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 }
|
||||
@@ -292,21 +410,27 @@ $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $
|
||||
$requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy })
|
||||
$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1
|
||||
$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" }
|
||||
$selfHealthPerformanceSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthPerformanceSeverity } else { "unknown" }
|
||||
$selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount } else { 0 }
|
||||
$alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1
|
||||
$alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1
|
||||
$commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1
|
||||
$blockingReasons = @()
|
||||
if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" }
|
||||
if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" }
|
||||
if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" }
|
||||
if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" }
|
||||
if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" }
|
||||
if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" }
|
||||
if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" }
|
||||
if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" }
|
||||
if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" }
|
||||
if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" }
|
||||
$decision = Get-AgentLivePreflightDecision `
|
||||
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
|
||||
-Manifest $manifest `
|
||||
-ExpectedRuntimeFileCount 14 `
|
||||
-TaskFailureCount $taskFailures.Count `
|
||||
-RelayListenerCount $relayListeners.Count `
|
||||
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `
|
||||
-RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count `
|
||||
-RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count `
|
||||
-SelfHealthSeverity $selfHealthSeverity `
|
||||
-SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount `
|
||||
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
|
||||
-AlertIncomingCount $alertIncoming.count `
|
||||
-PendingCommandCount $commandQueue.count
|
||||
$blockingReasons = @($decision.blockingReasons)
|
||||
$warningReasons = @($decision.warningReasons)
|
||||
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_live_preflight_v1"
|
||||
@@ -332,16 +456,24 @@ $result = [pscustomobject]@{
|
||||
requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count
|
||||
requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count
|
||||
selfHealthSeverity = $selfHealthSeverity
|
||||
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
|
||||
selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount
|
||||
alertIncomingCount = [int]$alertIncoming.count
|
||||
alertRunningCount = [int]$alertRunning.count
|
||||
pendingCommandCount = [int]$commandQueue.count
|
||||
relayListenerCount = $relayListeners.Count
|
||||
preflightGreen = [bool]($blockingReasons.Count -eq 0)
|
||||
preflightGreen = [bool]$decision.deploymentEligible
|
||||
deploymentEligible = [bool]$decision.deploymentEligible
|
||||
manifestIdentityReady = [bool]$decision.manifestIdentityReady
|
||||
manifestContentReady = [bool]$decision.manifestContentReady
|
||||
blockingReasons = $blockingReasons
|
||||
warningCount = [int]$decision.warningCount
|
||||
warningReasons = $warningReasons
|
||||
}
|
||||
}
|
||||
|
||||
Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)"
|
||||
Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"
|
||||
Write-Output "MODE=$Mode"
|
||||
Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)"
|
||||
Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
||||
@@ -351,13 +483,14 @@ Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
||||
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
||||
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
||||
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
||||
Write-Output "WARNING_REASONS=$($warningReasons -join ',')"
|
||||
Write-Output "SECRET_VALUE_READ=0"
|
||||
Write-Output "REMOTE_WRITE_PERFORMED=0"
|
||||
Write-Output "JSON_BEGIN"
|
||||
$result | ConvertTo-Json -Depth 8 -Compress
|
||||
Write-Output "JSON_END"
|
||||
|
||||
if ($result.summary.preflightGreen) {
|
||||
if ($result.summary.deploymentEligible) {
|
||||
exit 0
|
||||
}
|
||||
exit 2
|
||||
|
||||
@@ -33,6 +33,21 @@ $FixedRuntimeFiles = @(
|
||||
"agent99.config.99.example.json"
|
||||
)
|
||||
$script:DeployLock = $null
|
||||
$script:ReceiverRemoteWritePerformed = $false
|
||||
$script:ReceiverLiveRuntimeWritePerformed = $false
|
||||
$script:ReceiverLivePromotionAttempted = $false
|
||||
$script:ReceiverLivePromotionPerformed = $false
|
||||
$script:ReceiverScheduledTaskRestartPerformed = $false
|
||||
$script:ReceiverRollbackAttempted = $false
|
||||
$script:ReceiverRollbackVerified = $false
|
||||
$script:ReceiverOperationPhase = "envelope_validation"
|
||||
$script:ReceiverAttemptReceiptPath = ""
|
||||
$script:ReceiverCanonicalReceiptPath = ""
|
||||
$script:ReceiverTraceId = ""
|
||||
$script:ReceiverRunId = ""
|
||||
$script:ReceiverWorkItemId = ""
|
||||
$script:ReceiverSourceRevision = ""
|
||||
$script:ReceiverManifestDigest = ""
|
||||
|
||||
function Get-AgentSha256Bytes {
|
||||
param([byte[]]$Bytes)
|
||||
@@ -503,9 +518,26 @@ function Restore-AgentBundle {
|
||||
function Write-AgentRemoteDeployReceipt {
|
||||
param([string]$Path, [object]$Receipt)
|
||||
|
||||
$temporary = "$Path.tmp.$PID"
|
||||
$Receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $temporary -Encoding UTF8
|
||||
Move-Item -LiteralPath $temporary -Destination $Path -Force
|
||||
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
|
||||
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$json = $Receipt | ConvertTo-Json -Depth 12
|
||||
$null = $json | ConvertFrom-Json
|
||||
$utf8NoBom = New-Object System.Text.UTF8Encoding($false)
|
||||
$script:ReceiverRemoteWritePerformed = $true
|
||||
[IO.File]::WriteAllText($temporary, $json, $utf8NoBom)
|
||||
$storedJson = [IO.File]::ReadAllText($temporary, [Text.Encoding]::UTF8)
|
||||
$null = $storedJson | ConvertFrom-Json
|
||||
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
|
||||
# File.Move is the final operation: on Windows PowerShell/.NET it is atomic
|
||||
# within this volume and refuses to replace an existing destination.
|
||||
[IO.File]::Move($temporary, $Path)
|
||||
} catch {
|
||||
if (Test-Path -LiteralPath $temporary) {
|
||||
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue
|
||||
}
|
||||
throw
|
||||
}
|
||||
}
|
||||
|
||||
function Write-AgentResultAndExit {
|
||||
@@ -576,6 +608,19 @@ try {
|
||||
$stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20)
|
||||
$stagePath = Join-Path $DeployRoot "remote-source-$stageToken"
|
||||
$receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json"
|
||||
$attemptStamp = [DateTime]::UtcNow.ToString(
|
||||
"yyyyMMdd'T'HHmmssfffffff'Z'",
|
||||
[Globalization.CultureInfo]::InvariantCulture
|
||||
)
|
||||
$attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString('N'))"
|
||||
$attemptReceiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken-attempt-$attemptToken.json"
|
||||
$script:ReceiverAttemptReceiptPath = $attemptReceiptPath
|
||||
$script:ReceiverCanonicalReceiptPath = $receiptPath
|
||||
$script:ReceiverTraceId = $traceId
|
||||
$script:ReceiverRunId = $runId
|
||||
$script:ReceiverWorkItemId = $workItemId
|
||||
$script:ReceiverSourceRevision = $sourceRevision
|
||||
$script:ReceiverManifestDigest = $manifestDigest
|
||||
$currentManifest = Get-AgentSafeRuntimeManifest
|
||||
|
||||
if ($mode -eq "check") {
|
||||
@@ -591,6 +636,9 @@ try {
|
||||
livePreflight = [pscustomobject]@{ executed = $false; reason = "apply_only_after_validate_only" }
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $false
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
@@ -606,6 +654,8 @@ try {
|
||||
}) 0
|
||||
}
|
||||
|
||||
$script:ReceiverOperationPhase = "apply_state_initialization"
|
||||
$script:ReceiverRemoteWritePerformed = $true
|
||||
New-Item -ItemType Directory -Force -Path $StateDir, $EvidenceDir, $DeployRoot | Out-Null
|
||||
$lockPath = Join-Path $StateDir "remote-atomic-deploy.lock"
|
||||
try {
|
||||
@@ -619,18 +669,29 @@ try {
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
remoteWritePerformed = $false
|
||||
operationPhase = $script:ReceiverOperationPhase
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
}) 75
|
||||
}
|
||||
|
||||
$canonicalReceiptExists = $false
|
||||
$prior = $null
|
||||
$priorParseErrorType = ""
|
||||
if (Test-Path -LiteralPath $receiptPath -PathType Leaf) {
|
||||
try { $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json } catch { $prior = $null }
|
||||
$canonicalReceiptExists = $true
|
||||
try {
|
||||
$prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json
|
||||
if (-not $prior) { $priorParseErrorType = "NullCanonicalReceipt" }
|
||||
} catch {
|
||||
$prior = $null
|
||||
$priorParseErrorType = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
|
||||
$stageWrittenThisRun = $false
|
||||
$script:ReceiverOperationPhase = "staging"
|
||||
if (-not (Test-Path -LiteralPath $stagePath -PathType Container)) {
|
||||
$stageBuildPath = "$stagePath.partial.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
New-Item -ItemType Directory -Path $stageBuildPath | Out-Null
|
||||
@@ -640,7 +701,7 @@ try {
|
||||
[IO.File]::WriteAllBytes((Join-Path $stageBuildPath "agent99-live-preflight.ps1"), $preflightBytes)
|
||||
$sourceManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $stageBuildPath "source-manifest.json") -Encoding UTF8
|
||||
Move-Item -LiteralPath $stageBuildPath -Destination $stagePath
|
||||
$stageWrittenThisRun = $true
|
||||
$script:ReceiverRemoteWritePerformed = $true
|
||||
} else {
|
||||
foreach ($name in $FixedRuntimeFiles) {
|
||||
if ((Get-AgentSha256File (Join-Path $stagePath $name)) -ne ([string]($filesByName[$name].sha256)).ToLowerInvariant()) {
|
||||
@@ -678,6 +739,7 @@ try {
|
||||
$live.fileCount -eq 14 -and
|
||||
$live.mismatchCount -eq 0
|
||||
) {
|
||||
$script:ReceiverOperationPhase = "idempotent_replay_post_verifier"
|
||||
$replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched) {
|
||||
$originalOperationBoundaries = $prior.operationBoundaries
|
||||
@@ -686,8 +748,9 @@ try {
|
||||
$prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force
|
||||
$prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force
|
||||
$prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{
|
||||
remoteWritePerformed = $stageWrittenThisRun
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
@@ -702,7 +765,7 @@ try {
|
||||
}) -Force
|
||||
Write-AgentResultAndExit $prior 0
|
||||
}
|
||||
Write-AgentResultAndExit ([pscustomobject]@{
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "idempotent_replay_post_verifier_failed_no_promotion"
|
||||
mode = "apply"
|
||||
@@ -712,15 +775,66 @@ try {
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
runtimeManifest = $live
|
||||
livePreflight = $replayPreflight
|
||||
remoteWritePerformed = $stageWrittenThisRun
|
||||
durableReceiptWritten = $true
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
nextSafeAction = "repair_reported_live_preflight_blocker_then_retry_same_identity"
|
||||
}) 2
|
||||
}
|
||||
$script:ReceiverOperationPhase = "idempotent_replay_failure_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
if ($canonicalReceiptExists) {
|
||||
$canonicalStatus = if (-not $prior) {
|
||||
"blocked_canonical_receipt_invalid_no_promotion"
|
||||
} elseif ([string]$prior.status -eq "deployed_verified") {
|
||||
"blocked_canonical_success_receipt_runtime_drift_no_promotion"
|
||||
} else {
|
||||
"blocked_canonical_receipt_reserved_no_promotion"
|
||||
}
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = $canonicalStatus
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
canonicalReceiptStatus = if ($prior) { [string]$prior.status } else { "unparseable" }
|
||||
canonicalReceiptParseErrorType = $priorParseErrorType
|
||||
runtimeManifest = $live
|
||||
durableReceiptWritten = $true
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_trace_run_work_item_identity"
|
||||
}
|
||||
$script:ReceiverOperationPhase = "canonical_receipt_reserved_failure_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
$script:ReceiverOperationPhase = "validate_only"
|
||||
$validate = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision -ValidateOnly
|
||||
if (-not $validate.ok -or -not $validate.payload -or [string]$validate.payload.status -ne "validated") {
|
||||
$receipt = [pscustomobject]@{
|
||||
@@ -733,12 +847,22 @@ try {
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
remoteWritePerformed = $true
|
||||
durableReceiptWritten = $true
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
nextSafeAction = "repair_validate_only_blocker_then_retry_same_identity"
|
||||
}
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
$script:ReceiverOperationPhase = "validate_only_failure_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
@@ -773,10 +897,14 @@ try {
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut }
|
||||
relayRuntime = $beforeRelayRuntime
|
||||
remoteWritePerformed = $true
|
||||
durableReceiptWritten = $true
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $false
|
||||
livePromotionAttempted = $false
|
||||
livePromotionPerformed = $false
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
@@ -790,7 +918,8 @@ try {
|
||||
scheduledTaskModification = $false
|
||||
nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity"
|
||||
}
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
$script:ReceiverOperationPhase = "relay_precondition_failure_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 2
|
||||
}
|
||||
|
||||
@@ -803,6 +932,9 @@ try {
|
||||
$failureType = ""
|
||||
$backupPath = ""
|
||||
try {
|
||||
$script:ReceiverOperationPhase = "live_deploy"
|
||||
$script:ReceiverLiveRuntimeWritePerformed = $true
|
||||
$script:ReceiverLivePromotionAttempted = $true
|
||||
$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision
|
||||
if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) {
|
||||
$backupPath = [string]$deploy.payload.backupDir
|
||||
@@ -811,11 +943,15 @@ try {
|
||||
$failurePhase = "deploy"
|
||||
$failureType = "deploy_child_failed_or_invalid"
|
||||
} else {
|
||||
$script:ReceiverLivePromotionPerformed = $true
|
||||
$script:ReceiverOperationPhase = "relay_reload"
|
||||
$script:ReceiverScheduledTaskRestartPerformed = $true
|
||||
$relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations)
|
||||
if (-not $relayReload.ok) {
|
||||
$failurePhase = "relay_reload"
|
||||
$failureType = "relay_reload_failed"
|
||||
} else {
|
||||
$script:ReceiverOperationPhase = "independent_post_verifier"
|
||||
$runtimeManifest = Get-AgentSafeRuntimeManifest
|
||||
$livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1")
|
||||
$postVerified = [bool](
|
||||
@@ -845,6 +981,55 @@ try {
|
||||
$failureType = $_.Exception.GetType().Name
|
||||
}
|
||||
|
||||
$successReceipt = $null
|
||||
if (-not $failurePhase) {
|
||||
try {
|
||||
$script:ReceiverOperationPhase = "success_receipt_construction"
|
||||
$successReceipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "deployed_verified"
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
canonicalReceiptPath = $receiptPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
|
||||
relayReload = $relayReload
|
||||
runtimeManifest = $runtimeManifest
|
||||
livePreflight = $livePreflight
|
||||
durableReceiptWritten = $true
|
||||
idempotentReplay = $false
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $true
|
||||
liveRuntimeWritePerformed = $true
|
||||
livePromotionAttempted = $true
|
||||
livePromotionPerformed = $true
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $true
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
|
||||
}
|
||||
$script:ReceiverOperationPhase = "success_receipt_write"
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $successReceipt
|
||||
} catch {
|
||||
$failurePhase = $script:ReceiverOperationPhase
|
||||
$failureType = $_.Exception.GetType().Name
|
||||
}
|
||||
}
|
||||
|
||||
if ($failurePhase) {
|
||||
if (-not $backupPath) {
|
||||
$newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue |
|
||||
@@ -853,6 +1038,8 @@ try {
|
||||
Select-Object -First 1
|
||||
if ($newBackup) { $backupPath = $newBackup.FullName }
|
||||
}
|
||||
$script:ReceiverOperationPhase = "runtime_rollback"
|
||||
$script:ReceiverRollbackAttempted = $true
|
||||
$rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent
|
||||
$rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations)
|
||||
if ($relayReload) {
|
||||
@@ -902,6 +1089,7 @@ try {
|
||||
$rollbackRelayRestart.after.taskRunning -and
|
||||
$rollbackRelayRestart.after.listenerCount -gt 0
|
||||
)
|
||||
$script:ReceiverRollbackVerified = $rollbackVerified
|
||||
} catch {
|
||||
$rollbackVerified = $false
|
||||
$rollbackVerifierErrorType = $_.Exception.GetType().Name
|
||||
@@ -911,6 +1099,8 @@ try {
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = if ($failurePhase -eq "deploy") {
|
||||
if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" }
|
||||
} elseif ($failurePhase -in @("success_receipt_construction", "success_receipt_write")) {
|
||||
if ($rollbackVerified) { "rolled_back_success_receipt_write_failed" } else { "rollback_failed_after_success_receipt_write_failure" }
|
||||
} else {
|
||||
if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" }
|
||||
}
|
||||
@@ -926,6 +1116,8 @@ try {
|
||||
deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } }
|
||||
failurePhase = $failurePhase
|
||||
failureType = $failureType
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
relayReload = $relayReload
|
||||
failedRuntimeManifest = $runtimeManifest
|
||||
failedLivePreflight = $livePreflight
|
||||
@@ -935,6 +1127,24 @@ try {
|
||||
rollbackVerifierErrorType = $rollbackVerifierErrorType
|
||||
runtimeManifest = $afterManifest
|
||||
durableReceiptWritten = $true
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
|
||||
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
|
||||
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
|
||||
rollbackAttempted = $script:ReceiverRollbackAttempted
|
||||
rollbackVerified = $rollbackVerified
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted)
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
@@ -957,10 +1167,22 @@ try {
|
||||
sourceRevision = $sourceRevision
|
||||
failurePhase = $failurePhase
|
||||
failureType = $failureType
|
||||
canonicalReceiptPath = $receiptPath
|
||||
attemptReceiptPath = $attemptReceiptPath
|
||||
rollbackVerified = $false
|
||||
runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified
|
||||
receiptConstructionErrorType = $_.Exception.GetType().Name
|
||||
durableReceiptWritten = $true
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
|
||||
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
|
||||
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
|
||||
rollbackAttempted = $script:ReceiverRollbackAttempted
|
||||
rollbackVerified = $false
|
||||
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
secretValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
scheduledTaskModification = $false
|
||||
@@ -968,7 +1190,7 @@ try {
|
||||
$rollbackVerified = $false
|
||||
}
|
||||
try {
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt
|
||||
} catch {
|
||||
$receipt.status = "rollback_receipt_write_failed"
|
||||
$receipt.durableReceiptWritten = $false
|
||||
@@ -978,45 +1200,29 @@ try {
|
||||
Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 })
|
||||
}
|
||||
|
||||
$receipt = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "deployed_verified"
|
||||
mode = "apply"
|
||||
traceId = $traceId
|
||||
runId = $runId
|
||||
workItemId = $workItemId
|
||||
sourceRevision = $sourceRevision
|
||||
manifestSha256 = $manifestDigest
|
||||
stagingPath = $stagePath
|
||||
backupPath = $backupPath
|
||||
validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status }
|
||||
deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status }
|
||||
relayReload = $relayReload
|
||||
runtimeManifest = $runtimeManifest
|
||||
livePreflight = $livePreflight
|
||||
idempotentReplay = $false
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $true
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $true
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback"
|
||||
}
|
||||
Write-AgentRemoteDeployReceipt $receiptPath $receipt
|
||||
Write-AgentResultAndExit $receipt 0
|
||||
$script:ReceiverOperationPhase = "success_result_output"
|
||||
Write-AgentResultAndExit $successReceipt 0
|
||||
} catch {
|
||||
Write-AgentResultAndExit ([pscustomobject]@{
|
||||
$outerErrorType = $_.Exception.GetType().Name
|
||||
$outerFailure = [pscustomobject]@{
|
||||
schemaVersion = "agent99_remote_atomic_deploy_receipt_v1"
|
||||
status = "transport_receiver_failed"
|
||||
errorType = $_.Exception.GetType().Name
|
||||
errorType = $outerErrorType
|
||||
operationPhase = $script:ReceiverOperationPhase
|
||||
traceId = $script:ReceiverTraceId
|
||||
runId = $script:ReceiverRunId
|
||||
workItemId = $script:ReceiverWorkItemId
|
||||
sourceRevision = $script:ReceiverSourceRevision
|
||||
manifestSha256 = $script:ReceiverManifestDigest
|
||||
canonicalReceiptPath = if ($script:ReceiverCanonicalReceiptPath) { $script:ReceiverCanonicalReceiptPath } else { "unavailable" }
|
||||
attemptReceiptPath = if ($script:ReceiverAttemptReceiptPath) { $script:ReceiverAttemptReceiptPath } else { "unavailable" }
|
||||
durableReceiptWritten = $false
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
|
||||
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
|
||||
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
|
||||
rollbackAttempted = $script:ReceiverRollbackAttempted
|
||||
rollbackVerified = $script:ReceiverRollbackVerified
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
@@ -1026,9 +1232,40 @@ try {
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $false
|
||||
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
|
||||
scheduledTaskModification = $false
|
||||
}) 70
|
||||
operationBoundaries = [pscustomobject]@{
|
||||
remoteWritePerformed = $script:ReceiverRemoteWritePerformed
|
||||
liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed
|
||||
livePromotionAttempted = $script:ReceiverLivePromotionAttempted
|
||||
livePromotionPerformed = $script:ReceiverLivePromotionPerformed
|
||||
rollbackAttempted = $script:ReceiverRollbackAttempted
|
||||
rollbackVerified = $script:ReceiverRollbackVerified
|
||||
secretValueRead = $false
|
||||
privateKeyValueRead = $false
|
||||
tokenValueRead = $false
|
||||
environmentSecretRead = $false
|
||||
uiInteraction = $false
|
||||
vmPowerChange = $false
|
||||
hostReboot = $false
|
||||
serviceRestart = $false
|
||||
scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed
|
||||
scheduledTaskModification = $false
|
||||
}
|
||||
}
|
||||
if ($script:ReceiverAttemptReceiptPath) {
|
||||
try {
|
||||
$script:ReceiverRemoteWritePerformed = $true
|
||||
$outerFailure.remoteWritePerformed = $true
|
||||
$outerFailure.operationBoundaries.remoteWritePerformed = $true
|
||||
$outerFailure.durableReceiptWritten = $true
|
||||
Write-AgentRemoteDeployReceipt $script:ReceiverAttemptReceiptPath $outerFailure
|
||||
} catch {
|
||||
$outerFailure.durableReceiptWritten = $false
|
||||
$outerFailure | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force
|
||||
}
|
||||
}
|
||||
Write-AgentResultAndExit $outerFailure 70
|
||||
} finally {
|
||||
if ($script:DeployLock) {
|
||||
try { $script:DeployLock.Dispose() } catch {}
|
||||
|
||||
@@ -1,11 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host 112 guest readiness and bounded recovery executor.
|
||||
# Version: 2.0
|
||||
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
|
||||
# Version: 2.1
|
||||
# Last modified: 2026-07-14 13:15 (Asia/Taipei)
|
||||
# Last modified by: Codex
|
||||
# Change: add correlated, cooldown-guarded Wazuh manager recovery and an
|
||||
# independent process/port verifier without reboot, VM power, firewall, or
|
||||
# secret access.
|
||||
# Change: guard every runtime mutation with a fixed durable artifact WAL and
|
||||
# require independently verified receipt/readback persistence before success.
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
@@ -174,6 +173,18 @@ signal_readback_status="not_requested"
|
||||
signal_readback_path="none"
|
||||
signal_readback_sha256="none"
|
||||
action_failure_observed=0
|
||||
ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt"
|
||||
artifact_transaction_status="not_requested"
|
||||
artifact_transaction_phase="not_started"
|
||||
artifact_transaction_started=0
|
||||
artifact_transaction_blocked=0
|
||||
artifact_transaction_write_performed=0
|
||||
artifact_transaction_pair_verified=0
|
||||
artifact_transaction_prior_pair_verified=0
|
||||
artifact_expected_receipt_path="none"
|
||||
artifact_expected_readback_path="none"
|
||||
artifact_chain_failed=0
|
||||
artifact_receipt_ready=0
|
||||
|
||||
is_uint() {
|
||||
[[ "$1" =~ ^[0-9]+$ ]]
|
||||
@@ -274,6 +285,12 @@ start_if_inactive() {
|
||||
if [ "$state" = "active" ] || [ "$state" = "activating" ]; then
|
||||
return 0
|
||||
fi
|
||||
if ! authorize_runtime_mutation "start_${unit//[^A-Za-z0-9._-]/_}"; then
|
||||
record_action "${action}_artifact_transaction_blocked_failed"
|
||||
# Keep the executor on its fail-closed evidence path. The recorded
|
||||
# failure drives a degraded terminal without allowing this mutation.
|
||||
return 0
|
||||
fi
|
||||
# A mutating systemctl invocation may partially change runtime state before
|
||||
# returning non-zero or timing out. Record the attempt before execution so
|
||||
# those paths can never be classified as no-write timer observations.
|
||||
@@ -291,6 +308,10 @@ enable_if_disabled() {
|
||||
if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then
|
||||
return 0
|
||||
fi
|
||||
if ! authorize_runtime_mutation "enable_${unit//[^A-Za-z0-9._-]/_}"; then
|
||||
record_action "${action}_artifact_transaction_blocked_failed"
|
||||
return 0
|
||||
fi
|
||||
runtime_write_performed=1
|
||||
state_write_performed=1
|
||||
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then
|
||||
@@ -507,8 +528,268 @@ timer_slot_for_identity() {
|
||||
printf '%03d' "$((checksum % timer_ledger_capacity))"
|
||||
}
|
||||
|
||||
reset_durable_receipt_verdict() {
|
||||
durable_receipt_present=0
|
||||
durable_receipt_identity_match=0
|
||||
durable_receipt_boot_match=0
|
||||
durable_receipt_terminal="none"
|
||||
}
|
||||
|
||||
reset_durable_readback_verdict() {
|
||||
durable_readback_present=0
|
||||
durable_readback_identity_match=0
|
||||
durable_readback_boot_match=0
|
||||
durable_readback_receipt_link_match=0
|
||||
durable_readback_terminal_match=0
|
||||
}
|
||||
|
||||
artifact_paths_for_current_identity() {
|
||||
local receipt_name timer_slot
|
||||
if [ "$MODE" = "timer_apply" ]; then
|
||||
timer_slot="$(timer_slot_for_identity)" || return 1
|
||||
timer_ledger_slot="$timer_slot"
|
||||
receipt_name="timer-slot-${timer_slot}"
|
||||
artifact_expected_receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt"
|
||||
artifact_expected_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt"
|
||||
else
|
||||
receipt_name="$(receipt_name_for_identity)"
|
||||
artifact_expected_receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
|
||||
artifact_expected_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
|
||||
fi
|
||||
case "$artifact_expected_receipt_path" in *[[:space:]]*) return 1 ;; esac
|
||||
case "$artifact_expected_readback_path" in *[[:space:]]*) return 1 ;; esac
|
||||
}
|
||||
|
||||
controlled_artifact_paths() {
|
||||
local candidate_receipt="$1" candidate_readback="$2"
|
||||
case "$candidate_receipt" in
|
||||
"$STATE_DIR"/receipts/*.txt|"$STATE_DIR"/timer-ledger/*.receipt.txt) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
case "$candidate_readback" in
|
||||
"$STATE_DIR"/readbacks/*.txt|"$STATE_DIR"/timer-ledger/*.readback.txt) ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
verify_artifact_pair_paths() {
|
||||
local expected_trace="$1" expected_run="$2" expected_work_item="$3"
|
||||
local expected_boot="$4" candidate_receipt="$5" candidate_readback="$6"
|
||||
local receipt_schema receipt_trace receipt_run receipt_work_item receipt_boot receipt_terminal receipt_digest
|
||||
local readback_schema
|
||||
local readback_trace readback_run readback_work_item readback_boot
|
||||
local readback_receipt_path readback_receipt_sha readback_terminal
|
||||
controlled_artifact_paths "$candidate_receipt" "$candidate_readback" || return 1
|
||||
[ -f "$candidate_receipt" ] && [ -f "$candidate_readback" ] || return 1
|
||||
receipt_schema="$(receipt_token "$candidate_receipt" schema_version)"
|
||||
receipt_trace="$(receipt_token "$candidate_receipt" trace_id)"
|
||||
receipt_run="$(receipt_token "$candidate_receipt" run_id)"
|
||||
receipt_work_item="$(receipt_token "$candidate_receipt" work_item_id)"
|
||||
receipt_boot="$(receipt_token "$candidate_receipt" boot_id)"
|
||||
receipt_terminal="$(receipt_token "$candidate_receipt" terminal)"
|
||||
[ "$receipt_schema" = "host112_wazuh_manager_recovery_receipt_v2" ] \
|
||||
&& [ -n "$receipt_terminal" ] && [ "$receipt_terminal" != "none" ] \
|
||||
&& [ "$receipt_trace" = "$expected_trace" ] \
|
||||
&& [ "$receipt_run" = "$expected_run" ] \
|
||||
&& [ "$receipt_work_item" = "$expected_work_item" ] \
|
||||
&& [ "$receipt_boot" = "$expected_boot" ] || return 1
|
||||
receipt_digest="$(sha256sum "$candidate_receipt" | awk '{print $1}')"
|
||||
[ -n "$receipt_digest" ] || return 1
|
||||
readback_schema="$(receipt_token "$candidate_readback" schema_version)"
|
||||
readback_trace="$(receipt_token "$candidate_readback" trace_id)"
|
||||
readback_run="$(receipt_token "$candidate_readback" run_id)"
|
||||
readback_work_item="$(receipt_token "$candidate_readback" work_item_id)"
|
||||
readback_boot="$(receipt_token "$candidate_readback" boot_id)"
|
||||
readback_receipt_path="$(receipt_token "$candidate_readback" receipt_path)"
|
||||
readback_receipt_sha="$(receipt_token "$candidate_readback" receipt_sha256)"
|
||||
readback_terminal="$(receipt_token "$candidate_readback" durable_receipt_terminal)"
|
||||
[ "$readback_schema" = "host112_guest_recovery_v2" ] \
|
||||
&& [ "$readback_trace" = "$expected_trace" ] \
|
||||
&& [ "$readback_run" = "$expected_run" ] \
|
||||
&& [ "$readback_work_item" = "$expected_work_item" ] \
|
||||
&& [ "$readback_boot" = "$expected_boot" ] \
|
||||
&& [ "$readback_receipt_path" = "$candidate_receipt" ] \
|
||||
&& [ "$readback_receipt_sha" = "$receipt_digest" ] \
|
||||
&& [ "$readback_terminal" = "$receipt_terminal" ]
|
||||
}
|
||||
|
||||
write_artifact_transaction_record() {
|
||||
local requested_status="$1" requested_phase="$2" pair_verified="$3"
|
||||
local tmp stored_schema stored_status stored_phase stored_trace stored_run
|
||||
local stored_work_item stored_boot stored_receipt stored_readback stored_pair
|
||||
artifact_paths_for_current_identity || return 1
|
||||
install -d -m 0750 "$STATE_DIR" || return 1
|
||||
tmp="${ARTIFACT_TRANSACTION_PATH}.tmp.$$"
|
||||
if ! printf '%s\n' \
|
||||
"schema_version=host112_active_artifact_transaction_v1 updated_at=$(date --iso-8601=seconds) status=$requested_status phase=$requested_phase trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id receipt_path=$artifact_expected_receipt_path readback_path=$artifact_expected_readback_path pair_verified=$pair_verified" \
|
||||
>"$tmp" \
|
||||
|| ! chmod 0640 "$tmp" \
|
||||
|| ! mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"; then
|
||||
rm -f "$tmp" >/dev/null 2>&1 || true
|
||||
return 1
|
||||
fi
|
||||
stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"
|
||||
stored_status="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" status)"
|
||||
stored_phase="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" phase)"
|
||||
stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)"
|
||||
stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)"
|
||||
stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)"
|
||||
stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)"
|
||||
stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)"
|
||||
stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)"
|
||||
stored_pair="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" pair_verified)"
|
||||
if [ "$stored_schema" != "host112_active_artifact_transaction_v1" ] \
|
||||
|| [ "$stored_status" != "$requested_status" ] \
|
||||
|| [ "$stored_phase" != "$requested_phase" ] \
|
||||
|| [ "$stored_trace" != "$TRACE_ID" ] \
|
||||
|| [ "$stored_run" != "$RUN_ID" ] \
|
||||
|| [ "$stored_work_item" != "$WORK_ITEM_ID" ] \
|
||||
|| [ "$stored_boot" != "$boot_id" ] \
|
||||
|| [ "$stored_receipt" != "$artifact_expected_receipt_path" ] \
|
||||
|| [ "$stored_readback" != "$artifact_expected_readback_path" ] \
|
||||
|| [ "$stored_pair" != "$pair_verified" ]; then
|
||||
return 1
|
||||
fi
|
||||
artifact_transaction_status="$requested_status"
|
||||
artifact_transaction_phase="$requested_phase"
|
||||
artifact_transaction_write_performed=1
|
||||
state_write_performed=1
|
||||
return 0
|
||||
}
|
||||
|
||||
inspect_active_artifact_transaction() {
|
||||
local stored_schema stored_trace stored_run stored_work_item stored_boot
|
||||
local stored_receipt stored_readback
|
||||
[ -f "$ARTIFACT_TRANSACTION_PATH" ] || {
|
||||
artifact_transaction_status="none"
|
||||
artifact_transaction_phase="no_prior_transaction"
|
||||
return 0
|
||||
}
|
||||
stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"
|
||||
stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)"
|
||||
stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)"
|
||||
stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)"
|
||||
stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)"
|
||||
stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)"
|
||||
stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)"
|
||||
if [ "$stored_schema" = "host112_active_artifact_transaction_v1" ] \
|
||||
&& safe_identity "$stored_trace" \
|
||||
&& safe_identity "$stored_run" \
|
||||
&& safe_identity "$stored_work_item" \
|
||||
&& [ -n "$stored_boot" ] \
|
||||
&& verify_artifact_pair_paths "$stored_trace" "$stored_run" "$stored_work_item" \
|
||||
"$stored_boot" "$stored_receipt" "$stored_readback"; then
|
||||
artifact_transaction_prior_pair_verified=1
|
||||
artifact_transaction_status="prior_pair_verified"
|
||||
artifact_transaction_phase="prior_transaction_resolved"
|
||||
return 0
|
||||
fi
|
||||
artifact_transaction_blocked=1
|
||||
artifact_transaction_status="blocked_unresolved_prior_transaction"
|
||||
artifact_transaction_phase="mutation_denied"
|
||||
return 1
|
||||
}
|
||||
|
||||
authorize_runtime_mutation() {
|
||||
local phase="${1:-runtime_mutation}"
|
||||
[ "$artifact_transaction_started" -eq 0 ] || return 0
|
||||
[ "$artifact_transaction_blocked" -eq 0 ] || return 1
|
||||
if write_artifact_transaction_record "pending_artifact_commit" "$phase" 0; then
|
||||
artifact_transaction_started=1
|
||||
return 0
|
||||
fi
|
||||
artifact_transaction_blocked=1
|
||||
artifact_transaction_status="wal_write_or_readback_failed"
|
||||
artifact_transaction_phase="mutation_denied"
|
||||
return 1
|
||||
}
|
||||
|
||||
mark_artifact_transaction_partial() {
|
||||
local phase="$1"
|
||||
[ "$artifact_transaction_started" -eq 1 ] || return 0
|
||||
if ! write_artifact_transaction_record "partial_pending_public_artifact_commit" "$phase" 0; then
|
||||
artifact_transaction_status="pending_artifact_commit_partial_update_failed"
|
||||
artifact_transaction_phase="$phase"
|
||||
fi
|
||||
}
|
||||
|
||||
verify_current_receipt_from_disk() {
|
||||
local expected_terminal="${1:-}" stored_trace stored_run stored_work_item stored_boot stored_terminal
|
||||
reset_durable_receipt_verdict
|
||||
[ -f "$receipt_path" ] || return 1
|
||||
durable_receipt_present=1
|
||||
stored_trace="$(receipt_token "$receipt_path" trace_id)"
|
||||
stored_run="$(receipt_token "$receipt_path" run_id)"
|
||||
stored_work_item="$(receipt_token "$receipt_path" work_item_id)"
|
||||
stored_boot="$(receipt_token "$receipt_path" boot_id)"
|
||||
stored_terminal="$(receipt_token "$receipt_path" terminal)"
|
||||
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
||||
durable_receipt_terminal="$stored_terminal"
|
||||
if [ "$stored_trace" = "$TRACE_ID" ] \
|
||||
&& [ "$stored_run" = "$RUN_ID" ] \
|
||||
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
|
||||
durable_receipt_identity_match=1
|
||||
fi
|
||||
[ "$stored_boot" = "$boot_id" ] && durable_receipt_boot_match=1
|
||||
[ "$durable_receipt_identity_match" -eq 1 ] \
|
||||
&& [ "$durable_receipt_boot_match" -eq 1 ] \
|
||||
&& [ -n "$stored_terminal" ] \
|
||||
&& { [ -z "$expected_terminal" ] || [ "$stored_terminal" = "$expected_terminal" ]; }
|
||||
}
|
||||
|
||||
verify_current_readback_from_disk() {
|
||||
local stored_trace stored_run stored_work_item stored_boot
|
||||
local stored_receipt_path stored_receipt_sha stored_terminal
|
||||
reset_durable_readback_verdict
|
||||
[ -f "$durable_readback_path" ] || return 1
|
||||
durable_readback_present=1
|
||||
stored_trace="$(receipt_token "$durable_readback_path" trace_id)"
|
||||
stored_run="$(receipt_token "$durable_readback_path" run_id)"
|
||||
stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)"
|
||||
stored_boot="$(receipt_token "$durable_readback_path" boot_id)"
|
||||
stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)"
|
||||
stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)"
|
||||
stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)"
|
||||
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
||||
if [ "$stored_trace" = "$TRACE_ID" ] \
|
||||
&& [ "$stored_run" = "$RUN_ID" ] \
|
||||
&& [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then
|
||||
durable_readback_identity_match=1
|
||||
fi
|
||||
[ "$stored_boot" = "$boot_id" ] && durable_readback_boot_match=1
|
||||
if [ "$stored_receipt_path" = "$receipt_path" ] \
|
||||
&& [ "$stored_receipt_sha" = "$receipt_sha256" ]; then
|
||||
durable_readback_receipt_link_match=1
|
||||
fi
|
||||
[ "$stored_terminal" = "$durable_receipt_terminal" ] \
|
||||
&& durable_readback_terminal_match=1
|
||||
[ "$durable_readback_identity_match" -eq 1 ] \
|
||||
&& [ "$durable_readback_boot_match" -eq 1 ] \
|
||||
&& [ "$durable_readback_receipt_link_match" -eq 1 ] \
|
||||
&& [ "$durable_readback_terminal_match" -eq 1 ]
|
||||
}
|
||||
|
||||
commit_current_artifact_transaction() {
|
||||
[ "$artifact_transaction_blocked" -eq 0 ] || return 1
|
||||
artifact_paths_for_current_identity || return 1
|
||||
if ! verify_artifact_pair_paths "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$boot_id" \
|
||||
"$artifact_expected_receipt_path" "$artifact_expected_readback_path"; then
|
||||
mark_artifact_transaction_partial "independent_pair_verifier_failed"
|
||||
return 1
|
||||
fi
|
||||
artifact_transaction_pair_verified=1
|
||||
if [ "$artifact_transaction_started" -eq 1 ]; then
|
||||
write_artifact_transaction_record "committed_verified_pair" \
|
||||
"independent_pair_verifier_passed" 1 || return 1
|
||||
else
|
||||
artifact_transaction_status="existing_pair_verified"
|
||||
artifact_transaction_phase="independent_pair_verifier_passed"
|
||||
fi
|
||||
}
|
||||
|
||||
load_durable_receipt() {
|
||||
local receipt_name stored_trace stored_run stored_work_item stored_boot
|
||||
reset_durable_receipt_verdict
|
||||
receipt_name="$(receipt_name_for_identity)"
|
||||
receipt_path="$STATE_DIR/receipts/${receipt_name}.txt"
|
||||
[ -f "$receipt_path" ] || return 1
|
||||
@@ -539,6 +820,7 @@ load_durable_receipt() {
|
||||
load_durable_readback() {
|
||||
local receipt_name stored_trace stored_run stored_work_item stored_boot
|
||||
local stored_receipt_path stored_receipt_sha stored_terminal
|
||||
reset_durable_readback_verdict
|
||||
receipt_name="$(receipt_name_for_identity)"
|
||||
durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt"
|
||||
[ -f "$durable_readback_path" ] || {
|
||||
@@ -618,13 +900,10 @@ write_public_receipt() {
|
||||
&& mv "$tmp" "$receipt_path"; then
|
||||
receipt_write_performed=1
|
||||
receipt_status="written_bounded_timer_slot"
|
||||
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
||||
receipt_ref="host112-timer-ledger:${receipt_name}"
|
||||
durable_receipt_present=1
|
||||
durable_receipt_identity_match=1
|
||||
durable_receipt_boot_match=1
|
||||
durable_receipt_terminal="$run_terminal"
|
||||
return 0
|
||||
if verify_current_receipt_from_disk "$run_terminal"; then
|
||||
return 0
|
||||
fi
|
||||
elif [ "$MODE" != "timer_apply" ] \
|
||||
&& printf '%s\n' "$receipt_line" >"$tmp" \
|
||||
&& chmod 0640 "$tmp" \
|
||||
@@ -632,13 +911,10 @@ write_public_receipt() {
|
||||
rm -f "$tmp"
|
||||
receipt_write_performed=1
|
||||
receipt_status="written_immutable"
|
||||
receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')"
|
||||
receipt_ref="host112-manager-recovery:${receipt_name}"
|
||||
durable_receipt_present=1
|
||||
durable_receipt_identity_match=1
|
||||
durable_receipt_boot_match=1
|
||||
durable_receipt_terminal="$run_terminal"
|
||||
return 0
|
||||
if verify_current_receipt_from_disk "$run_terminal"; then
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
rm -f "$tmp" >/dev/null 2>&1 || true
|
||||
if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then
|
||||
@@ -720,19 +996,15 @@ write_bounded_timer_mutation_readback() {
|
||||
return 1
|
||||
}
|
||||
tmp="${durable_readback_path}.tmp"
|
||||
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
|
||||
if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite stored_readback_kind=bounded_timer_slot durable_readback_path=%s\n' \
|
||||
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \
|
||||
"$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \
|
||||
&& chmod 0640 "$tmp" \
|
||||
&& mv "$tmp" "$durable_readback_path"; then
|
||||
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
||||
durable_readback_status="written_bounded_timer_slot"
|
||||
durable_readback_present=1
|
||||
durable_readback_identity_match=1
|
||||
durable_readback_boot_match=1
|
||||
durable_readback_receipt_link_match=1
|
||||
durable_readback_terminal_match=1
|
||||
return 0
|
||||
if verify_current_readback_from_disk; then
|
||||
durable_readback_status="written_bounded_timer_slot"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
rm -f "$tmp" >/dev/null 2>&1 || true
|
||||
durable_readback_status="write_failed"
|
||||
@@ -806,7 +1078,7 @@ write_immutable_readback() {
|
||||
fi
|
||||
install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1
|
||||
tmp="${durable_readback_path}.tmp"
|
||||
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \
|
||||
if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s stored_readback_kind=immutable durable_readback_path=%s\n' \
|
||||
"$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \
|
||||
|| ! chmod 0640 "$tmp" \
|
||||
|| ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then
|
||||
@@ -815,13 +1087,11 @@ write_immutable_readback() {
|
||||
return 1
|
||||
fi
|
||||
rm -f "$tmp"
|
||||
durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')"
|
||||
if ! verify_current_readback_from_disk; then
|
||||
durable_readback_status="written_immutable_disk_verification_failed"
|
||||
return 1
|
||||
fi
|
||||
durable_readback_status="written_immutable"
|
||||
durable_readback_present=1
|
||||
durable_readback_identity_match=1
|
||||
durable_readback_boot_match=1
|
||||
durable_readback_receipt_link_match=1
|
||||
durable_readback_terminal_match=1
|
||||
|
||||
# Preserve the historical convenience readback as an explicitly mutable
|
||||
# pointer. It is never used as execution proof; the immutable path/hash is.
|
||||
@@ -834,8 +1104,11 @@ write_immutable_readback() {
|
||||
return 0
|
||||
fi
|
||||
rm -f "$latest_tmp" >/dev/null 2>&1 || true
|
||||
durable_readback_status="immutable_written_latest_pointer_failed"
|
||||
return 1
|
||||
# The immutable artifact remains verified even when this auxiliary mutable
|
||||
# convenience pointer cannot be refreshed. Keep the stable consumer status;
|
||||
# latest_readback_pointer_written=0 carries the narrower pointer failure.
|
||||
durable_readback_status="written_immutable"
|
||||
return 0
|
||||
}
|
||||
|
||||
boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
|
||||
@@ -875,6 +1148,41 @@ if [ "$durable_receipt_present" -eq 1 ]; then
|
||||
fi
|
||||
fi
|
||||
|
||||
# Every dry-run first proves that no unresolved fixed WAL makes the planned
|
||||
# apply impossible. When a WAL exists, hold a shared lock on the already-
|
||||
# existing lock file so an apply cannot race this read-only verifier. Only a
|
||||
# post-apply dry-run with a current immutable pair performs the extra in-memory
|
||||
# pair verification; neither path creates or mutates state.
|
||||
if [ "$MODE" = "dry_run" ]; then
|
||||
if [ -f "$ARTIFACT_TRANSACTION_PATH" ]; then
|
||||
if ! { [ -f "$MANAGER_LOCK_FILE" ] \
|
||||
&& exec 8<"$MANAGER_LOCK_FILE" \
|
||||
&& flock -s -n 8 \
|
||||
&& inspect_active_artifact_transaction; }; then
|
||||
artifact_chain_failed=1
|
||||
artifact_transaction_blocked=1
|
||||
if [ "$artifact_transaction_status" = "not_requested" ]; then
|
||||
artifact_transaction_status="blocked_artifact_transaction_verifier_lock_unavailable"
|
||||
artifact_transaction_phase="read_only_verifier_denied"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
inspect_active_artifact_transaction
|
||||
fi
|
||||
if [ "$artifact_chain_failed" -eq 0 ] \
|
||||
&& [ "$durable_receipt_present" -eq 1 ]; then
|
||||
if [ "$artifact_transaction_prior_pair_verified" -ne 1 ]; then
|
||||
artifact_chain_failed=1
|
||||
artifact_transaction_blocked=1
|
||||
artifact_transaction_status="blocked_fixed_wal_not_verified_for_durable_pair"
|
||||
artifact_transaction_phase="read_only_pair_verifier_denied"
|
||||
elif ! commit_current_artifact_transaction; then
|
||||
artifact_chain_failed=1
|
||||
artifact_transaction_blocked=1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
# Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on
|
||||
# unset state, even when interruption lands during the general guest repair.
|
||||
capture_manager_state
|
||||
@@ -901,16 +1209,18 @@ handle_executor_signal() {
|
||||
fi
|
||||
capture_manager_state || true
|
||||
if [ "$runtime_write_performed" -eq 1 ]; then
|
||||
if write_public_receipt && write_signal_readback; then
|
||||
exit 75
|
||||
if write_public_receipt; then
|
||||
mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}"
|
||||
write_signal_readback || true
|
||||
else
|
||||
mark_artifact_transaction_partial "receipt_write_failed_after_signal_${signal_name}"
|
||||
fi
|
||||
exit 74
|
||||
exit 1
|
||||
fi
|
||||
exit 75
|
||||
}
|
||||
|
||||
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
||||
&& [ "$immutable_replay" -eq 0 ]; then
|
||||
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
if [ "${EUID:-$(id -u)}" -ne 0 ]; then
|
||||
echo "BLOCKER host112_apply_requires_root" >&2
|
||||
exit 77
|
||||
@@ -920,18 +1230,30 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
||||
echo "BLOCKER host112_manager_single_flight_busy" >&2
|
||||
exit 75
|
||||
fi
|
||||
if ! inspect_active_artifact_transaction; then
|
||||
record_action "artifact_transaction_unresolved_prior_run_failed"
|
||||
artifact_chain_failed=1
|
||||
fi
|
||||
trap 'handle_executor_signal TERM' TERM
|
||||
trap 'handle_executor_signal INT' INT
|
||||
trap 'handle_executor_signal HUP' HUP
|
||||
fi
|
||||
|
||||
if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
||||
&& [ "$immutable_replay" -eq 0 ] \
|
||||
&& [ "$artifact_transaction_blocked" -eq 0 ]; then
|
||||
current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)"
|
||||
if [ "$current_default" != "graphical.target" ]; then
|
||||
runtime_write_performed=1
|
||||
state_write_performed=1
|
||||
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
|
||||
record_action "set_default_graphical_target"
|
||||
if authorize_runtime_mutation "set_default_graphical_target"; then
|
||||
runtime_write_performed=1
|
||||
state_write_performed=1
|
||||
if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then
|
||||
record_action "set_default_graphical_target"
|
||||
else
|
||||
record_action "set_default_graphical_target_failed"
|
||||
fi
|
||||
else
|
||||
record_action "set_default_graphical_target_failed"
|
||||
record_action "set_default_graphical_target_artifact_transaction_blocked_failed"
|
||||
fi
|
||||
fi
|
||||
|
||||
@@ -964,18 +1286,23 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
||||
fi
|
||||
is_uint "$last_retry_epoch" || last_retry_epoch=0
|
||||
if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then
|
||||
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
|
||||
if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then
|
||||
# A marker write can partially mutate state before a non-zero return.
|
||||
runtime_write_performed=1
|
||||
state_write_performed=1
|
||||
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
|
||||
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
||||
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
||||
record_action "retry_wazuh_indexer"
|
||||
if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then
|
||||
bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \
|
||||
systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true
|
||||
if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then
|
||||
record_action "retry_wazuh_indexer"
|
||||
else
|
||||
record_action "retry_wazuh_indexer_failed"
|
||||
fi
|
||||
else
|
||||
record_action "retry_wazuh_indexer_failed"
|
||||
record_action "wazuh_indexer_marker_write_failed"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_indexer_marker_write_failed_no_write"
|
||||
record_action "wazuh_indexer_artifact_transaction_blocked_failed"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_indexer_retry_cooldown"
|
||||
@@ -988,11 +1315,15 @@ now_epoch="$(date +%s)"
|
||||
evaluate_manager_preflight "$now_epoch" "$uptime_seconds"
|
||||
|
||||
if [ "$MODE" = "dry_run" ]; then
|
||||
case "$manager_preflight_status" in
|
||||
ready) manager_terminal="dry_run_ready" ;;
|
||||
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
|
||||
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
|
||||
esac
|
||||
if [ "$artifact_chain_failed" -eq 1 ]; then
|
||||
manager_terminal="dry_run_artifact_transaction_blocked"
|
||||
else
|
||||
case "$manager_preflight_status" in
|
||||
ready) manager_terminal="dry_run_ready" ;;
|
||||
already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;;
|
||||
*) manager_terminal="dry_run_${manager_preflight_status}" ;;
|
||||
esac
|
||||
fi
|
||||
elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \
|
||||
&& [ "$immutable_replay" -eq 1 ]; then
|
||||
manager_terminal="immutable_receipt_replay"
|
||||
@@ -1002,48 +1333,52 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
manager_terminal="idempotent_already_verified_healthy"
|
||||
;;
|
||||
ready)
|
||||
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
|
||||
if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then
|
||||
# The cooldown marker is a durable runtime state mutation even if the
|
||||
# following systemctl operation fails. Account for it immediately so
|
||||
# no failure path can emit a false no-write receipt.
|
||||
# marker writer returns non-zero. Account for the attempt immediately.
|
||||
runtime_write_performed=1
|
||||
state_write_performed=1
|
||||
manager_marker_write_performed=1
|
||||
manager_attempt_count=1
|
||||
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
|
||||
record_action "reset_failed_wazuh_manager"
|
||||
runtime_write_performed=1
|
||||
manager_runtime_write_performed=1
|
||||
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
|
||||
manager_start_exit=0
|
||||
else
|
||||
manager_start_exit=$?
|
||||
fi
|
||||
if wait_for_manager_verifier; then
|
||||
if [ "$manager_start_exit" = "0" ]; then
|
||||
record_action "start_wazuh_manager_verified"
|
||||
manager_terminal="verified_healthy"
|
||||
if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then
|
||||
manager_marker_write_performed=1
|
||||
manager_attempt_count=1
|
||||
if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then
|
||||
record_action "reset_failed_wazuh_manager"
|
||||
runtime_write_performed=1
|
||||
manager_runtime_write_performed=1
|
||||
if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then
|
||||
manager_start_exit=0
|
||||
else
|
||||
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
|
||||
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
|
||||
manager_start_exit=$?
|
||||
fi
|
||||
if wait_for_manager_verifier; then
|
||||
if [ "$manager_start_exit" = "0" ]; then
|
||||
record_action "start_wazuh_manager_verified"
|
||||
manager_terminal="verified_healthy"
|
||||
else
|
||||
record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed"
|
||||
manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"
|
||||
fi
|
||||
else
|
||||
record_action "start_wazuh_manager_verifier_failed"
|
||||
rollback_manager_to_inactive
|
||||
if [ "$rollback_verified" -eq 1 ]; then
|
||||
manager_terminal="rolled_back_verifier_failed"
|
||||
else
|
||||
manager_terminal="rollback_failed_degraded"
|
||||
fi
|
||||
fi
|
||||
else
|
||||
record_action "start_wazuh_manager_verifier_failed"
|
||||
rollback_manager_to_inactive
|
||||
if [ "$rollback_verified" -eq 1 ]; then
|
||||
manager_terminal="rolled_back_verifier_failed"
|
||||
else
|
||||
manager_terminal="rollback_failed_degraded"
|
||||
fi
|
||||
manager_start_exit="reset_failed_no_start"
|
||||
record_action "reset_failed_wazuh_manager_failed_no_start"
|
||||
manager_terminal="no_start_reset_failed"
|
||||
fi
|
||||
else
|
||||
manager_start_exit="reset_failed_no_start"
|
||||
record_action "reset_failed_wazuh_manager_failed_no_start"
|
||||
manager_terminal="no_start_reset_failed"
|
||||
record_action "wazuh_manager_marker_write_failed"
|
||||
manager_terminal="cooldown_marker_write_failed_degraded"
|
||||
fi
|
||||
else
|
||||
record_action "wazuh_manager_marker_write_failed_no_write"
|
||||
manager_terminal="no_write_cooldown_marker_write_failed"
|
||||
record_action "wazuh_manager_artifact_transaction_blocked_failed"
|
||||
manager_terminal="blocked_unresolved_artifact_transaction"
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
@@ -1159,13 +1494,22 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
if [ "$MODE" = "timer_apply" ]; then
|
||||
timer_artifact_policy="bounded_ring_128_for_runtime_mutation"
|
||||
fi
|
||||
if ! write_public_receipt; then
|
||||
if ! authorize_runtime_mutation "public_artifact_commit"; then
|
||||
artifact_chain_failed=1
|
||||
run_terminal="${run_terminal}_artifact_transaction_blocked"
|
||||
elif write_public_receipt; then
|
||||
artifact_receipt_ready=1
|
||||
else
|
||||
artifact_chain_failed=1
|
||||
mark_artifact_transaction_partial "receipt_write_or_disk_verification_failed"
|
||||
run_terminal="${run_terminal}_receipt_failed"
|
||||
fi
|
||||
else
|
||||
receipt_status="timer_no_runtime_mutation_aggregated"
|
||||
timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"
|
||||
fi
|
||||
elif [ "$durable_receipt_present" -eq 1 ]; then
|
||||
artifact_receipt_ready=1
|
||||
fi
|
||||
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
|
||||
if ! write_bounded_timer_observation; then
|
||||
@@ -1178,7 +1522,7 @@ fi
|
||||
trace_output="${TRACE_ID:-none}"
|
||||
run_output="${RUN_ID:-none}"
|
||||
work_item_output="${WORK_ITEM_ID:-none}"
|
||||
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
||||
readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed artifact_transaction_status=$artifact_transaction_status artifact_transaction_phase=$artifact_transaction_phase artifact_transaction_path=$ARTIFACT_TRANSACTION_PATH artifact_transaction_started=$artifact_transaction_started artifact_transaction_blocked=$artifact_transaction_blocked artifact_transaction_write_performed=$artifact_transaction_write_performed artifact_transaction_pair_verified=$artifact_transaction_pair_verified artifact_transaction_prior_pair_verified=$artifact_transaction_prior_pair_verified receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write"
|
||||
|
||||
canonical_readback_base() {
|
||||
printf '%s\n' "$readback" | awk '
|
||||
@@ -1188,6 +1532,9 @@ canonical_readback_base() {
|
||||
if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) {
|
||||
continue
|
||||
}
|
||||
if ($i ~ /^artifact_transaction_/) {
|
||||
continue
|
||||
}
|
||||
printf "%s%s", separator, $i
|
||||
separator = " "
|
||||
}
|
||||
@@ -1197,8 +1544,12 @@ canonical_readback_base() {
|
||||
}
|
||||
|
||||
emit_readback() {
|
||||
printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
|
||||
"$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \
|
||||
printf '%s artifact_transaction_status=%s artifact_transaction_phase=%s artifact_transaction_path=%s artifact_transaction_started=%s artifact_transaction_blocked=%s artifact_transaction_write_performed=%s artifact_transaction_pair_verified=%s artifact_transaction_prior_pair_verified=%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \
|
||||
"$(canonical_readback_base)" "$artifact_transaction_status" "$artifact_transaction_phase" \
|
||||
"$ARTIFACT_TRANSACTION_PATH" "$artifact_transaction_started" \
|
||||
"$artifact_transaction_blocked" "$artifact_transaction_write_performed" \
|
||||
"$artifact_transaction_pair_verified" "$artifact_transaction_prior_pair_verified" \
|
||||
"$durable_readback_status" "$durable_readback_path" \
|
||||
"$durable_readback_sha256" "$durable_readback_present" \
|
||||
"$durable_readback_identity_match" "$durable_readback_boot_match" \
|
||||
"$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \
|
||||
@@ -1206,24 +1557,39 @@ emit_readback() {
|
||||
}
|
||||
|
||||
if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
|
||||
if [ "$artifact_chain_failed" -eq 1 ]; then
|
||||
emit_readback
|
||||
exit 1
|
||||
elif [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then
|
||||
if [ "$timer_observation_failed" -eq 1 ]; then
|
||||
emit_readback
|
||||
exit 74
|
||||
exit 1
|
||||
fi
|
||||
elif [ "$MODE" = "timer_apply" ]; then
|
||||
if ! write_bounded_timer_mutation_readback; then
|
||||
mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed"
|
||||
emit_readback
|
||||
exit 74
|
||||
exit 1
|
||||
fi
|
||||
elif ! write_immutable_readback; then
|
||||
mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed"
|
||||
emit_readback
|
||||
exit 74
|
||||
exit 1
|
||||
fi
|
||||
if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then
|
||||
if ! commit_current_artifact_transaction; then
|
||||
mark_artifact_transaction_partial "artifact_pair_commit_failed"
|
||||
emit_readback
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
emit_readback
|
||||
|
||||
if [ "$MODE" = "dry_run" ]; then
|
||||
if [ "$artifact_chain_failed" -eq 1 ]; then
|
||||
exit 1
|
||||
fi
|
||||
case "$manager_preflight_status" in
|
||||
ready|already_verified_healthy) exit 0 ;;
|
||||
*) exit 1 ;;
|
||||
@@ -1237,6 +1603,8 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then
|
||||
fi
|
||||
[ "$guest_ready" -eq 1 ] \
|
||||
&& [ "$receipt_status" != "write_failed" ] \
|
||||
&& { [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ] \
|
||||
|| [ "$artifact_transaction_pair_verified" -eq 1 ]; } \
|
||||
&& { [ "$successful_terminal" = "verified_healthy" ] \
|
||||
|| [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; }
|
||||
exit $?
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
#!/usr/bin/env bash
|
||||
# Host 112 guest recovery installer.
|
||||
# Version: 2.1
|
||||
# Last modified: 2026-07-14 12:10 (Asia/Taipei)
|
||||
# Last modified: 2026-07-14 13:15 (Asia/Taipei)
|
||||
# Last modified by: Codex
|
||||
# Change: make installation transactional, verify the Windows99 direct route
|
||||
# and exact sudo policy, and keep installation separate from recovery runs.
|
||||
# Change: make timer shutdown state-aware on clean installs while preserving
|
||||
# transactional rollback, exact sudo policy, and recovery-run separation.
|
||||
|
||||
set -Eeuo pipefail
|
||||
|
||||
@@ -66,6 +66,9 @@ exact correlated apply/dry-run sudo rules, the host-110 forced readback key,
|
||||
and the existing Windows99 direct Agent99 authorization. It starts only the
|
||||
timer after all static gates pass; it never synchronously starts the recovery
|
||||
oneshot, reboots the host, or changes VM power.
|
||||
|
||||
HOST112_CONTROL_PUBLIC_KEY_PATH must point to an explicitly pre-staged public
|
||||
key file. The installer never discovers, reads, copies, or derives a private key.
|
||||
USAGE
|
||||
}
|
||||
|
||||
@@ -487,7 +490,9 @@ static_source_preflight() {
|
||||
for tool in bash cksum cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do
|
||||
command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; }
|
||||
done
|
||||
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do
|
||||
[ -f "$CONTROL_PUBLIC_KEY_PATH" ] \
|
||||
|| { echo "BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH" >&2; return 1; }
|
||||
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do
|
||||
[ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; return 1; }
|
||||
done
|
||||
bash -n "$READINESS_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"
|
||||
@@ -565,6 +570,25 @@ wait_for_recovery_service_idle() {
|
||||
done
|
||||
}
|
||||
|
||||
stop_recovery_timer_if_running() {
|
||||
local state
|
||||
state="$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
|
||||
case "${state:-unknown}" in
|
||||
active|activating|reloading|deactivating)
|
||||
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1
|
||||
;;
|
||||
inactive|failed|not-found|unknown)
|
||||
# A clean install has no timer unit yet. Treat that as an idempotent
|
||||
# no-op instead of asking systemd to stop a unit which does not exist.
|
||||
return 0
|
||||
;;
|
||||
*)
|
||||
echo "BLOCKER unexpected_recovery_timer_state=${state}" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
resolve_target_user
|
||||
|
||||
if [ "$MODE" = "check" ]; then
|
||||
@@ -612,7 +636,7 @@ TRANSACTION_ACTIVE=1
|
||||
trap 'handle_apply_failure $?' ERR
|
||||
trap 'handle_apply_failure 130' INT TERM
|
||||
|
||||
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1
|
||||
stop_recovery_timer_if_running
|
||||
wait_for_recovery_service_idle
|
||||
|
||||
install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET"
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
from copy import deepcopy
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
|
||||
|
||||
|
||||
def _function_source() -> str:
|
||||
source = PREFLIGHT.read_text(encoding="utf-8")
|
||||
start = source.index("function Get-AgentLivePreflightDecision")
|
||||
end = source.index("function Get-TaskReadback", start)
|
||||
return source[start:end]
|
||||
|
||||
|
||||
def _base_case() -> dict[str, Any]:
|
||||
return {
|
||||
"agentRootPresent": True,
|
||||
"manifest": {
|
||||
"exists": True,
|
||||
"sourceRevision": "a" * 40,
|
||||
"runtimeMatched": True,
|
||||
"fileCount": 14,
|
||||
"mismatchCount": 0,
|
||||
"parseError": "",
|
||||
},
|
||||
"taskFailureCount": 0,
|
||||
"relayListenerCount": 1,
|
||||
"requiredEvidenceStaleCount": 0,
|
||||
"requiredEvidenceParseFailureCount": 0,
|
||||
"requiredEvidenceContentFailureCount": 0,
|
||||
"selfHealthSeverity": "ok",
|
||||
"selfHealthPerformanceIssueCount": 0,
|
||||
"staleAlertExecutionCount": 0,
|
||||
"alertIncomingCount": 0,
|
||||
"pendingCommandCount": 0,
|
||||
}
|
||||
|
||||
|
||||
def _with_overrides(**overrides: Any) -> dict[str, Any]:
|
||||
case = deepcopy(_base_case())
|
||||
manifest_overrides = overrides.pop("manifest", None)
|
||||
case.update(overrides)
|
||||
if manifest_overrides:
|
||||
case["manifest"].update(manifest_overrides)
|
||||
return case
|
||||
|
||||
|
||||
def _as_list(value: Any) -> list[str]:
|
||||
if value is None:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return [str(item) for item in value]
|
||||
return [str(value)]
|
||||
|
||||
|
||||
def _run_decision(powershell: str, case: dict[str, Any]) -> dict[str, Any]:
|
||||
fixture = json.dumps(case, separators=(",", ":"))
|
||||
command = f"""
|
||||
{_function_source()}
|
||||
$case = @'
|
||||
{fixture}
|
||||
'@ | ConvertFrom-Json
|
||||
$parameters = @{{
|
||||
AgentRootPresent = [bool]$case.agentRootPresent
|
||||
Manifest = $case.manifest
|
||||
ExpectedRuntimeFileCount = 14
|
||||
TaskFailureCount = [int]$case.taskFailureCount
|
||||
RelayListenerCount = [int]$case.relayListenerCount
|
||||
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
||||
RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount
|
||||
RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount
|
||||
SelfHealthSeverity = [string]$case.selfHealthSeverity
|
||||
SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount
|
||||
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
|
||||
AlertIncomingCount = [int]$case.alertIncomingCount
|
||||
PendingCommandCount = [int]$case.pendingCommandCount
|
||||
}}
|
||||
Get-AgentLivePreflightDecision @parameters |
|
||||
ConvertTo-Json -Depth 6 -Compress
|
||||
"""
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
payload = next(
|
||||
line for line in reversed(result.stdout.splitlines()) if line.startswith("{")
|
||||
)
|
||||
return json.loads(payload)
|
||||
|
||||
|
||||
def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> None:
|
||||
source = PREFLIGHT.read_text(encoding="utf-8")
|
||||
decision = _function_source()
|
||||
|
||||
assert '$warningReasons += "self_health_warning"' in decision
|
||||
assert '$warningReasons += "performance_warning"' in decision
|
||||
assert '"critical" { $blockingReasons += "self_health_critical" }' in decision
|
||||
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
|
||||
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
|
||||
assert '$blockingReasons += "runtime_manifest_mismatch"' in decision
|
||||
assert '$blockingReasons += "scheduled_task_unhealthy"' in decision
|
||||
assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision
|
||||
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
||||
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
||||
assert "warningReasons = $warningReasons" in source
|
||||
assert "selfHealthPerformanceIssues = @(" in source
|
||||
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
|
||||
assert 'if ($reason -match "^[a-z0-9_]{1,80}$")' in source
|
||||
assert (
|
||||
"[int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount"
|
||||
in source
|
||||
)
|
||||
assert 'if ($result.summary.deploymentEligible)' in source
|
||||
assert 'if ($selfHealthSeverity -ne "ok")' not in source
|
||||
|
||||
|
||||
def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
cases = (
|
||||
(_base_case(), True, (), ()),
|
||||
(
|
||||
_with_overrides(
|
||||
selfHealthSeverity="warning",
|
||||
selfHealthPerformanceIssueCount=1,
|
||||
),
|
||||
True,
|
||||
(),
|
||||
("self_health_warning", "performance_warning"),
|
||||
),
|
||||
(
|
||||
_with_overrides(selfHealthSeverity="critical"),
|
||||
False,
|
||||
("self_health_critical",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(taskFailureCount=1),
|
||||
False,
|
||||
("scheduled_task_unhealthy",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}),
|
||||
False,
|
||||
("runtime_manifest_mismatch",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(manifest={"sourceRevision": "not-a-full-sha"}),
|
||||
False,
|
||||
("runtime_manifest_identity_invalid",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(relayListenerCount=0),
|
||||
False,
|
||||
("sre_alert_relay_not_listening",),
|
||||
(),
|
||||
),
|
||||
)
|
||||
|
||||
for case, expected_eligible, expected_blockers, expected_warnings in cases:
|
||||
result = _run_decision(powershell, case)
|
||||
assert result["deploymentEligible"] is expected_eligible
|
||||
assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers)
|
||||
assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings)
|
||||
|
||||
preflight_path = str(PREFLIGHT).replace("'", "''")
|
||||
parser_command = (
|
||||
"$tokens=$null; $errors=$null; "
|
||||
"[System.Management.Automation.Language.Parser]::ParseFile("
|
||||
f"'{preflight_path}', [ref]$tokens, [ref]$errors) | Out-Null; "
|
||||
"if ($errors.Count -gt 0) { "
|
||||
"$errors | ForEach-Object { $_.Message }; exit 1 }"
|
||||
)
|
||||
parsed = subprocess.run(
|
||||
[
|
||||
powershell,
|
||||
"-NoProfile",
|
||||
"-NonInteractive",
|
||||
"-Command",
|
||||
parser_command,
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert parsed.returncode == 0, parsed.stdout + parsed.stderr
|
||||
@@ -109,7 +109,15 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None:
|
||||
assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight
|
||||
assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight
|
||||
assert 'Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"' in preflight
|
||||
assert 'if ($selfHealthSeverity -ne "ok")' in preflight
|
||||
assert "function Get-AgentLivePreflightDecision" in preflight
|
||||
assert '"warning" {' in preflight
|
||||
assert '$warningReasons += "self_health_warning"' in preflight
|
||||
assert '$warningReasons += "performance_warning"' in preflight
|
||||
assert '"critical" { $blockingReasons += "self_health_critical" }' in preflight
|
||||
assert 'default { $blockingReasons += "self_health_unclassified" }' in preflight
|
||||
assert 'if ($selfHealthSeverity -ne "ok")' not in preflight
|
||||
assert 'Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"' in preflight
|
||||
assert 'Write-Output "WARNING_REASONS=$($warningReasons -join \',\')"' in preflight
|
||||
assert "runtime-manifest.json" in preflight
|
||||
assert "Get-ScheduledTask" in preflight
|
||||
assert "Get-NetTCPConnection -LocalPort 8787" in preflight
|
||||
@@ -143,6 +151,9 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() ->
|
||||
"updatesSeen",
|
||||
"processedCount",
|
||||
"alertedCount",
|
||||
"selfHealthPerformanceSeverity",
|
||||
"selfHealthPerformanceIssueCount",
|
||||
"selfHealthPerformanceIssues",
|
||||
):
|
||||
assert safe_field in preflight
|
||||
for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"):
|
||||
|
||||
@@ -7,6 +7,8 @@ import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SENDER = ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
|
||||
RECEIVER = (
|
||||
@@ -259,6 +261,178 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
|
||||
assert "[bool]$prior.relayReload.newGenerationVerified" in source
|
||||
|
||||
|
||||
def test_receiver_creates_canonical_success_receipt_once_with_atomic_no_replace() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
writer = source[
|
||||
source.index("function Write-AgentRemoteDeployReceipt") : source.index(
|
||||
"function Write-AgentResultAndExit"
|
||||
)
|
||||
]
|
||||
|
||||
assert (
|
||||
source.count(
|
||||
"Write-AgentRemoteDeployReceipt $receiptPath $successReceipt"
|
||||
)
|
||||
== 1
|
||||
)
|
||||
assert 'throw "immutable_receipt_path_already_exists"' in writer
|
||||
assert writer.count("[IO.File]::Move($temporary, $Path)") == 1
|
||||
assert "Move-Item -LiteralPath $temporary -Destination $Path -Force" not in writer
|
||||
assert "$json = $Receipt | ConvertTo-Json -Depth 12" in writer
|
||||
assert "$null = $storedJson | ConvertFrom-Json" in writer
|
||||
|
||||
atomic_move = writer.index("[IO.File]::Move($temporary, $Path)")
|
||||
catch_boundary = writer.index("} catch {", atomic_move)
|
||||
assert writer[atomic_move:catch_boundary].strip().endswith(
|
||||
"[IO.File]::Move($temporary, $Path)"
|
||||
)
|
||||
|
||||
|
||||
def test_receiver_uses_unique_filesystem_safe_attempt_failure_receipts() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
assert '"yyyyMMdd\'T\'HHmmssfffffff\'Z\'"' in source
|
||||
assert "[Globalization.CultureInfo]::InvariantCulture" in source
|
||||
assert (
|
||||
'$attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString(\'N\'))"'
|
||||
in source
|
||||
)
|
||||
assert (
|
||||
'"agent99-remote-deploy-$stageToken-attempt-$attemptToken.json"'
|
||||
in source
|
||||
)
|
||||
assert source.count(
|
||||
"Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt"
|
||||
) >= 5
|
||||
assert "yyyyMMddTHHmmssfffffffK" not in source
|
||||
assert 'ToString("K")' not in source
|
||||
|
||||
|
||||
def test_success_receipt_failure_rolls_back_before_durable_attempt_receipt() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
success_construction = source.index(
|
||||
'$script:ReceiverOperationPhase = "success_receipt_construction"'
|
||||
)
|
||||
canonical_write = source.index(
|
||||
"Write-AgentRemoteDeployReceipt $receiptPath $successReceipt"
|
||||
)
|
||||
failure_branch = source.index("if ($failurePhase)", canonical_write)
|
||||
rollback = source.index(
|
||||
"$rollback = Restore-AgentBundle $backupPath $beforeFilePresence",
|
||||
failure_branch,
|
||||
)
|
||||
rollback_restart = source.index(
|
||||
"Invoke-AgentRelayTaskRestart -ForbiddenGenerations "
|
||||
"$rollbackForbiddenGenerations",
|
||||
rollback,
|
||||
)
|
||||
failure_receipt = source.index(
|
||||
"Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt",
|
||||
rollback_restart,
|
||||
)
|
||||
|
||||
assert success_construction < canonical_write < failure_branch
|
||||
assert failure_branch < rollback < rollback_restart < failure_receipt
|
||||
assert (
|
||||
'$failurePhase = $script:ReceiverOperationPhase' in source
|
||||
)
|
||||
assert (
|
||||
'$failurePhase -in @("success_receipt_construction", '
|
||||
'"success_receipt_write")'
|
||||
in source
|
||||
)
|
||||
assert '"rolled_back_success_receipt_write_failed"' in source
|
||||
assert '"rollback_failed_after_success_receipt_write_failure"' in source
|
||||
assert "canonicalReceiptPath = $receiptPath" in source
|
||||
assert "attemptReceiptPath = $attemptReceiptPath" in source
|
||||
assert "durableReceiptWritten = $true" in source
|
||||
|
||||
|
||||
def test_existing_canonical_receipt_blocks_live_promotion_without_overwrite() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
canonical_guard = source.index("if ($canonicalReceiptExists)")
|
||||
validate_only = source.index(
|
||||
"$validate = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
"-SourceRevision $sourceRevision -ValidateOnly"
|
||||
)
|
||||
live_deploy = source.index(
|
||||
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
|
||||
"-SourceRevision $sourceRevision"
|
||||
)
|
||||
assert canonical_guard < validate_only < live_deploy
|
||||
assert '"blocked_canonical_receipt_invalid_no_promotion"' in source
|
||||
assert (
|
||||
'"blocked_canonical_success_receipt_runtime_drift_no_promotion"'
|
||||
in source
|
||||
)
|
||||
assert '"blocked_canonical_receipt_reserved_no_promotion"' in source
|
||||
assert (
|
||||
'nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_'
|
||||
'trace_run_work_item_identity"'
|
||||
in source
|
||||
)
|
||||
|
||||
|
||||
def test_outer_catch_reports_conservative_operation_boundaries() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
transport_failure = source.index('status = "transport_receiver_failed"')
|
||||
outer_catch = source[
|
||||
source.rfind("} catch {", 0, transport_failure) : source.index(
|
||||
"} finally {", transport_failure
|
||||
)
|
||||
]
|
||||
|
||||
for tracker in (
|
||||
"$script:ReceiverRemoteWritePerformed",
|
||||
"$script:ReceiverLiveRuntimeWritePerformed",
|
||||
"$script:ReceiverLivePromotionAttempted",
|
||||
"$script:ReceiverLivePromotionPerformed",
|
||||
"$script:ReceiverRollbackAttempted",
|
||||
"$script:ReceiverRollbackVerified",
|
||||
"$script:ReceiverScheduledTaskRestartPerformed",
|
||||
):
|
||||
assert tracker in outer_catch
|
||||
assert "operationBoundaries = [pscustomobject]@{" in outer_catch
|
||||
assert "secretValueRead = $false" in outer_catch
|
||||
assert "privateKeyValueRead = $false" in outer_catch
|
||||
assert "tokenValueRead = $false" in outer_catch
|
||||
assert "environmentSecretRead = $false" in outer_catch
|
||||
assert "rawEnvelopeStored = $false" in outer_catch
|
||||
assert "$outerErrorType = $_.Exception.GetType().Name" in outer_catch
|
||||
assert "$script:ReceiverAttemptReceiptPath" in outer_catch
|
||||
assert (
|
||||
"Write-AgentRemoteDeployReceipt "
|
||||
"$script:ReceiverAttemptReceiptPath $outerFailure"
|
||||
) in outer_catch
|
||||
assert "durableReceiptWritten = $false" in outer_catch
|
||||
assert "$outerFailure.durableReceiptWritten = $true" in outer_catch
|
||||
assert "$outerFailure.durableReceiptWritten = $false" in outer_catch
|
||||
|
||||
|
||||
def test_receiver_parses_with_powershell_ast_when_runtime_is_available() -> None:
|
||||
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
||||
if powershell is None:
|
||||
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
||||
|
||||
receiver_path = str(RECEIVER).replace("'", "''")
|
||||
parser_command = (
|
||||
"$tokens=$null; $errors=$null; "
|
||||
"[System.Management.Automation.Language.Parser]::ParseFile("
|
||||
f"'{receiver_path}', [ref]$tokens, [ref]$errors) | Out-Null; "
|
||||
"if ($errors.Count -gt 0) { "
|
||||
"$errors | ForEach-Object { $_.Message }; exit 1 }"
|
||||
)
|
||||
result = subprocess.run(
|
||||
[powershell, "-NoProfile", "-NonInteractive", "-Command", parser_command],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None:
|
||||
source = RECEIVER.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -180,37 +181,41 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
|
||||
assert "return 1" in marker_function
|
||||
|
||||
manager_start = source.index(
|
||||
' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then'
|
||||
' if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then'
|
||||
)
|
||||
manager_end = source.index(" ;;", manager_start)
|
||||
manager_branch = source[manager_start:manager_end]
|
||||
marker_failure_at = manager_branch.index(
|
||||
' else\n record_action "wazuh_manager_marker_write_failed_no_write"'
|
||||
' else\n record_action "wazuh_manager_marker_write_failed"'
|
||||
)
|
||||
success = manager_branch[:marker_failure_at]
|
||||
failure = manager_branch[marker_failure_at:]
|
||||
authorization_at = success.index("authorize_runtime_mutation")
|
||||
marker_write_at = success.index(
|
||||
'write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"'
|
||||
)
|
||||
marker_success_at = success.index(
|
||||
' manager_marker_write_performed=1'
|
||||
' manager_marker_write_performed=1'
|
||||
)
|
||||
reset_at = success.index("systemctl reset-failed wazuh-manager.service")
|
||||
assert marker_success_at < reset_at
|
||||
assert "runtime_write_performed=1" in success[:reset_at]
|
||||
assert "state_write_performed=1" in success[:reset_at]
|
||||
assert authorization_at < marker_write_at < marker_success_at < reset_at
|
||||
assert "runtime_write_performed=1" in success[:marker_write_at]
|
||||
assert "state_write_performed=1" in success[:marker_write_at]
|
||||
assert "systemctl reset-failed wazuh-manager.service" in success
|
||||
assert "systemctl start wazuh-manager.service" in success
|
||||
assert 'bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
|
||||
assert "manager_attempt_count=1" in success
|
||||
assert "systemctl reset-failed wazuh-manager.service" not in failure
|
||||
assert "systemctl start wazuh-manager.service" not in failure
|
||||
assert "wazuh_manager_marker_write_failed_no_write" in failure
|
||||
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
|
||||
assert "wazuh_manager_marker_write_failed" in failure
|
||||
assert 'manager_terminal="cooldown_marker_write_failed_degraded"' in failure
|
||||
assert "manager_marker_write_performed=1" not in failure
|
||||
|
||||
assert "state_write_performed=$state_write_performed" in source
|
||||
assert "manager_marker_write_performed=$manager_marker_write_performed" in source
|
||||
|
||||
indexer_start = source.index(
|
||||
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
|
||||
' if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then'
|
||||
)
|
||||
indexer_end = source.index(" else\n record_action", indexer_start)
|
||||
indexer_branch = source[indexer_start:indexer_end]
|
||||
@@ -219,7 +224,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
|
||||
assert "systemctl start wazuh-indexer.service" in indexer_success
|
||||
assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure
|
||||
assert "systemctl start wazuh-indexer.service" not in indexer_failure
|
||||
assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure
|
||||
assert "wazuh_indexer_marker_write_failed" in indexer_failure
|
||||
|
||||
|
||||
def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
@@ -271,6 +276,46 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No
|
||||
assert "latest_readback_pointer_written=%s" in source
|
||||
|
||||
|
||||
def test_runtime_mutations_are_guarded_by_a_fixed_durable_artifact_wal() -> None:
|
||||
source = read(READINESS)
|
||||
transaction = source[
|
||||
source.index("artifact_paths_for_current_identity() {") : source.index(
|
||||
"load_durable_receipt() {"
|
||||
)
|
||||
]
|
||||
|
||||
assert 'ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt"' in source
|
||||
assert "schema_version=host112_active_artifact_transaction_v1" in transaction
|
||||
assert 'mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"' in transaction
|
||||
assert 'stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"' in transaction
|
||||
assert 'artifact_transaction_status="blocked_unresolved_prior_transaction"' in transaction
|
||||
assert 'write_artifact_transaction_record "pending_artifact_commit"' in transaction
|
||||
assert 'write_artifact_transaction_record "partial_pending_public_artifact_commit"' in transaction
|
||||
assert 'write_artifact_transaction_record "committed_verified_pair"' in transaction
|
||||
assert "verify_artifact_pair_paths" in transaction
|
||||
|
||||
apply = source[source.index(' exec 9>"$MANAGER_LOCK_FILE"') :]
|
||||
inspect_at = apply.index("inspect_active_artifact_transaction")
|
||||
first_mutation_at = apply.index("systemctl set-default graphical.target")
|
||||
assert inspect_at < first_mutation_at
|
||||
for phase in (
|
||||
"set_default_graphical_target",
|
||||
"wazuh_indexer_cooldown_marker",
|
||||
"wazuh_manager_cooldown_marker",
|
||||
"public_artifact_commit",
|
||||
):
|
||||
assert f'authorize_runtime_mutation "{phase}"' in source
|
||||
|
||||
immutable_writer = source[
|
||||
source.index("write_immutable_readback() {") : source.index(
|
||||
'boot_id="$(cat /proc/sys/kernel/random/boot_id'
|
||||
)
|
||||
]
|
||||
assert "durable_readback_present=1" not in immutable_writer
|
||||
assert "durable_readback_identity_match=1" not in immutable_writer
|
||||
assert "verify_current_readback_from_disk" in immutable_writer
|
||||
|
||||
|
||||
def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -352,7 +397,9 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No
|
||||
assert "trap 'handle_executor_signal TERM' TERM" in source
|
||||
assert "trap 'handle_executor_signal INT' INT" in source
|
||||
assert "trap 'handle_executor_signal HUP' HUP" in source
|
||||
assert "if write_public_receipt && write_signal_readback; then" in source
|
||||
assert "if write_public_receipt; then" in source
|
||||
assert 'mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}"' in source
|
||||
assert "write_signal_readback || true" in source
|
||||
assert "action_count=${#actions[@]} actions=$receipt_action_text" in source
|
||||
assert "schema_version=host112_guest_recovery_signal_readback_v1" in source
|
||||
assert 'ln "$tmp" "$signal_readback_path"' in source
|
||||
@@ -394,8 +441,9 @@ def test_apply_success_is_bound_to_same_run_durable_terminal() -> None:
|
||||
assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate
|
||||
assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate
|
||||
guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]')
|
||||
receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier)
|
||||
receipt_commit = source.index("elif write_public_receipt; then", guest_verifier)
|
||||
assert guest_verifier < receipt_commit
|
||||
assert "commit_current_artifact_transaction" in source[receipt_commit:]
|
||||
assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source
|
||||
assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source
|
||||
assert 'action_failure_observed=1' in source
|
||||
@@ -471,7 +519,13 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
|
||||
assert "-not $dryRun.artifactTransactionBlocked" in function
|
||||
assert '$dryRun.artifactTransactionStatus -eq "prior_pair_verified"' in function
|
||||
assert 'receiptStatus -eq "written_immutable"' in function
|
||||
assert '$applyReceipt.artifactTransactionStatus -eq "committed_verified_pair"' in function
|
||||
assert "$applyReceipt.artifactTransactionPairVerified" in function
|
||||
assert "$durableReceiptReadback.artifactTransactionPriorPairVerified" in function
|
||||
assert '$durableReceiptReadback.artifactTransactionStatus -eq "existing_pair_verified"' in function
|
||||
assert '$after.managerVerified' in function
|
||||
assert 'name = "wazuh_manager_service_active"' in function
|
||||
assert 'name = "wazuh_analysisd_process_present"' in function
|
||||
@@ -570,11 +624,56 @@ def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None:
|
||||
|
||||
assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy
|
||||
assert 'name = "host112_canonical_direct_route"' in deploy
|
||||
assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy
|
||||
assert 'Host112 canonical SSH and VMX config post-verifier failed.' in deploy
|
||||
assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap
|
||||
assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap
|
||||
assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap
|
||||
assert 'Host112 canonical SSH and VMX config bootstrap verifier failed.' in bootstrap
|
||||
for config in (generic_config, host_config):
|
||||
assert '"192.168.0.112": "kali"' in config
|
||||
assert '"directHosts"' in config
|
||||
assert '"192.168.0.112"' in config
|
||||
|
||||
|
||||
def test_deploy_and_bootstrap_canonicalize_host112_ssd_vmx_identity() -> None:
|
||||
deploy = read(ROOT / "agent99-deploy.ps1")
|
||||
bootstrap = read(BOOTSTRAP)
|
||||
contract = read(ROOT / "agent99-contract-check.ps1")
|
||||
synthetic = read(ROOT / "agent99-synthetic-tests.ps1")
|
||||
host_config = read(ROOT / "agent99.config.99.example.json")
|
||||
canonical_ps = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
|
||||
canonical_json = r"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"
|
||||
policy = json.loads(host_config)
|
||||
host112_rows = [
|
||||
row
|
||||
for row in policy["vms"]
|
||||
if row.get("host") == "192.168.0.112" or row.get("name") == "host112-kali"
|
||||
]
|
||||
|
||||
assert canonical_json in host_config
|
||||
assert host112_rows == [
|
||||
{
|
||||
"name": "host112-kali",
|
||||
"host": "192.168.0.112",
|
||||
"vmx": canonical_ps,
|
||||
"priority": 30,
|
||||
}
|
||||
]
|
||||
assert canonical_ps not in deploy
|
||||
assert canonical_ps not in bootstrap
|
||||
assert "Get-AgentHost112CanonicalVm $policyConfigPath" in deploy
|
||||
assert "Get-AgentHost112CanonicalVm $PolicyPath" in bootstrap
|
||||
bootstrap_preflight = (
|
||||
'$null = Get-AgentHost112CanonicalVm $host112BootstrapPolicyPath'
|
||||
)
|
||||
bootstrap_first_write = "foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir"
|
||||
assert bootstrap.index(bootstrap_preflight) < bootstrap.index(bootstrap_first_write)
|
||||
assert 'name = "host112_canonical_vmx_path"' in deploy
|
||||
assert "Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf" in deploy
|
||||
assert "Host112 canonical SSH and VMX config post-verifier failed." in deploy
|
||||
assert "Host112 canonical SSH and VMX config bootstrap verifier failed." in bootstrap
|
||||
assert '$persistedHost112Vm.Count -ne 1' in deploy
|
||||
assert '$persistedHost112Vm.Count -ne 1' in bootstrap
|
||||
assert 'Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in deploy
|
||||
assert 'Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in bootstrap
|
||||
assert 'recovery:host112_canonical_vmx' in contract
|
||||
assert 'recovery:host112_canonical_vmx' in synthetic
|
||||
|
||||
@@ -81,6 +81,21 @@ if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]
|
||||
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
|
||||
fi
|
||||
/bin/mv "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ln",
|
||||
"""#!/usr/bin/env bash
|
||||
set -u
|
||||
destination=""
|
||||
for argument in "$@"; do destination="$argument"; done
|
||||
if [ "${FAKE_RECEIPT_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */receipts/*.txt ]]; then
|
||||
exit 1
|
||||
fi
|
||||
if [ "${FAKE_READBACK_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */readbacks/*.txt ]] && [[ "$destination" != *.signal.txt ]]; then
|
||||
exit 1
|
||||
fi
|
||||
/bin/ln "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
@@ -118,6 +133,11 @@ printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
|
||||
"""#!/usr/bin/env bash
|
||||
set -u
|
||||
state="${FAKE_HOST_STATE_DIR:?}"
|
||||
require_wal() {
|
||||
if [ ! -f "${HOST112_RECOVERY_STATE_DIR:?}/active-artifact-transaction.txt" ]; then
|
||||
: >"$state/mutation_without_wal"
|
||||
fi
|
||||
}
|
||||
cmd="${1:-}"
|
||||
[ "$#" -gt 0 ] && shift
|
||||
case "$cmd" in
|
||||
@@ -154,10 +174,12 @@ case "$cmd" in
|
||||
;;
|
||||
get-default) printf '%s\n' 'graphical.target' ;;
|
||||
set-default)
|
||||
require_wal
|
||||
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
|
||||
exit "${FAKE_SET_DEFAULT_RC:-0}"
|
||||
;;
|
||||
start)
|
||||
require_wal
|
||||
unit="${1:-}"
|
||||
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
|
||||
if [ "$unit" = "lightdm.service" ]; then
|
||||
@@ -168,9 +190,10 @@ case "$cmd" in
|
||||
fi
|
||||
;;
|
||||
enable)
|
||||
require_wal
|
||||
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
|
||||
;;
|
||||
reset-failed|stop) ;;
|
||||
reset-failed|stop) require_wal ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
""",
|
||||
@@ -211,6 +234,81 @@ def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str
|
||||
)
|
||||
|
||||
|
||||
def _dry_run(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(script),
|
||||
"--dry-run",
|
||||
"--trace-id",
|
||||
"trace:runtime-harness",
|
||||
"--run-id",
|
||||
"run:runtime-harness",
|
||||
"--work-item-id",
|
||||
"HOST112-RUNTIME-HARNESS",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
def test_post_apply_dry_run_read_only_verifies_fixed_wal_and_immutable_pair(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, _ = _runtime_harness(tmp_path)
|
||||
|
||||
applied = _apply(script, env)
|
||||
assert applied.returncode == 0, applied.stdout + applied.stderr
|
||||
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
||||
transaction_before = transaction.read_bytes()
|
||||
|
||||
verified = _dry_run(script, env)
|
||||
output = verified.stdout + verified.stderr
|
||||
assert verified.returncode == 0, output
|
||||
assert "durable_readback_status=loaded_immutable_verified " in output
|
||||
assert "artifact_transaction_status=existing_pair_verified" in output
|
||||
assert "artifact_transaction_started=0" in output
|
||||
assert "artifact_transaction_blocked=0" in output
|
||||
assert "artifact_transaction_write_performed=0" in output
|
||||
assert "artifact_transaction_pair_verified=1" in output
|
||||
assert "artifact_transaction_prior_pair_verified=1" in output
|
||||
assert transaction.read_bytes() == transaction_before
|
||||
|
||||
|
||||
def test_pre_apply_dry_run_blocks_unresolved_prior_transaction(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, fake_state = _runtime_harness(tmp_path)
|
||||
state = tmp_path / "state"
|
||||
state.mkdir()
|
||||
transaction = state / "active-artifact-transaction.txt"
|
||||
unresolved = (
|
||||
"schema_version=host112_active_artifact_transaction_v1 "
|
||||
"updated_at=2026-07-14T12:01:00+08:00 "
|
||||
"status=pending_artifact_commit phase=prior_runtime_mutation "
|
||||
"trace_id=trace:prior run_id=run:prior "
|
||||
"work_item_id=HOST112-PRIOR boot_id=boot:prior "
|
||||
f"receipt_path={state / 'receipts' / 'prior.txt'} "
|
||||
f"readback_path={state / 'readbacks' / 'prior.txt'} "
|
||||
"pair_verified=0\n"
|
||||
)
|
||||
transaction.write_text(unresolved, encoding="utf-8")
|
||||
Path(env["HOST112_MANAGER_LOCK_FILE"]).touch()
|
||||
|
||||
dry_run = _dry_run(script, env)
|
||||
output = dry_run.stdout + dry_run.stderr
|
||||
assert dry_run.returncode == 1, output
|
||||
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
|
||||
assert "artifact_transaction_blocked=1" in output
|
||||
assert "runtime_write_performed=0" in output
|
||||
assert "receipt_write_performed=0" in output
|
||||
assert transaction.read_text(encoding="utf-8") == unresolved
|
||||
assert not (fake_state / "mutations.log").exists()
|
||||
|
||||
|
||||
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -223,16 +321,28 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
||||
assert failed.returncode == 1
|
||||
assert "runtime_write_performed=1" in failed_output
|
||||
assert "receipt_status=written_immutable" in failed_output
|
||||
assert "durable_readback_status=written_immutable" in failed_output
|
||||
assert "durable_readback_status=written_immutable " in failed_output
|
||||
assert (
|
||||
"durable_receipt_terminal=execution_failed_or_unattributed_"
|
||||
"idempotent_already_verified_healthy"
|
||||
) in failed_output
|
||||
assert "guest_ready=1" in failed_output
|
||||
assert "action_failure_observed=1" in failed_output
|
||||
assert "artifact_transaction_status=committed_verified_pair" in failed_output
|
||||
assert "artifact_transaction_pair_verified=1" in failed_output
|
||||
assert not (fake_state / "mutation_without_wal").exists()
|
||||
|
||||
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
||||
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
|
||||
stored_readback = readback.read_text(encoding="utf-8")
|
||||
for derived in (
|
||||
"durable_readback_present=1",
|
||||
"durable_readback_identity_match=1",
|
||||
"durable_readback_boot_match=1",
|
||||
"durable_readback_receipt_link_match=1",
|
||||
"durable_readback_terminal_match=1",
|
||||
):
|
||||
assert derived not in stored_readback
|
||||
receipt_before = receipt.read_bytes()
|
||||
readback_before = readback.read_bytes()
|
||||
|
||||
@@ -250,6 +360,114 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
||||
assert readback.read_bytes() == readback_before
|
||||
|
||||
|
||||
def test_immutable_replay_cannot_ignore_newer_unresolved_fixed_transaction(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, fake_state = _runtime_harness(tmp_path)
|
||||
|
||||
first = _apply(script, env)
|
||||
assert first.returncode == 0, first.stdout + first.stderr
|
||||
state = tmp_path / "state"
|
||||
receipt = next((state / "receipts").glob("*.txt"))
|
||||
readback = next((state / "readbacks").glob("*.txt"))
|
||||
receipt_before = receipt.read_bytes()
|
||||
readback_before = readback.read_bytes()
|
||||
transaction = state / "active-artifact-transaction.txt"
|
||||
unresolved = (
|
||||
"schema_version=host112_active_artifact_transaction_v1 "
|
||||
"updated_at=2026-07-14T12:01:00+08:00 "
|
||||
"status=pending_artifact_commit phase=newer_runtime_mutation "
|
||||
"trace_id=trace:newer run_id=run:newer "
|
||||
"work_item_id=HOST112-NEWER boot_id=boot:newer "
|
||||
f"receipt_path={state / 'receipts' / 'newer.txt'} "
|
||||
f"readback_path={state / 'readbacks' / 'newer.txt'} "
|
||||
"pair_verified=0\n"
|
||||
)
|
||||
transaction.write_text(unresolved, encoding="utf-8")
|
||||
mutations = fake_state / "mutations.log"
|
||||
mutations_before = mutations.read_bytes()
|
||||
|
||||
replay = _apply(script, env)
|
||||
output = replay.stdout + replay.stderr
|
||||
assert replay.returncode == 1, output
|
||||
assert "receipt_status=reused_immutable" in output
|
||||
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
|
||||
assert "artifact_transaction_blocked=1" in output
|
||||
assert "artifact_transaction_pair_verified=0" in output
|
||||
assert transaction.read_text(encoding="utf-8") == unresolved
|
||||
assert receipt.read_bytes() == receipt_before
|
||||
assert readback.read_bytes() == readback_before
|
||||
assert mutations.read_bytes() == mutations_before
|
||||
|
||||
|
||||
def test_receipt_failure_leaves_durable_partial_without_readback_and_blocks_retry(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, fake_state = _runtime_harness(tmp_path)
|
||||
env.update(
|
||||
{
|
||||
"FAKE_LIGHTDM_START_RC": "124",
|
||||
"FAKE_RECEIPT_LINK_FAIL": "1",
|
||||
}
|
||||
)
|
||||
|
||||
failed = _apply(script, env)
|
||||
output = failed.stdout + failed.stderr
|
||||
assert failed.returncode == 1
|
||||
assert "runtime_write_performed=1" in output
|
||||
assert "receipt_status=write_failed" in output
|
||||
assert "artifact_transaction_status=partial_pending_public_artifact_commit" in output
|
||||
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
||||
stored = transaction.read_text(encoding="utf-8")
|
||||
assert "status=partial_pending_public_artifact_commit" in stored
|
||||
assert "phase=receipt_write_or_disk_verification_failed" in stored
|
||||
assert not (tmp_path / "state" / "readbacks").exists()
|
||||
assert not (fake_state / "mutation_without_wal").exists()
|
||||
|
||||
mutations = fake_state / "mutations.log"
|
||||
mutations.write_text("", encoding="utf-8")
|
||||
retry_env = env.copy()
|
||||
retry_env.pop("FAKE_RECEIPT_LINK_FAIL")
|
||||
retry = _apply(script, retry_env)
|
||||
retry_output = retry.stdout + retry.stderr
|
||||
assert retry.returncode == 1
|
||||
assert "schema_version=host112_guest_recovery_v2" in retry_output
|
||||
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
|
||||
retry_output
|
||||
)
|
||||
assert "terminal=execution_failed_or_unattributed_" in retry_output
|
||||
assert "artifact_transaction_blocked" in retry_output
|
||||
assert "receipt_write_performed=0" in retry_output
|
||||
assert mutations.read_text(encoding="utf-8") == ""
|
||||
|
||||
|
||||
def test_readback_failure_keeps_receipt_and_durable_partial_not_false_success(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, fake_state = _runtime_harness(tmp_path)
|
||||
env.update(
|
||||
{
|
||||
"FAKE_LIGHTDM_START_RC": "124",
|
||||
"FAKE_READBACK_LINK_FAIL": "1",
|
||||
}
|
||||
)
|
||||
|
||||
failed = _apply(script, env)
|
||||
output = failed.stdout + failed.stderr
|
||||
assert failed.returncode == 1
|
||||
assert "receipt_status=written_immutable" in output
|
||||
assert "durable_readback_status=write_failed" in output
|
||||
assert "artifact_transaction_pair_verified=0" in output
|
||||
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
||||
assert "terminal=" in receipt.read_text(encoding="utf-8")
|
||||
assert not list((tmp_path / "state" / "readbacks").glob("*.txt"))
|
||||
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
||||
stored = transaction.read_text(encoding="utf-8")
|
||||
assert "status=partial_pending_public_artifact_commit" in stored
|
||||
assert "phase=durable_readback_write_or_disk_verification_failed" in stored
|
||||
assert not (fake_state / "mutation_without_wal").exists()
|
||||
|
||||
|
||||
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
@@ -282,8 +500,12 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
mutation_root.mkdir()
|
||||
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
|
||||
env["FAKE_LIGHTDM_START_RC"] = "124"
|
||||
for index in range(12):
|
||||
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
|
||||
mutation_slots: list[str] = []
|
||||
# These five identities cover every four-slot ring position and then reuse
|
||||
# one slot. That proves both capacity coverage and rollover without twelve
|
||||
# expensive full shell-harness invocations.
|
||||
for index in range(5):
|
||||
run_env = env | {"INVOCATION_ID": f"slot-{index}"}
|
||||
result = subprocess.run(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
@@ -296,7 +518,14 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
assert result.returncode == 1
|
||||
assert "runtime_write_performed=1" in output
|
||||
assert "receipt_status=written_bounded_timer_slot" in output
|
||||
assert "durable_readback_status=written_bounded_timer_slot" in output
|
||||
assert "durable_readback_status=written_bounded_timer_slot " in output
|
||||
mutation_slots.append(
|
||||
next(
|
||||
token.split("=", 1)[1]
|
||||
for token in output.split()
|
||||
if token.startswith("timer_ledger_slot=")
|
||||
)
|
||||
)
|
||||
for field in (
|
||||
"durable_readback_present",
|
||||
"durable_readback_identity_match",
|
||||
@@ -308,12 +537,15 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
assert f"{field}=0" not in output
|
||||
assert "timer_no_runtime_mutation_aggregated" not in output
|
||||
|
||||
assert len(set(mutation_slots)) == 4
|
||||
assert len(mutation_slots) > len(set(mutation_slots))
|
||||
|
||||
ledger_files = [
|
||||
path
|
||||
for path in (mutation_root / "state" / "timer-ledger").iterdir()
|
||||
if path.is_file()
|
||||
]
|
||||
assert len(ledger_files) <= 8
|
||||
assert len(ledger_files) == 8
|
||||
assert not (mutation_root / "state" / "receipts").exists()
|
||||
assert not (mutation_root / "state" / "readbacks").exists()
|
||||
for readback in (mutation_root / "state" / "timer-ledger").glob(
|
||||
@@ -327,8 +559,10 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
"durable_readback_receipt_link_match",
|
||||
"durable_readback_terminal_match",
|
||||
):
|
||||
assert stored.count(f"{field}=1") == 1
|
||||
assert f"{field}=1" not in stored
|
||||
assert f"{field}=0" not in stored
|
||||
transaction = mutation_root / "state" / "active-artifact-transaction.txt"
|
||||
assert "status=committed_verified_pair" in transaction.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
|
||||
@@ -337,33 +571,50 @@ def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
|
||||
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
|
||||
env["FAKE_LIGHTDM_START_RC"] = "124"
|
||||
|
||||
for index in range(12):
|
||||
marker = tmp_path / f"receipt-mv-{index}.entered"
|
||||
run_env = env | {
|
||||
"INVOCATION_ID": f"sigkill-{index}",
|
||||
"FAKE_RECEIPT_MV_MARKER": str(marker),
|
||||
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=run_env,
|
||||
start_new_session=True,
|
||||
)
|
||||
deadline = time.monotonic() + 8
|
||||
while not marker.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert marker.exists()
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate(timeout=5)
|
||||
assert process.returncode == -signal.SIGKILL
|
||||
marker = tmp_path / "receipt-mv.entered"
|
||||
run_env = env | {
|
||||
"INVOCATION_ID": "sigkill-0",
|
||||
"FAKE_RECEIPT_MV_MARKER": str(marker),
|
||||
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=run_env,
|
||||
start_new_session=True,
|
||||
)
|
||||
deadline = time.monotonic() + 8
|
||||
while not marker.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert marker.exists()
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate(timeout=5)
|
||||
assert process.returncode == -signal.SIGKILL
|
||||
|
||||
ledger = tmp_path / "state" / "timer-ledger"
|
||||
files = [path for path in ledger.iterdir() if path.is_file()]
|
||||
assert len(files) <= 4
|
||||
assert len(files) == 1
|
||||
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
|
||||
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
||||
assert "status=pending_artifact_commit" in transaction.read_text(encoding="utf-8")
|
||||
|
||||
mutations = tmp_path / "fake-host" / "mutations.log"
|
||||
before = mutations.read_bytes()
|
||||
retry = subprocess.run(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env | {"INVOCATION_ID": "sigkill-retry"},
|
||||
check=False,
|
||||
timeout=15,
|
||||
)
|
||||
assert retry.returncode == 1
|
||||
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
|
||||
retry.stdout + retry.stderr
|
||||
)
|
||||
assert mutations.read_bytes() == before
|
||||
|
||||
|
||||
def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
||||
@@ -400,7 +651,7 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
||||
assert marker.exists()
|
||||
os.kill(process.pid, signal.SIGTERM)
|
||||
stdout, stderr = process.communicate(timeout=12)
|
||||
assert process.returncode == 75, stdout + stderr
|
||||
assert process.returncode == 1, stdout + stderr
|
||||
|
||||
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
||||
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
|
||||
@@ -411,6 +662,15 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
||||
assert f"receipt_path={receipt}" in signal_text
|
||||
assert "terminal=interrupted_signal_TERM" in signal_text
|
||||
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
|
||||
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
||||
transaction_text = transaction.read_text(encoding="utf-8")
|
||||
assert "status=partial_pending_public_artifact_commit" in transaction_text
|
||||
assert "phase=interrupted_before_durable_readback_TERM" in transaction_text
|
||||
assert not [
|
||||
path
|
||||
for path in (tmp_path / "state" / "readbacks").glob("*.txt")
|
||||
if not path.name.endswith(".signal.txt")
|
||||
]
|
||||
|
||||
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
|
||||
assert replay.returncode == 65
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
@@ -67,6 +68,8 @@ def test_windows99_direct_route_is_fingerprint_bound_and_not_forced() -> None:
|
||||
assert 'cat "$AUTH_FILE"' not in text
|
||||
assert 'from="%s",restrict,command="%s --check"' in text
|
||||
assert 'CONTROL_SOURCE_ADDRESS:-192.168.0.110' in text
|
||||
assert 'BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH' in text
|
||||
assert "never discovers, reads, copies, or derives a private key" in text
|
||||
|
||||
|
||||
def test_apply_is_transactional_and_has_a_verified_rollback() -> None:
|
||||
@@ -149,3 +152,73 @@ def test_check_gate_requires_policy_route_timer_and_no_write_readback() -> None:
|
||||
assert 'runtime_write_performed=0' in text
|
||||
assert 'manager_runtime_write_performed=0' in text
|
||||
assert 'receipt_write_performed=0' in text
|
||||
|
||||
|
||||
def test_clean_install_timer_stop_is_state_aware_and_idempotent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
text = source()
|
||||
function = text[
|
||||
text.index("stop_recovery_timer_if_running() {") : text.index(
|
||||
"\n\nresolve_target_user",
|
||||
text.index("stop_recovery_timer_if_running() {"),
|
||||
)
|
||||
]
|
||||
runner = tmp_path / "run-stop-contract.sh"
|
||||
runner.write_text(
|
||||
"#!/usr/bin/env bash\nset -u\n" + function + "\nstop_recovery_timer_if_running\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
runner.chmod(0o755)
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
log = tmp_path / "systemctl.log"
|
||||
systemctl = fake_bin / "systemctl"
|
||||
systemctl.write_text(
|
||||
"""#!/usr/bin/env bash
|
||||
printf '%s\n' "$*" >>"${SYSTEMCTL_LOG:?}"
|
||||
if [ "${1:-}" = "is-active" ]; then
|
||||
printf '%s\n' "${FAKE_TIMER_STATE:-not-found}"
|
||||
[ "${FAKE_TIMER_STATE:-not-found}" = "active" ]
|
||||
fi
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
systemctl.chmod(0o755)
|
||||
timeout = fake_bin / "timeout"
|
||||
timeout.write_text(
|
||||
"#!/usr/bin/env bash\nshift\n\"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
timeout.chmod(0o755)
|
||||
env = os.environ | {
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"SYSTEMCTL_LOG": str(log),
|
||||
}
|
||||
|
||||
clean = subprocess.run(
|
||||
["bash", str(runner)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env | {"FAKE_TIMER_STATE": "not-found"},
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert clean.returncode == 0, clean.stdout + clean.stderr
|
||||
assert "stop awoooi-host112-guest-recovery.timer" not in log.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
log.write_text("", encoding="utf-8")
|
||||
active = subprocess.run(
|
||||
["bash", str(runner)],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env | {"FAKE_TIMER_STATE": "active"},
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert active.returncode == 0, active.stdout + active.stderr
|
||||
assert "stop awoooi-host112-guest-recovery.timer" in log.read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
@@ -88,6 +88,22 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]:
|
||||
assert field in host_probe
|
||||
|
||||
canonical_host112_vmx = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx"
|
||||
assert canonical_host112_vmx in windows99
|
||||
assert r"D:\Downloads\kali-linux-2025.4-vmware-amd64" not in windows99
|
||||
assert r'"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"' in read(
|
||||
"agent99.config.99.example.json"
|
||||
)
|
||||
assert canonical_host112_vmx in read(
|
||||
"apps/api/src/services/reboot_auto_recovery_slo_scorecard.py"
|
||||
)
|
||||
for locator in (
|
||||
"scripts/reboot-recovery/windows99-vmx-source-locator.ps1",
|
||||
"scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1",
|
||||
"scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1",
|
||||
):
|
||||
assert r"S:\VMs" in read(locator)
|
||||
|
||||
host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh")
|
||||
host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh")
|
||||
host112_service = read(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
param(
|
||||
[string[]]$VmNameAllowList = @("110", "188", "120", "121", "112"),
|
||||
[string[]]$SearchRoots = @("C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"),
|
||||
[string[]]$SearchRoots = @("S:\VMs", "C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"),
|
||||
[string]$TaskName = "AWOOOI VMware VM AutoStart",
|
||||
[switch]$WhatIfOnly
|
||||
)
|
||||
|
||||
@@ -12,6 +12,7 @@ param(
|
||||
[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112"),
|
||||
[switch]$EnableRecursiveDiscovery,
|
||||
[string[]]$DiscoveryRoot = @(
|
||||
"S:\VMs",
|
||||
"D:\Documents\Virtual Machines",
|
||||
"D:\Downloads",
|
||||
"D:\VMs",
|
||||
@@ -41,7 +42,7 @@ $KnownVmxCandidates = @{
|
||||
"188" = @("D:\Documents\Virtual Machines\Ollama_Ubuntu_64-bit\Ollama_Ubuntu_64-bit.vmx")
|
||||
"120" = @("D:\Documents\Virtual Machines\192.168.0.120_Ubuntu_64-bit\192.168.0.120_Ubuntu_64-bit.vmx")
|
||||
"121" = @("D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx")
|
||||
"112" = @("D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx")
|
||||
"112" = @("S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx")
|
||||
}
|
||||
|
||||
function Resolve-VmxPath {
|
||||
|
||||
@@ -2,6 +2,7 @@ param(
|
||||
[string]$Alias = "111",
|
||||
[string]$ExpectedVmxPath = "D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx",
|
||||
[string[]]$DiscoveryRoot = @(
|
||||
"S:\VMs",
|
||||
"D:\Documents\Virtual Machines",
|
||||
"D:\Downloads",
|
||||
"D:\VMs",
|
||||
|
||||
@@ -6,6 +6,7 @@ param(
|
||||
"D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx"
|
||||
),
|
||||
[string[]]$DiscoveryRoot = @(
|
||||
"S:\VMs",
|
||||
"D:\Documents\Virtual Machines",
|
||||
"D:\Downloads",
|
||||
"D:\VMs",
|
||||
|
||||
Reference in New Issue
Block a user