From a6307e785de8c98edf4d4375e19b3c72c91d0c03 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 13:43:38 +0800 Subject: [PATCH] feat(automation): harden D037 runtime closure receipts --- .gitea/workflows/cd.yaml | 150 +++-- agent99-bootstrap.ps1 | 71 ++- agent99-contract-check.ps1 | 14 +- agent99-control-plane.ps1 | 182 +++++- agent99-deploy.ps1 | 79 ++- agent99-synthetic-tests.ps1 | 5 +- agent99.config.99.example.json | 2 +- .../executor_trust_boundary_readback.py | 166 +++++- .../reboot_auto_recovery_slo_scorecard.py | 2 +- ..._agent99_control_plane_outcome_contract.py | 25 +- ...nt99_transport_recovery_deploy_contract.py | 6 + ...awoooi_priority_work_order_readback_api.py | 4 +- .../test_executor_trust_boundary_readback.py | 111 +++- ..._signal_worker_ansible_executor_binding.py | 33 +- ...t_workload_database_identity_projection.py | 12 +- .../awoooi-workload-db-identity-bootstrap.sh | 86 +-- .../agent99-live-preflight.ps1 | 161 ++++- .../agent99-remote-atomic-deploy-receiver.ps1 | 345 +++++++++-- .../host112-guest-readiness.sh | 562 +++++++++++++++--- .../install-host112-guest-recovery.sh | 34 +- .../test_agent99_live_preflight_decision.py | 203 +++++++ ...t_agent99_recovery_coordinator_contract.py | 13 +- ...remote_atomic_deploy_transport_contract.py | 174 ++++++ .../test_host112_manager_recovery_contract.py | 127 +++- .../test_host112_recovery_runtime_harness.py | 322 +++++++++- ...install_host112_guest_recovery_contract.py | 73 +++ .../test_reboot_p0_operational_contract.py | 16 + .../windows-99/configure-vmware-autostart.ps1 | 2 +- .../windows99-vmware-autostart.ps1 | 3 +- .../windows99-vmx-source-locate-check.ps1 | 1 + .../windows99-vmx-source-locator.ps1 | 1 + 31 files changed, 2616 insertions(+), 369 deletions(-) create mode 100644 scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py diff --git a/.gitea/workflows/cd.yaml b/.gitea/workflows/cd.yaml index 1a2ed80c5..d0fbfb035 100644 --- a/.gitea/workflows/cd.yaml +++ b/.gitea/workflows/cd.yaml @@ -3267,10 +3267,12 @@ jobs: read_workload_db_identity() { workload="$1" container="$2" - concurrent_probe_count="$3" + representative_probe_count="$3" + required_connection_budget="$4" $KUBECTL exec -i "deployment/$workload" -n awoooi-prod \ -c "$container" -- env \ - AWOOOI_DB_CONCURRENT_PROBE_COUNT="$concurrent_probe_count" \ + AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT="$representative_probe_count" \ + AWOOOI_DB_REQUIRED_CONNECTION_BUDGET="$required_connection_budget" \ python - <<'PY' import asyncio import hashlib @@ -3288,7 +3290,13 @@ jobs: result = await db.execute(text(""" SELECT current_user AS role_name, - rolconnlimit AS connection_limit + rolconnlimit AS connection_limit, + ( + SELECT count(*) + FROM pg_stat_activity + WHERE usename = current_user + AND pid <> pg_backend_pid() + ) AS active_role_connection_count FROM pg_roles WHERE rolname = current_user """)) @@ -3336,8 +3344,22 @@ jobs: ) """))).scalar_one()) - concurrent_probe_count = int( - os.environ["AWOOOI_DB_CONCURRENT_PROBE_COUNT"] + representative_probe_count = int( + os.environ["AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT"] + ) + required_connection_budget = int( + os.environ["AWOOOI_DB_REQUIRED_CONNECTION_BUDGET"] + ) + if representative_probe_count < 1: + raise RuntimeError("representative_probe_count_invalid") + if required_connection_budget < 1: + raise RuntimeError("required_connection_budget_invalid") + connection_limit = int(row["connection_limit"]) + active_role_connection_count = int( + row["active_role_connection_count"] + ) + available_connection_headroom = ( + connection_limit - active_role_connection_count ) async def probe_connection() -> None: @@ -3346,13 +3368,16 @@ jobs: await asyncio.gather(*( probe_connection() - for _ in range(concurrent_probe_count) + for _ in range(representative_probe_count) )) except Exception as exc: print(json.dumps({ "role_fingerprint": "", "connection_limit": None, - "concurrent_connection_probe_count": 0, + "required_connection_budget": None, + "active_role_connection_count": None, + "available_connection_headroom": None, + "representative_connection_probe_count": 0, "representative_preflight_passed": False, "table_privilege_gap_count": None, "sequence_privilege_gap_count": None, @@ -3363,10 +3388,20 @@ jobs: "role_fingerprint": hashlib.sha256( str(row["role_name"]).encode("utf-8") ).hexdigest(), - "connection_limit": int(row["connection_limit"]), - "concurrent_connection_probe_count": concurrent_probe_count, + "connection_limit": connection_limit, + "required_connection_budget": required_connection_budget, + "active_role_connection_count": active_role_connection_count, + "available_connection_headroom": available_connection_headroom, + "representative_connection_probe_count": representative_probe_count, "representative_preflight_passed": ( - table_gap_count == 0 and sequence_gap_count == 0 + table_gap_count == 0 + and sequence_gap_count == 0 + and representative_probe_count >= 1 + and connection_limit >= required_connection_budget + and ( + available_connection_headroom + >= required_connection_budget + ) ), "table_privilege_gap_count": table_gap_count, "sequence_privilege_gap_count": sequence_gap_count, @@ -3405,12 +3440,12 @@ jobs: fi } - API_DB_IDENTITY=$(read_workload_db_identity awoooi-api api 8 \ + API_DB_IDENTITY=$(read_workload_db_identity awoooi-api api 1 8 \ | tail -n 1 || true) - WORKER_DB_IDENTITY=$(read_workload_db_identity awoooi-worker worker 5 \ + WORKER_DB_IDENTITY=$(read_workload_db_identity awoooi-worker worker 1 5 \ | tail -n 1 || true) BROKER_DB_IDENTITY=$(read_workload_db_identity \ - awoooi-ansible-executor-broker broker 2 | tail -n 1 || true) + awoooi-ansible-executor-broker broker 1 2 | tail -n 1 || true) API_DB_ROLE_FINGERPRINT=$(printf '%s' "$API_DB_IDENTITY" \ | json_field role_fingerprint 2>/dev/null || true) WORKER_DB_ROLE_FINGERPRINT=$(printf '%s' "$WORKER_DB_IDENTITY" \ @@ -3423,12 +3458,30 @@ jobs: | json_field connection_limit 2>/dev/null || true) BROKER_DB_CONNECTION_LIMIT=$(printf '%s' "$BROKER_DB_IDENTITY" \ | json_field connection_limit 2>/dev/null || true) - API_DB_CONCURRENT_PROBE_COUNT=$(printf '%s' "$API_DB_IDENTITY" \ - | json_field concurrent_connection_probe_count 2>/dev/null || true) - WORKER_DB_CONCURRENT_PROBE_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \ - | json_field concurrent_connection_probe_count 2>/dev/null || true) - BROKER_DB_CONCURRENT_PROBE_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \ - | json_field concurrent_connection_probe_count 2>/dev/null || true) + API_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field required_connection_budget 2>/dev/null || true) + WORKER_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field required_connection_budget 2>/dev/null || true) + BROKER_DB_REQUIRED_CONNECTION_BUDGET=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field required_connection_budget 2>/dev/null || true) + API_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field active_role_connection_count 2>/dev/null || true) + WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field active_role_connection_count 2>/dev/null || true) + BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field active_role_connection_count 2>/dev/null || true) + API_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field available_connection_headroom 2>/dev/null || true) + WORKER_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field available_connection_headroom 2>/dev/null || true) + BROKER_DB_AVAILABLE_CONNECTION_HEADROOM=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field available_connection_headroom 2>/dev/null || true) + API_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$API_DB_IDENTITY" \ + | json_field representative_connection_probe_count 2>/dev/null || true) + WORKER_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$WORKER_DB_IDENTITY" \ + | json_field representative_connection_probe_count 2>/dev/null || true) + BROKER_DB_REPRESENTATIVE_PROBE_COUNT=$(printf '%s' "$BROKER_DB_IDENTITY" \ + | json_field representative_connection_probe_count 2>/dev/null || true) API_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$API_DB_IDENTITY" \ | json_field representative_preflight_passed 2>/dev/null || true)") WORKER_DB_REPRESENTATIVE_PREFLIGHT=$(normalize_bool "$(printf '%s' "$WORKER_DB_IDENTITY" \ @@ -3450,9 +3503,18 @@ jobs: API_DB_CONNECTION_LIMIT=${API_DB_CONNECTION_LIMIT:-0} WORKER_DB_CONNECTION_LIMIT=${WORKER_DB_CONNECTION_LIMIT:-0} BROKER_DB_CONNECTION_LIMIT=${BROKER_DB_CONNECTION_LIMIT:-0} - API_DB_CONCURRENT_PROBE_COUNT=${API_DB_CONCURRENT_PROBE_COUNT:-0} - WORKER_DB_CONCURRENT_PROBE_COUNT=${WORKER_DB_CONCURRENT_PROBE_COUNT:-0} - BROKER_DB_CONCURRENT_PROBE_COUNT=${BROKER_DB_CONCURRENT_PROBE_COUNT:-0} + API_DB_REQUIRED_CONNECTION_BUDGET=${API_DB_REQUIRED_CONNECTION_BUDGET:-0} + WORKER_DB_REQUIRED_CONNECTION_BUDGET=${WORKER_DB_REQUIRED_CONNECTION_BUDGET:-0} + BROKER_DB_REQUIRED_CONNECTION_BUDGET=${BROKER_DB_REQUIRED_CONNECTION_BUDGET:-0} + API_DB_ACTIVE_ROLE_CONNECTION_COUNT=${API_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1} + WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=${WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1} + BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT=${BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT:--1} + API_DB_AVAILABLE_CONNECTION_HEADROOM=${API_DB_AVAILABLE_CONNECTION_HEADROOM:--1} + WORKER_DB_AVAILABLE_CONNECTION_HEADROOM=${WORKER_DB_AVAILABLE_CONNECTION_HEADROOM:--1} + BROKER_DB_AVAILABLE_CONNECTION_HEADROOM=${BROKER_DB_AVAILABLE_CONNECTION_HEADROOM:--1} + API_DB_REPRESENTATIVE_PROBE_COUNT=${API_DB_REPRESENTATIVE_PROBE_COUNT:-0} + WORKER_DB_REPRESENTATIVE_PROBE_COUNT=${WORKER_DB_REPRESENTATIVE_PROBE_COUNT:-0} + BROKER_DB_REPRESENTATIVE_PROBE_COUNT=${BROKER_DB_REPRESENTATIVE_PROBE_COUNT:-0} API_DB_TABLE_PRIVILEGE_GAP_COUNT=${API_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT=${BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT:-1} @@ -3511,6 +3573,7 @@ jobs: DB_IDENTITY_ISOLATED=false if [ "$API_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \ && [ "$WORKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \ + && [ "$BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true" ] \ && [ "$DISTINCT_DATABASE_SECRET_REFS" = "true" ] \ && [ "$DISTINCT_DB_ROLE_FINGERPRINTS" = "true" ]; then DB_IDENTITY_ISOLATED=true @@ -3519,9 +3582,9 @@ jobs: if [ "$API_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ && [ "$WORKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ && [ "$BROKER_DB_REPRESENTATIVE_PREFLIGHT" = "true" ] \ - && [ "$API_DB_CONCURRENT_PROBE_COUNT" -ge 8 ] \ - && [ "$WORKER_DB_CONCURRENT_PROBE_COUNT" -ge 5 ] \ - && [ "$BROKER_DB_CONCURRENT_PROBE_COUNT" -ge 2 ] \ + && [ "$API_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \ + && [ "$WORKER_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \ + && [ "$BROKER_DB_REPRESENTATIVE_PROBE_COUNT" -ge 1 ] \ && [ "$API_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ && [ "$WORKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ && [ "$BROKER_DB_TABLE_PRIVILEGE_GAP_COUNT" -eq 0 ] \ @@ -3536,9 +3599,18 @@ jobs: && [ "$API_DB_NULL_POOL" = "true" ] \ && [ "$WORKER_DB_NULL_POOL" = "true" ] \ && [ "$BROKER_DB_NULL_POOL" = "true" ] \ - && [ "$API_DB_CONNECTION_LIMIT" -ge 12 ] \ - && [ "$WORKER_DB_CONNECTION_LIMIT" -ge 8 ] \ - && [ "$BROKER_DB_CONNECTION_LIMIT" -ge 4 ]; then + && [ "$API_DB_CONNECTION_LIMIT" -eq 12 ] \ + && [ "$WORKER_DB_CONNECTION_LIMIT" -eq 8 ] \ + && [ "$BROKER_DB_CONNECTION_LIMIT" -eq 4 ] \ + && [ "$API_DB_REQUIRED_CONNECTION_BUDGET" -eq 8 ] \ + && [ "$WORKER_DB_REQUIRED_CONNECTION_BUDGET" -eq 5 ] \ + && [ "$BROKER_DB_REQUIRED_CONNECTION_BUDGET" -eq 2 ] \ + && [ "$API_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ + && [ "$WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ + && [ "$BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" -ge 0 ] \ + && [ "$API_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$API_DB_REQUIRED_CONNECTION_BUDGET" ] \ + && [ "$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$WORKER_DB_REQUIRED_CONNECTION_BUDGET" ] \ + && [ "$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" -ge "$BROKER_DB_REQUIRED_CONNECTION_BUDGET" ]; then CONNECTION_BUDGET_VERIFIED=true fi @@ -3587,7 +3659,7 @@ jobs: BOUNDARY_VERIFIED_AT=$(date -u +"%Y-%m-%dT%H:%M:%SZ") $KUBECTL create configmap awoooi-executor-boundary-verification \ -n awoooi-prod \ - --from-literal=schema_version=awoooi_executor_boundary_verification_v3 \ + --from-literal=schema_version=awoooi_executor_boundary_verification_v4 \ --from-literal=verified_source_sha="$SOURCE_REVISION" \ --from-literal=verified_at="$BOUNDARY_VERIFIED_AT" \ --from-literal=workflow_run_id="$WORKFLOW_RUN_ID" \ @@ -3628,7 +3700,7 @@ jobs: --from-literal=learning_failure_fail_closed=true \ --from-literal=trust_failure_fail_closed=true \ --from-literal=learning_receipts_public_safe=true \ - --from-literal=db_identity_verifier=runtime_role_fingerprint_secret_projection_and_representative_preflight_v2 \ + --from-literal=db_identity_verifier=runtime_role_fingerprint_role_limit_null_pool_http_concurrency_and_representative_preflight_v4 \ --from-literal=api_database_secret_ref="$API_DATABASE_SECRET_REF" \ --from-literal=worker_database_secret_ref="$WORKER_DATABASE_SECRET_REF" \ --from-literal=broker_database_secret_ref="$BROKER_DATABASE_SECRET_REF" \ @@ -3638,12 +3710,18 @@ jobs: --from-literal=api_db_connection_limit="$API_DB_CONNECTION_LIMIT" \ --from-literal=worker_db_connection_limit="$WORKER_DB_CONNECTION_LIMIT" \ --from-literal=broker_db_connection_limit="$BROKER_DB_CONNECTION_LIMIT" \ - --from-literal=api_required_connection_budget=8 \ - --from-literal=worker_required_connection_budget=5 \ - --from-literal=broker_required_connection_budget=2 \ - --from-literal=api_concurrent_connection_probe_count="$API_DB_CONCURRENT_PROBE_COUNT" \ - --from-literal=worker_concurrent_connection_probe_count="$WORKER_DB_CONCURRENT_PROBE_COUNT" \ - --from-literal=broker_concurrent_connection_probe_count="$BROKER_DB_CONCURRENT_PROBE_COUNT" \ + --from-literal=api_required_connection_budget="$API_DB_REQUIRED_CONNECTION_BUDGET" \ + --from-literal=worker_required_connection_budget="$WORKER_DB_REQUIRED_CONNECTION_BUDGET" \ + --from-literal=broker_required_connection_budget="$BROKER_DB_REQUIRED_CONNECTION_BUDGET" \ + --from-literal=api_active_role_connection_count="$API_DB_ACTIVE_ROLE_CONNECTION_COUNT" \ + --from-literal=worker_active_role_connection_count="$WORKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" \ + --from-literal=broker_active_role_connection_count="$BROKER_DB_ACTIVE_ROLE_CONNECTION_COUNT" \ + --from-literal=api_available_connection_headroom="$API_DB_AVAILABLE_CONNECTION_HEADROOM" \ + --from-literal=worker_available_connection_headroom="$WORKER_DB_AVAILABLE_CONNECTION_HEADROOM" \ + --from-literal=broker_available_connection_headroom="$BROKER_DB_AVAILABLE_CONNECTION_HEADROOM" \ + --from-literal=api_representative_connection_probe_count="$API_DB_REPRESENTATIVE_PROBE_COUNT" \ + --from-literal=worker_representative_connection_probe_count="$WORKER_DB_REPRESENTATIVE_PROBE_COUNT" \ + --from-literal=broker_representative_connection_probe_count="$BROKER_DB_REPRESENTATIVE_PROBE_COUNT" \ --from-literal=api_representative_db_preflight_passed="$API_DB_REPRESENTATIVE_PREFLIGHT" \ --from-literal=worker_representative_db_preflight_passed="$WORKER_DB_REPRESENTATIVE_PREFLIGHT" \ --from-literal=broker_representative_db_preflight_passed="$BROKER_DB_REPRESENTATIVE_PREFLIGHT" \ diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index bc452ec22..dfbe7101f 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -18,10 +18,6 @@ $PlaybookDir = Join-Path $AgentRoot "playbooks" $DashboardDir = Join-Path $AgentRoot "dashboard" $StateDir = Join-Path $AgentRoot "state" -foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir, $EvidenceDir, $LogDir, $QueueDir, $RequestDir, $AlertDir, $PlaybookDir, $DashboardDir, $StateDir)) { - New-Item -ItemType Directory -Force -Path $dir | Out-Null -} - function Copy-AgentFile { param([string]$Name) $src = Join-Path $SourceRoot $Name @@ -50,8 +46,48 @@ function Set-AgentBootstrapProperty { } } +function Get-AgentHost112CanonicalVm { + param([string]$PolicyPath) + if (-not (Test-Path -LiteralPath $PolicyPath -PathType Leaf)) { + throw "Agent99 Host112 canonical policy config is unavailable: $PolicyPath" + } + $policy = Get-Content -LiteralPath $PolicyPath -Raw | ConvertFrom-Json + $candidates = @($policy.vms | Where-Object { + [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" + }) + if ($candidates.Count -ne 1) { + throw "Agent99 Host112 canonical policy must contain exactly one Host112 VM row; found $($candidates.Count)." + } + $candidate = $candidates[0] + $candidateVmx = ([string]$candidate.vmx).Trim() + if ( + [string]$candidate.name -ne "host112-kali" -or + [string]$candidate.host -ne "192.168.0.112" -or + -not $candidateVmx + ) { + throw "Agent99 Host112 canonical policy identity is invalid." + } + if (-not (Test-Path -LiteralPath $candidateVmx -PathType Leaf)) { + throw "Host112 canonical VMX path from staged policy is unavailable: $candidateVmx" + } + return ($candidate | ConvertTo-Json -Depth 10 | ConvertFrom-Json) +} + +# Fail before the first live Agent99 write when the staged canonical Host112 +# policy or its VMX target is unavailable. Bootstrap is intentionally not +# allowed to leave a new runtime/manifest paired with the old live config. +$host112BootstrapPolicyPath = Join-Path $SourceRoot "agent99.config.99.example.json" +$null = Get-AgentHost112CanonicalVm $host112BootstrapPolicyPath + +foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir, $EvidenceDir, $LogDir, $QueueDir, $RequestDir, $AlertDir, $PlaybookDir, $DashboardDir, $StateDir)) { + New-Item -ItemType Directory -Force -Path $dir | Out-Null +} + function Ensure-AgentHost112CanonicalSshConfig { - param([string]$Path) + param([string]$Path, [string]$PolicyPath) + # Resolve and verify the staged source of truth before mutating the live config. + $canonicalHost112Vm = Get-AgentHost112CanonicalVm $PolicyPath + $canonicalHost112Vmx = [string]$canonicalHost112Vm.vmx $config = Get-Content $Path -Raw | ConvertFrom-Json if (-not $config.PSObject.Properties["sshIdentityFile"] -or -not $config.sshIdentityFile) { Set-AgentBootstrapProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") @@ -71,13 +107,32 @@ function Ensure-AgentHost112CanonicalSshConfig { } $directHosts = if ($config.k3s.PSObject.Properties["directHosts"]) { @($config.k3s.directHosts | ForEach-Object { [string]$_ } | Where-Object { $_ }) } else { @() } Set-AgentBootstrapProperty $config.k3s "directHosts" @($directHosts + @("192.168.0.110", "192.168.0.112") | Select-Object -Unique) + if (-not $config.PSObject.Properties["vms"] -or $null -eq $config.vms) { + Set-AgentBootstrapProperty $config "vms" @() + } + $nonHost112Vms = @($config.vms | Where-Object { + -not ([string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali") + }) + Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm)) $config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $Path $persisted = Get-Content $Path -Raw | ConvertFrom-Json $persistedUser = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } $persistedDirectHosts = if ($persisted.k3s -and $persisted.k3s.PSObject.Properties["directHosts"]) { @($persisted.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } - if ($persistedUser -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) { - throw "Host112 canonical direct SSH config bootstrap verifier failed." + $persistedHost112Vm = @($persisted.vms | Where-Object { + [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" + }) + $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } + if ( + $persistedUser -ne "kali" -or + "192.168.0.112" -notin $persistedDirectHosts -or + $persistedHost112Vm.Count -ne 1 -or + [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or + [string]$persistedHost112Vm[0].host -ne [string]$canonicalHost112Vm.host -or + $persistedHost112Vmx -ne $canonicalHost112Vmx -or + -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) + ) { + throw "Host112 canonical SSH and VMX config bootstrap verifier failed." } } @@ -229,7 +284,7 @@ if (-not (Test-Path $configPath)) { } } -Ensure-AgentHost112CanonicalSshConfig $configPath +Ensure-AgentHost112CanonicalSshConfig $configPath $host112BootstrapPolicyPath $launcher = Join-Path $AgentRoot "agent99-run.ps1" @" diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index d6dce1169..181b0c58c 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -64,8 +64,15 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $config.completionCallback.url -match "^https://" -and $config.completionCallback.tokenEnv -eq "AGENT99_SRE_RELAY_TOKEN" ) - $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback" + $hasCanonicalHost112Vmx = if ($name -eq "agent99.config.99.example.json") { + [bool](@($config.vms | Where-Object { + [string]$_.name -eq "host112-kali" -and + [string]$_.host -eq "192.168.0.112" -and + [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" + }).Count -eq 1) + } else { $true } + $ok = $hasHosts -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } @@ -87,8 +94,9 @@ Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -a Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" -Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, attributed start, immutable receipt/readback and independent verifier" +Add-Check "recovery:host112_immutable_apply_closure" ($control.Contains("managerApplyVerified") -and $control.Contains("durableReceiptVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply closure binds transport, fixed WAL, attributed start, immutable receipt/readback and independent verifier" Add-Check "recovery:host112_direct_transport" ($control.Contains("Invoke-AgentHost112SshText") -and $control.Contains('expectedUser = "kali"') -and $control.Contains('route = "direct_host112_kali"') -and $deployer.Contains('name = "host112_canonical_direct_route"') -and $bootstrap.Contains("Ensure-AgentHost112CanonicalSshConfig")) "Host112 recovery uses the canonical 99 direct key route" +Add-Check "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf') -and $bootstrap.Contains('Get-AgentHost112CanonicalVm $PolicyPath') -and $bootstrap.Contains('Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))')) "Host112 VM identity is normalized from one staged policy row and fails closed when absent" Add-Check "recovery:host112_timeout_reserve" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120")) "Host112 timeout chain has executor, queue and client reserves" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" Add-Check "problem_counts:v2" ($control.Contains('schemaVersion = "agent99_problem_counts_v2"') -and $control.Contains("totalVerifiedResolved")) "verified problem counts are present" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 150a5503c..797fbe601 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -3451,6 +3451,7 @@ function Get-AgentBackupHealthConfig { protectionMetricsPath = if ($backupHealth -and $backupHealth.protectionMetricsPath) { [string]$backupHealth.protectionMetricsPath } else { "/home/wooo/node_exporter_textfiles/backup_health.prom" } protectionReadTimeoutSeconds = if ($backupHealth -and $backupHealth.protectionReadTimeoutSeconds) { [int]$backupHealth.protectionReadTimeoutSeconds } else { 45 } protectionMaxAgeMinutes = if ($backupHealth -and $backupHealth.protectionMaxAgeMinutes) { [double]$backupHealth.protectionMaxAgeMinutes } else { 60 } + protectionEscrowExpectedCount = if ($backupHealth -and $backupHealth.protectionEscrowExpectedCount) { [int]$backupHealth.protectionEscrowExpectedCount } else { 5 } } } @@ -3467,11 +3468,41 @@ function Get-AgentBackupMetricValues { @($values) } +function Get-AgentBackupProviderMetricRows { + param([string]$Output, [string]$MetricName) + + $rows = @() + $pattern = "(?m)^" + [regex]::Escape($MetricName) + "\{(?[^}]*)\}\s+(?-?[0-9]+(?:\.[0-9]+)?)\s*$" + foreach ($match in [regex]::Matches([string]$Output, $pattern)) { + $providerMatch = [regex]::Match( + $match.Groups["labels"].Value, + '(?:^|,)\s*provider="(?[A-Za-z0-9_.-]+)"(?:\s*,|$)' + ) + $parsed = 0.0 + if ( + $providerMatch.Success -and + [double]::TryParse( + $match.Groups["value"].Value, + [Globalization.NumberStyles]::Float, + [Globalization.CultureInfo]::InvariantCulture, + [ref]$parsed + ) + ) { + $rows += [pscustomobject]@{ + provider = $providerMatch.Groups["provider"].Value + value = $parsed + } + } + } + @($rows) +} + function Convert-AgentBackupProtectionReadback { param( [object]$Readback, [double]$MaxAgeMinutes = 60, - [double]$NowUnix = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds() + [double]$NowUnix = [DateTimeOffset]::UtcNow.ToUnixTimeSeconds(), + [int]$ExpectedEscrowFreshCount = 5 ) $readback = $Readback @@ -3482,18 +3513,63 @@ function Convert-AgentBackupProtectionReadback { $configured = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_configured") $offsiteFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_fresh") $remoteVerify = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_offsite_remote_verify_ok") + $configuredRows = @(Get-AgentBackupProviderMetricRows $readback.output "awoooi_backup_offsite_configured") + $offsiteFreshRows = @(Get-AgentBackupProviderMetricRows $readback.output "awoooi_backup_offsite_fresh") + $remoteVerifyRows = @(Get-AgentBackupProviderMetricRows $readback.output "awoooi_backup_offsite_remote_verify_ok") $escrowFresh = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_credential_escrow_fresh") $escrowMissingValues = @(Get-AgentBackupMetricValues $readback.output "awoooi_backup_dr_credential_escrow_missing_count") - $escrowMissing = if ($escrowMissingValues.Count -gt 0 -and @($escrowMissingValues | Where-Object { $_ -lt 0 }).Count -eq 0) { [int](($escrowMissingValues | Measure-Object -Sum).Sum) } else { -1 } + $escrowMissing = if ($escrowMissingValues.Count -eq 1 -and @($escrowMissingValues | Where-Object { $_ -lt 0 }).Count -eq 0) { [int]$escrowMissingValues[0] } else { -1 } + + # The exporter publishes one row for every known provider, including + # optional compatibility providers that are intentionally unconfigured. + # Require at least one configured provider, then verify freshness and remote + # readback only for those configured providers. Contradictory duplicate rows + # or provider-less rows fail closed rather than silently becoming optional. + $configuredShapeValid = [bool]( + $configured.Count -gt 0 -and + $configured.Count -eq $configuredRows.Count -and + $offsiteFresh.Count -eq $offsiteFreshRows.Count -and + $remoteVerify.Count -eq $remoteVerifyRows.Count + ) + $configuredProviderNames = @() + foreach ($group in @($configuredRows | Group-Object provider)) { + $providerValues = @($group.Group | ForEach-Object { [double]$_.value }) + $hasConfigured = @($providerValues | Where-Object { $_ -ge 1 }).Count -gt 0 + $hasUnconfigured = @($providerValues | Where-Object { $_ -lt 1 }).Count -gt 0 + if ($hasConfigured -and $hasUnconfigured) { + $configuredShapeValid = $false + } elseif ($hasConfigured) { + $configuredProviderNames += [string]$group.Name + } + } + $configuredProviderNames = @($configuredProviderNames | Sort-Object -Unique) + $verifiedProviderNames = @() + foreach ($providerName in $configuredProviderNames) { + $providerFreshRows = @($offsiteFreshRows | Where-Object { $_.provider -eq $providerName }) + $providerRemoteRows = @($remoteVerifyRows | Where-Object { $_.provider -eq $providerName }) + $providerFresh = [bool]( + $providerFreshRows.Count -gt 0 -and + @($providerFreshRows | Where-Object { $_.value -lt 1 }).Count -eq 0 + ) + $providerRemoteVerified = [bool]( + $providerRemoteRows.Count -gt 0 -and + @($providerRemoteRows | Where-Object { $_.value -lt 1 }).Count -eq 0 + ) + if ($providerFresh -and $providerRemoteVerified) { + $verifiedProviderNames += $providerName + } + } $offsiteOk = [bool]( $readback.ok -and $mtime -and $evidenceFresh -and - $configured.Count -gt 0 -and @($configured | Where-Object { $_ -ge 1 }).Count -eq $configured.Count -and - $offsiteFresh.Count -gt 0 -and @($offsiteFresh | Where-Object { $_ -ge 1 }).Count -eq $offsiteFresh.Count -and - $remoteVerify.Count -gt 0 -and @($remoteVerify | Where-Object { $_ -ge 1 }).Count -eq $remoteVerify.Count + $configuredShapeValid -and + $configuredProviderNames.Count -gt 0 -and + $verifiedProviderNames.Count -eq $configuredProviderNames.Count ) $escrowOk = [bool]( $readback.ok -and $mtime -and $evidenceFresh -and $escrowMissing -eq 0 -and - $escrowFresh.Count -gt 0 -and @($escrowFresh | Where-Object { $_ -ge 1 }).Count -eq $escrowFresh.Count + $ExpectedEscrowFreshCount -gt 0 -and + $escrowFresh.Count -eq $ExpectedEscrowFreshCount -and + @($escrowFresh | Where-Object { $_ -ge 1 }).Count -eq $ExpectedEscrowFreshCount ) [pscustomobject]@{ offsiteVerify = [pscustomobject]@{ @@ -3501,6 +3577,9 @@ function Convert-AgentBackupProtectionReadback { receiptId = if ($offsiteOk) { "offsite-verify:$mtime" } else { $null } reason = if ($offsiteOk) { "read_only_metrics_verified" } elseif (-not $readback.ok) { "protection_metrics_readback_failed" } else { "offsite_verify_not_green" } configuredMetricCount = $configured.Count + configuredProviderCount = $configuredProviderNames.Count + verifiedProviderCount = $verifiedProviderNames.Count + configuredShapeValid = $configuredShapeValid freshMetricCount = $offsiteFresh.Count remoteVerifyMetricCount = $remoteVerify.Count evidenceFresh = $evidenceFresh @@ -3513,6 +3592,7 @@ function Convert-AgentBackupProtectionReadback { receiptId = if ($escrowOk) { "escrow-status:$mtime:missing-0" } else { $null } reason = if ($escrowOk) { "non_secret_metrics_verified" } elseif ($escrowMissing -lt 0) { "escrow_missing_metric_absent" } else { "escrow_evidence_incomplete" } freshMetricCount = $escrowFresh.Count + expectedFreshMetricCount = $ExpectedEscrowFreshCount evidenceFresh = $evidenceFresh evidenceAgeMinutes = $evidenceAgeMinutes maxAgeMinutes = $MaxAgeMinutes @@ -3549,7 +3629,7 @@ function Test-AgentBackupProtectionReceipts { $metricPattern = "awoooi_backup_(offsite_configured|offsite_fresh|offsite_remote_verify_ok|credential_escrow_fresh|dr_credential_escrow_missing_count)" $command = "if [ -r $quotedPath ]; then stat -c '__PROTECTION_MTIME__=%Y' $quotedPath; grep -E '^$metricPattern([ {])' $quotedPath || true; else echo __PROTECTION_MISSING__; fi" $readback = Invoke-HostSshText $hostIp $command $timeoutSeconds 1 - Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes + Convert-AgentBackupProtectionReadback $readback $maxAgeMinutes ([DateTimeOffset]::UtcNow.ToUnixTimeSeconds()) ([int]$BackupConfig.protectionEscrowExpectedCount) } function Invoke-AgentBackupReadbackContractSelfTest { @@ -3559,32 +3639,75 @@ function Invoke-AgentBackupReadbackContractSelfTest { exitCode = 0 output = @" __PROTECTION_MTIME__=1783785600 +awoooi_backup_offsite_configured{host="110",provider="b2"} 0 awoooi_backup_offsite_configured{host="110",provider="rclone"} 1 +awoooi_backup_offsite_fresh{host="110",provider="b2"} 0 awoooi_backup_offsite_fresh{host="110",provider="rclone"} 1 awoooi_backup_offsite_remote_verify_ok{host="110",provider="rclone"} 1 -awoooi_backup_credential_escrow_fresh{host="110",item="redacted-id"} 1 +awoooi_backup_credential_escrow_fresh{host="110",item="item-1"} 1 +awoooi_backup_credential_escrow_fresh{host="110",item="item-2"} 1 +awoooi_backup_credential_escrow_fresh{host="110",item="item-3"} 1 +awoooi_backup_credential_escrow_fresh{host="110",item="item-4"} 1 +awoooi_backup_credential_escrow_fresh{host="110",item="item-5"} 1 awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 "@ } $result = Convert-AgentBackupProtectionReadback $fixture 60 1783785660 $staleResult = Convert-AgentBackupProtectionReadback $fixture 60 1783792800 - $partialFixture = [pscustomobject]@{ + $noProviderFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace 'awoooi_backup_offsite_configured\{host="110",provider="rclone"\} 1', 'awoooi_backup_offsite_configured{host="110",provider="rclone"} 0' + } + $staleProviderFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace 'awoooi_backup_offsite_fresh\{host="110",provider="rclone"\} 1', 'awoooi_backup_offsite_fresh{host="110",provider="rclone"} 0' + } + $remoteVerifyFixture = [pscustomobject]@{ ok = $true route = "selftest-read-only" exitCode = 0 output = $fixture.output -replace 'awoooi_backup_offsite_remote_verify_ok\{host="110",provider="rclone"\} 1', 'awoooi_backup_offsite_remote_verify_ok{host="110",provider="rclone"} 0' } - $partialResult = Convert-AgentBackupProtectionReadback $partialFixture 60 1783785660 + $escrowMissingFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace 'awoooi_backup_dr_credential_escrow_missing_count\{host="110"\} 0', 'awoooi_backup_dr_credential_escrow_missing_count{host="110"} 1' + } + $escrowPartialFixture = [pscustomobject]@{ + ok = $true + route = "selftest-read-only" + exitCode = 0 + output = $fixture.output -replace 'awoooi_backup_credential_escrow_fresh\{host="110",item="item-5"\} 1', 'awoooi_backup_credential_escrow_fresh{host="110",item="item-5"} 0' + } + $noProviderResult = Convert-AgentBackupProtectionReadback $noProviderFixture 60 1783785660 + $staleProviderResult = Convert-AgentBackupProtectionReadback $staleProviderFixture 60 1783785660 + $remoteVerifyResult = Convert-AgentBackupProtectionReadback $remoteVerifyFixture 60 1783785660 + $escrowMissingResult = Convert-AgentBackupProtectionReadback $escrowMissingFixture 60 1783785660 + $escrowPartialResult = Convert-AgentBackupProtectionReadback $escrowPartialFixture 60 1783785660 + $terminalEligible = [bool]($result.offsiteVerify.ok -and $result.escrow.ok) $ok = [bool]( - $result.offsiteVerify.ok -and + $terminalEligible -and $result.offsiteVerify.receiptId -eq "offsite-verify:1783785600" -and + $result.offsiteVerify.configuredProviderCount -eq 1 -and + $result.offsiteVerify.verifiedProviderCount -eq 1 -and $result.escrow.ok -and + $result.escrow.freshMetricCount -eq 5 -and + $result.escrow.expectedFreshMetricCount -eq 5 -and $result.escrow.missingCount -eq 0 -and $result.escrow.receiptId -eq "escrow-status:1783785600:missing-0" -and $result.readOnly -and -not $staleResult.offsiteVerify.ok -and -not $staleResult.escrow.ok -and - -not $partialResult.offsiteVerify.ok -and + -not $noProviderResult.offsiteVerify.ok -and + -not $staleProviderResult.offsiteVerify.ok -and + -not $remoteVerifyResult.offsiteVerify.ok -and + -not $escrowMissingResult.escrow.ok -and + -not $escrowPartialResult.escrow.ok -and -not $result.backupRunPerformed -and -not $result.restoreRunPerformed -and -not $result.markerWritePerformed @@ -3592,6 +3715,15 @@ awoooi_backup_dr_credential_escrow_missing_count{host="110"} 0 [pscustomobject]@{ schemaVersion = "agent99_backup_readback_contract_selftest_v1" ok = $ok + terminalEligible = $terminalEligible + cases = [pscustomobject]@{ + optionalB2ProviderTerminalEligible = $terminalEligible + allProvidersUnconfiguredBlocked = [bool](-not $noProviderResult.offsiteVerify.ok) + configuredProviderStaleBlocked = [bool](-not $staleProviderResult.offsiteVerify.ok) + configuredProviderRemoteVerifyFailedBlocked = [bool](-not $remoteVerifyResult.offsiteVerify.ok) + escrowMissingBlocked = [bool](-not $escrowMissingResult.escrow.ok) + escrowPartialFreshnessBlocked = [bool](-not $escrowPartialResult.escrow.ok) + } result = $result } } @@ -5616,6 +5748,13 @@ function Convert-AgentHost112GuestReadback { stateWritePerformed = (& $asBool "state_write_performed") managerMarkerWritePerformed = (& $asBool "manager_marker_write_performed") managerRuntimeWritePerformed = (& $asBool "manager_runtime_write_performed") + artifactTransactionStatus = & $get "artifact_transaction_status" "unknown" + artifactTransactionPhase = & $get "artifact_transaction_phase" "unknown" + artifactTransactionStarted = (& $asBool "artifact_transaction_started") + artifactTransactionBlocked = (& $asBool "artifact_transaction_blocked") + artifactTransactionWritePerformed = (& $asBool "artifact_transaction_write_performed") + artifactTransactionPairVerified = (& $asBool "artifact_transaction_pair_verified") + artifactTransactionPriorPairVerified = (& $asBool "artifact_transaction_prior_pair_verified") receiptWritePerformed = (& $asBool "receipt_write_performed") receiptStatus = & $get "receipt_status" receiptRef = & $get "receipt_ref" "none" @@ -5792,6 +5931,14 @@ function Invoke-AgentHost112GuestRecovery { $dryRun.identityMatches -and $dryRun.bootId -eq $before.bootId -and $dryRun.managerPreflightPresent -and + -not $dryRun.artifactTransactionBlocked -and + ( + $dryRun.artifactTransactionStatus -eq "none" -or + ( + $dryRun.artifactTransactionStatus -eq "prior_pair_verified" -and + $dryRun.artifactTransactionPriorPairVerified + ) + ) -and -not $dryRun.runtimeWritePerformed -and -not $dryRun.managerRuntimeWritePerformed -and [string]$dryRun.managerAttemptCount -eq "0" -and @@ -5866,6 +6013,11 @@ function Invoke-AgentHost112GuestRecovery { $applyReceipt.identityMatches -and $applyReceipt.bootId -eq $before.bootId -and $applyReceipt.runtimeWritePerformed -and + $applyReceipt.artifactTransactionStarted -and + -not $applyReceipt.artifactTransactionBlocked -and + $applyReceipt.artifactTransactionWritePerformed -and + $applyReceipt.artifactTransactionPairVerified -and + $applyReceipt.artifactTransactionStatus -eq "committed_verified_pair" -and $applyReceipt.receiptWritePerformed -and $applyReceipt.receiptStatus -eq "written_immutable" -and -not $applyReceipt.receiptReused -and @@ -5900,6 +6052,12 @@ function Invoke-AgentHost112GuestRecovery { $durableReceiptReadback.durableReadbackReceiptLinkMatch -and $durableReceiptReadback.durableReadbackTerminalMatch -and $durableReceiptReadback.durableReadbackStatus -in @("loaded_immutable_verified", "reused_immutable_verified") -and + -not $durableReceiptReadback.artifactTransactionStarted -and + -not $durableReceiptReadback.artifactTransactionBlocked -and + -not $durableReceiptReadback.artifactTransactionWritePerformed -and + $durableReceiptReadback.artifactTransactionPairVerified -and + $durableReceiptReadback.artifactTransactionPriorPairVerified -and + $durableReceiptReadback.artifactTransactionStatus -eq "existing_pair_verified" -and $durableReceiptReadback.durableReadbackSha256 -eq $applyReceipt.durableReadbackSha256 -and $durableReceiptReadback.receiptRef -eq $applyReceipt.receiptRef -and $durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath -and diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 25f96b168..6e1b08a63 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -65,6 +65,30 @@ function Set-AgentProperty { } } +function Get-AgentHost112CanonicalVm { + param([string]$PolicyPath) + if (-not (Test-Path -LiteralPath $PolicyPath -PathType Leaf)) { + throw "Agent99 Host112 canonical policy config is unavailable: $PolicyPath" + } + $policy = Get-Content -LiteralPath $PolicyPath -Raw | ConvertFrom-Json + $candidates = @($policy.vms | Where-Object { + [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" + }) + if ($candidates.Count -ne 1) { + throw "Agent99 Host112 canonical policy must contain exactly one Host112 VM row; found $($candidates.Count)." + } + $candidate = $candidates[0] + $candidateVmx = ([string]$candidate.vmx).Trim() + if ( + [string]$candidate.name -ne "host112-kali" -or + [string]$candidate.host -ne "192.168.0.112" -or + -not $candidateVmx + ) { + throw "Agent99 Host112 canonical policy identity is invalid." + } + return ($candidate | ConvertTo-Json -Depth 10 | ConvertFrom-Json) +} + function Copy-AgentRuntimeFileWithRetry { param( [string]$Source, @@ -165,6 +189,10 @@ $stagedFiles = @($runtimeFiles | ForEach-Object { $path = Join-Path $StageDir $_ [pscustomobject]@{ name = $_; sha256 = (Get-FileHash -Algorithm SHA256 -Path $path).Hash.ToLowerInvariant() } }) +$policyConfigPath = Join-Path $StageDir "agent99.config.99.example.json" +$policyConfig = Get-Content -LiteralPath $policyConfigPath -Raw | ConvertFrom-Json +$canonicalHost112Vm = Get-AgentHost112CanonicalVm $policyConfigPath +$canonicalHost112Vmx = [string]$canonicalHost112Vm.vmx if ($ValidateOnly) { Write-DeployEvidence $true "validated" $null $stagedFiles $null @@ -172,6 +200,10 @@ if ($ValidateOnly) { exit 0 } +if (-not (Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf)) { + throw "Host112 canonical VMX path from staged policy is unavailable: $canonicalHost112Vmx" +} + New-Item -ItemType Directory -Force -Path $BinDir, $BackupDir | Out-Null $promotionRows = @() $configBackedUp = $false @@ -226,6 +258,36 @@ try { to = ((@($directHosts + "192.168.0.112") | Select-Object -Unique) -join ",") } } + Add-DefaultProperty $config "vms" @() + $host112Vm = @($config.vms | Where-Object { + [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" + }) + $nonHost112Vms = @($config.vms | Where-Object { + -not ([string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali") + }) + $canonicalHost112Json = $canonicalHost112Vm | ConvertTo-Json -Depth 10 -Compress + $currentHost112Json = if ($host112Vm.Count -eq 1) { + $host112Vm[0] | ConvertTo-Json -Depth 10 -Compress + } else { + "" + } + $host112MigrationRequired = [bool]( + $host112Vm.Count -ne 1 -or + $currentHost112Json -cne $canonicalHost112Json + ) + if ($host112MigrationRequired) { + $previousHost112Identity = if ($host112Vm.Count -eq 0) { + "missing" + } else { + (@($host112Vm | ForEach-Object { "$([string]$_.name)|$([string]$_.host)|$([string]$_.vmx)" }) -join ";") + } + $script:ConfigMigrations += [pscustomobject]@{ + name = "host112_canonical_vmx_path" + from = $previousHost112Identity + to = $canonicalHost112Vmx + } + } + Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm)) Add-DefaultProperty $config "autoRecovery" ([pscustomobject]@{}) Add-DefaultProperty $config.autoRecovery "enabled" $true Add-DefaultProperty $config.autoRecovery "triggerOnHostUnreachable" $true @@ -259,7 +321,6 @@ try { Add-DefaultProperty $config.performance "processCpuHostPercentWarning" 25 Add-DefaultProperty $config.performance "processCpuHostPercentCritical" 50 - $policyConfig = Get-Content (Join-Path $StageDir "agent99.config.99.example.json") -Raw | ConvertFrom-Json foreach ($sourceAction in @($policyConfig.performance.loadShedding.actions | Where-Object { $_.PSObject.Properties["policyVersion"] })) { $liveAction = $config.performance.loadShedding.actions | Where-Object { $_.name -eq $sourceAction.name } | Select-Object -First 1 if (-not $liveAction) { continue } @@ -281,8 +342,20 @@ try { $persistedConfig = Get-Content $ConfigPath -Raw | ConvertFrom-Json $persistedHost112User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } $persistedDirectHosts = if ($persistedConfig.k3s -and $persistedConfig.k3s.PSObject.Properties["directHosts"]) { @($persistedConfig.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } - if ($persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts) { - throw "Host112 canonical direct SSH config post-verifier failed." + $persistedHost112Vm = @($persistedConfig.vms | Where-Object { + [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" + }) + $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } + if ( + $persistedHost112User -ne "kali" -or + "192.168.0.112" -notin $persistedDirectHosts -or + $persistedHost112Vm.Count -ne 1 -or + [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or + [string]$persistedHost112Vm[0].host -ne [string]$canonicalHost112Vm.host -or + $persistedHost112Vmx -ne $canonicalHost112Vmx -or + -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) + ) { + throw "Host112 canonical SSH and VMX config post-verifier failed." } $manifestRows = @() diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index ce1b84c53..0e1358b90 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -261,9 +261,12 @@ Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_a Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" -Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed by a later healthy observation without its attributed start and immutable receipt/readback" +Add-SyntheticCheck "recovery:host112_apply_receipt_closure" ($control.Contains("managerApplyVerified") -and $control.Contains('durableReceiptTerminal -eq "verified_healthy"') -and $control.Contains('durableReadbackReceiptLinkMatch') -and $control.Contains('durableReadbackTerminalMatch') -and $control.Contains('artifactTransactionStatus -eq "committed_verified_pair"') -and $control.Contains('artifactTransactionPriorPairVerified') -and $control.Contains('managerAttemptCount -eq "1"') -and $control.Contains('managerStartExit -eq "0"') -and $control.Contains("durableReceiptVerified") -and $control.Contains('afterVerifierRole = "independent_second_verifier_only"')) "Host112 apply cannot be closed without its fixed-WAL commit, attributed start and immutable receipt/readback" Add-SyntheticCheck "recovery:host112_candidate_split" ($control.Contains("generalCandidateRequired") -and $control.Contains("managerCandidateRequired") -and $control.Contains("generalApplyEligible") -and $control.Contains("managerApplyEligible") -and $control.Contains("managerNoAttemptPreserved")) "Host112 general guest repair is independent from manager eligibility" Add-SyntheticCheck "recovery:host112_timeout_contract" ($control.Contains("Get-AgentHost112TimeoutContract") -and $control.Contains("executorReserveSeconds -ge 60") -and $control.Contains("queueReserveSeconds -ge 300") -and $control.Contains("clientReserveSeconds -ge 120") -and $control.Contains('$process.WaitForExit($script:Agent99QueueOuterDeadlineSeconds * 1000)')) "Host112 executor, SSH lock, queue and client deadlines have explicit reserves" +$deployer = Get-Content (Join-Path $SourceRoot "agent99-deploy.ps1") -Raw +$hostConfig = Get-Content (Join-Path $SourceRoot "agent99.config.99.example.json") -Raw +Add-SyntheticCheck "recovery:host112_canonical_vmx" ($deployer.Contains('name = "host112_canonical_vmx_path"') -and $deployer.Contains('Get-AgentHost112CanonicalVm $policyConfigPath') -and $deployer.Contains('Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))') -and $hostConfig.Contains('S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx')) "Agent99 normalizes Host112 from one staged VMX identity with a bounded post-verifier" $failures = @($checks | Where-Object { -not $_.ok }) $result = [pscustomobject]@{ diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 5099fb06b..4e6d52afc 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -41,7 +41,7 @@ { "name": "host112-kali", "host": "192.168.0.112", - "vmx": "D:\\Downloads\\kali-linux-2025.4-vmware-amd64\\kali-linux-2025.4-vmware-amd64.vmwarevm\\kali-linux-2025.4-vmware-amd64.vmx", + "vmx": "S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx", "priority": 30 }, { diff --git a/apps/api/src/services/executor_trust_boundary_readback.py b/apps/api/src/services/executor_trust_boundary_readback.py index 723c3dc2d..4810413ba 100644 --- a/apps/api/src/services/executor_trust_boundary_readback.py +++ b/apps/api/src/services/executor_trust_boundary_readback.py @@ -8,14 +8,15 @@ import os import re import time from collections.abc import Mapping +from datetime import UTC, datetime from typing import Any import structlog logger = structlog.get_logger(__name__) -SCHEMA_VERSION = "awoooi_executor_trust_boundary_readback_v3" -RECEIPT_SCHEMA_VERSION = "awoooi_executor_boundary_verification_v3" +SCHEMA_VERSION = "awoooi_executor_trust_boundary_readback_v4" +RECEIPT_SCHEMA_VERSION = "awoooi_executor_boundary_verification_v4" DEFAULT_NAMESPACE = "awoooi-prod" DEFAULT_CONFIGMAP_NAME = "awoooi-executor-boundary-verification" _CACHE_TTL_SECONDS = 20.0 @@ -59,6 +60,17 @@ _CANONICAL_LEARNING_REQUIRED_TRUE_FIELDS = ( "learning_receipts_public_safe", ) _DB_WORKLOAD_IDS = ("api", "worker", "broker") +_DB_WORKLOAD_BUDGETS = { + "api": {"connection_limit": 12, "required_connection_budget": 8}, + "worker": {"connection_limit": 8, "required_connection_budget": 5}, + "broker": {"connection_limit": 4, "required_connection_budget": 2}, +} +_DB_IDENTITY_VERIFIER = ( + "runtime_role_fingerprint_role_limit_null_pool_http_concurrency_and_" + "representative_preflight_v4" +) +_DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS = 15 * 60 +_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS = 5 * 60 _ROLE_FINGERPRINT_PATTERN = re.compile(r"^[0-9a-f]{64}$") _cache: tuple[float, dict[str, Any]] | None = None @@ -79,11 +91,40 @@ def _role_fingerprint(value: str | None) -> str | None: return normalized if _ROLE_FINGERPRINT_PATTERN.fullmatch(normalized) else None +def _parse_utc_timestamp(value: str | None) -> datetime | None: + normalized = str(value or "").strip() + if not normalized: + return None + if normalized.endswith("Z"): + normalized = f"{normalized[:-1]}+00:00" + try: + parsed = datetime.fromisoformat(normalized) + except ValueError: + return None + if parsed.tzinfo is None: + return None + return parsed.astimezone(UTC) + + def _build_database_identity_boundary( receipt: Mapping[str, str], *, receipt_ref: str, + now_utc: datetime, ) -> dict[str, Any]: + verified_at = receipt.get("verified_at") + verified_at_utc = _parse_utc_timestamp(verified_at) + receipt_age_seconds = ( + int((now_utc - verified_at_utc).total_seconds()) + if verified_at_utc is not None + else None + ) + capacity_receipt_fresh = bool( + receipt_age_seconds is not None + and -_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS + <= receipt_age_seconds + <= _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS + ) workloads: dict[str, dict[str, Any]] = {} for workload_id in _DB_WORKLOAD_IDS: workloads[workload_id] = { @@ -99,9 +140,15 @@ def _build_database_identity_boundary( "required_connection_budget": _int_or_none( receipt.get(f"{workload_id}_required_connection_budget") ), - "concurrent_connection_probe_count": _int_or_none( + "active_role_connection_count": _int_or_none( + receipt.get(f"{workload_id}_active_role_connection_count") + ), + "available_connection_headroom": _int_or_none( + receipt.get(f"{workload_id}_available_connection_headroom") + ), + "representative_connection_probe_count": _int_or_none( receipt.get( - f"{workload_id}_concurrent_connection_probe_count" + f"{workload_id}_representative_connection_probe_count" ) ), "database_null_pool": _truthy( @@ -143,23 +190,36 @@ def _build_database_identity_boundary( and len(set(role_fingerprints)) == len(_DB_WORKLOAD_IDS) and _truthy(receipt.get("distinct_db_role_fingerprints")) ) - monolithic_env_from_absent = bool( - workloads["api"]["monolithic_secret_env_from_absent"] - and workloads["worker"]["monolithic_secret_env_from_absent"] + monolithic_env_from_absent = all( + workloads[workload_id]["monolithic_secret_env_from_absent"] is True + for workload_id in _DB_WORKLOAD_IDS ) null_pool_verified = all( workloads[workload_id]["database_null_pool"] is True for workload_id in _DB_WORKLOAD_IDS ) workload_budget_verified = all( - isinstance(workloads[workload_id]["connection_limit"], int) - and isinstance(workloads[workload_id]["required_connection_budget"], int) - and workloads[workload_id]["connection_limit"] >= workloads[workload_id][ - "required_connection_budget" - ] - and workloads[workload_id]["required_connection_budget"] > 0 + workloads[workload_id]["connection_limit"] + == _DB_WORKLOAD_BUDGETS[workload_id]["connection_limit"] + and workloads[workload_id]["required_connection_budget"] + == _DB_WORKLOAD_BUDGETS[workload_id]["required_connection_budget"] + and isinstance( + workloads[workload_id]["active_role_connection_count"], int + ) + and workloads[workload_id]["active_role_connection_count"] >= 0 + and isinstance( + workloads[workload_id]["available_connection_headroom"], int + ) + and workloads[workload_id]["available_connection_headroom"] + == ( + workloads[workload_id]["connection_limit"] + - workloads[workload_id]["active_role_connection_count"] + ) + and workloads[workload_id]["available_connection_headroom"] + >= workloads[workload_id]["required_connection_budget"] for workload_id in _DB_WORKLOAD_IDS ) + verifier_verified = receipt.get("db_identity_verifier") == _DB_IDENTITY_VERIFIER api_http_concurrency_verified = _truthy( receipt.get("api_http_concurrency_probe_passed") ) @@ -170,12 +230,16 @@ def _build_database_identity_boundary( workloads[workload_id]["required_connection_budget"], int ) and isinstance( - workloads[workload_id]["concurrent_connection_probe_count"], + workloads[workload_id][ + "representative_connection_probe_count" + ], int, ) and workloads[workload_id]["required_connection_budget"] > 0 - and workloads[workload_id]["concurrent_connection_probe_count"] - >= workloads[workload_id]["required_connection_budget"] + and workloads[workload_id][ + "representative_connection_probe_count" + ] + >= 1 and workloads[workload_id]["table_privilege_gap_count"] == 0 and workloads[workload_id]["sequence_privilege_gap_count"] == 0 for workload_id in _DB_WORKLOAD_IDS @@ -194,6 +258,8 @@ def _build_database_identity_boundary( and null_pool_verified and workload_budget_verified and representative_preflights_verified + and verifier_verified + and capacity_receipt_fresh and _truthy(receipt.get("connection_budget_verified")) ) @@ -202,12 +268,22 @@ def _build_database_identity_boundary( blockers.append("api_monolithic_secret_env_from_present") if not workloads["worker"]["monolithic_secret_env_from_absent"]: blockers.append("worker_monolithic_secret_env_from_present") + if not workloads["broker"]["monolithic_secret_env_from_absent"]: + blockers.append("broker_monolithic_secret_env_from_present") if not distinct_secret_refs: blockers.append("database_secret_refs_not_distinct") if not distinct_role_fingerprints: blockers.append("database_role_fingerprints_not_distinct") if not api_http_concurrency_verified: blockers.append("api_http_concurrency_probe_not_verified") + if not verifier_verified: + blockers.append("database_identity_verifier_not_verified") + if verified_at_utc is None: + blockers.append("database_capacity_receipt_timestamp_invalid") + elif receipt_age_seconds < -_DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS: + blockers.append("database_capacity_receipt_timestamp_in_future") + elif receipt_age_seconds > _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS: + blockers.append("database_capacity_receipt_stale") for workload_id in _DB_WORKLOAD_IDS: workload = workloads[workload_id] if workload["role_fingerprint"] is None: @@ -217,11 +293,10 @@ def _build_database_identity_boundary( if ( workload["representative_preflight_passed"] is not True or not isinstance( - workload["concurrent_connection_probe_count"], int + workload["representative_connection_probe_count"], int ) or not isinstance(workload["required_connection_budget"], int) - or workload["concurrent_connection_probe_count"] - < workload["required_connection_budget"] + or workload["representative_connection_probe_count"] < 1 or workload["table_privilege_gap_count"] != 0 or workload["sequence_privilege_gap_count"] != 0 ): @@ -230,11 +305,17 @@ def _build_database_identity_boundary( ) connection_limit = workload["connection_limit"] required_budget = workload["required_connection_budget"] + active_role_connections = workload["active_role_connection_count"] + available_headroom = workload["available_connection_headroom"] + expected_budget = _DB_WORKLOAD_BUDGETS[workload_id] if ( - not isinstance(connection_limit, int) - or not isinstance(required_budget, int) - or required_budget <= 0 - or connection_limit < required_budget + connection_limit != expected_budget["connection_limit"] + or required_budget != expected_budget["required_connection_budget"] + or not isinstance(active_role_connections, int) + or active_role_connections < 0 + or not isinstance(available_headroom, int) + or available_headroom != connection_limit - active_role_connections + or available_headroom < required_budget ): blockers.append(f"{workload_id}_connection_budget_not_verified") if not db_identity_isolated: @@ -250,8 +331,16 @@ def _build_database_identity_boundary( workloads[workload_id]["required_connection_budget"] for workload_id in _DB_WORKLOAD_IDS ] + active_role_connection_counts = [ + workloads[workload_id]["active_role_connection_count"] + for workload_id in _DB_WORKLOAD_IDS + ] + available_headrooms = [ + workloads[workload_id]["available_connection_headroom"] + for workload_id in _DB_WORKLOAD_IDS + ] return { - "schema_version": "awoooi_workload_db_identity_budget_readback_v2", + "schema_version": "awoooi_workload_db_identity_budget_readback_v4", "status": "verified_ready" if not blockers else "in_progress", "receipt_ref": receipt_ref, "verifier": receipt.get("db_identity_verifier") or None, @@ -265,10 +354,15 @@ def _build_database_identity_boundary( "worker_monolithic_secret_env_from_absent": workloads["worker"][ "monolithic_secret_env_from_absent" ], + "broker_monolithic_secret_env_from_absent": workloads["broker"][ + "monolithic_secret_env_from_absent" + ], "distinct_database_secret_refs": distinct_secret_refs, "distinct_db_role_fingerprints": distinct_role_fingerprints, "database_null_pool_verified": null_pool_verified, "workload_connection_budgets_verified": workload_budget_verified, + "database_identity_verifier_verified": verifier_verified, + "database_capacity_receipt_fresh": capacity_receipt_fresh, "representative_db_preflights_verified": ( representative_preflights_verified ), @@ -287,6 +381,28 @@ def _build_database_identity_boundary( if all(isinstance(value, int) for value in required_budgets) else None ), + "active_role_connection_total": ( + sum(active_role_connection_counts) + if all( + isinstance(value, int) + for value in active_role_connection_counts + ) + else None + ), + "available_headroom_total": ( + sum(available_headrooms) + if all(isinstance(value, int) for value in available_headrooms) + else None + ), + }, + "signal_freshness": { + "verified_at": verified_at or None, + "age_seconds": receipt_age_seconds, + "max_age_seconds": _DB_CAPACITY_RECEIPT_MAX_AGE_SECONDS, + "future_clock_skew_seconds": ( + _DB_CAPACITY_RECEIPT_FUTURE_SKEW_SECONDS + ), + "fresh": capacity_receipt_fresh, }, "active_blockers": blockers, "writes_on_read": False, @@ -407,6 +523,7 @@ def build_executor_trust_boundary_readback( namespace: str = DEFAULT_NAMESPACE, configmap_name: str = DEFAULT_CONFIGMAP_NAME, error_type: str | None = None, + now_utc: datetime | None = None, ) -> dict[str, Any]: receipt = {str(key): str(value) for key, value in (data or {}).items()} receipt_ref = f"configmap:{namespace}/{configmap_name}" @@ -433,6 +550,7 @@ def build_executor_trust_boundary_readback( database_identity_boundary = _build_database_identity_boundary( receipt, receipt_ref=receipt_ref, + now_utc=(now_utc or datetime.now(UTC)).astimezone(UTC), ) autonomous_single_writer_source_boundary = ( _build_autonomous_single_writer_source_boundary( diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 2a636b14e..5b2266948 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -163,7 +163,7 @@ _WINDOWS99_EXPECTED_VMX_CANDIDATES = { r"D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx", ], "112": [ - r"D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx", + r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx", ], } _WINDOWS99_VMX_REPAIR_FORBIDDEN_ACTIONS = [ diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index fe9994589..3f688d80c 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -121,12 +121,29 @@ def test_agent99_backupcheck_reads_offsite_and_escrow_without_mutation() -> None assert '"offsite_verify_evidence_ref"' in source assert '"escrow_evidence_ref"' in source assert "protectionMaxAgeMinutes" in source + assert "protectionEscrowExpectedCount" in source assert "$evidenceAgeMinutes -ge -5" in source assert "$evidenceAgeMinutes -le" in source - assert "-eq $configured.Count" in source - assert "-eq $offsiteFresh.Count" in source - assert "-eq $remoteVerify.Count" in source + assert "function Get-AgentBackupProviderMetricRows" in source + assert "$offsiteFresh.Count -eq $offsiteFreshRows.Count" in source + assert "$remoteVerify.Count -eq $remoteVerifyRows.Count" in source + assert "$configuredProviderNames.Count -gt 0" in source + assert "$verifiedProviderNames.Count -eq $configuredProviderNames.Count" in source + assert "$escrowFresh.Count -eq $ExpectedEscrowFreshCount" in source + assert "$escrowMissingValues.Count -eq 1" in source + assert 'provider="b2"} 0' in source + assert 'provider="rclone"} 1' in source + assert "optionalB2ProviderTerminalEligible" in source + assert "allProvidersUnconfiguredBlocked" in source + assert "configuredProviderStaleBlocked" in source + assert "configuredProviderRemoteVerifyFailedBlocked" in source + assert "escrowMissingBlocked" in source + assert "escrowPartialFreshnessBlocked" in source assert "-not $staleResult.offsiteVerify.ok" in source - assert "-not $partialResult.offsiteVerify.ok" in source + assert "-not $noProviderResult.offsiteVerify.ok" in source + assert "-not $staleProviderResult.offsiteVerify.ok" in source + assert "-not $remoteVerifyResult.offsiteVerify.ok" in source + assert "-not $escrowMissingResult.escrow.ok" in source + assert "-not $escrowPartialResult.escrow.ok" in source for mutation in ("rclone ", "rsync ", " rm ", "restore ", "Set-Content"): assert mutation not in readback diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index bef686919..6b0243e4b 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -133,10 +133,16 @@ def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> assert function.count("Invoke-AgentHost112SshText $dryRunCommand") == 2 assert 'schemaVersion = "agent99_host112_guest_recovery_v3"' in function assert '$dryRun.managerPreflightStatus -eq "ready"' in function + assert "-not $dryRun.artifactTransactionBlocked" in function + assert '$dryRun.artifactTransactionStatus -eq "prior_pair_verified"' in function assert '$before.transportOk -and' in function assert '$before.identityMatches -and' in function assert "Invoke-AgentHost112SshText $applyCommand $script:Agent99Host112ApplyTransportTimeoutSeconds 1" in function assert 'receiptStatus -eq "written_immutable"' in function + assert '$applyReceipt.artifactTransactionStatus -eq "committed_verified_pair"' in function + assert "$applyReceipt.artifactTransactionPairVerified" in function + assert "$durableReceiptReadback.artifactTransactionPriorPairVerified" in function + assert '$durableReceiptReadback.artifactTransactionStatus -eq "existing_pair_verified"' in function assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function assert "$applyReceipt.durableReadbackTerminalMatch" in function assert "$before.durableReadbackReceiptLinkMatch" in function diff --git a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py index 23da26b41..a2aaec103 100644 --- a/apps/api/tests/test_awoooi_priority_work_order_readback_api.py +++ b/apps/api/tests/test_awoooi_priority_work_order_readback_api.py @@ -127,7 +127,7 @@ def _autonomous_runtime_control_trust_boundary_closed() -> dict: } } payload["executor_trust_boundary"] = { - "schema_version": "awoooi_executor_trust_boundary_readback_v3", + "schema_version": "awoooi_executor_trust_boundary_readback_v4", "status": "verified_ready", "production_boundary_verified": True, "full_workload_boundary_verified": True, @@ -137,7 +137,7 @@ def _autonomous_runtime_control_trust_boundary_closed() -> dict: "configmap:awoooi-prod/awoooi-executor-boundary-verification" ), "database_identity_boundary": { - "schema_version": "awoooi_workload_db_identity_budget_readback_v1", + "schema_version": "awoooi_workload_db_identity_budget_readback_v4", "status": "verified_ready", "receipt_ref": ( "configmap:awoooi-prod/awoooi-executor-boundary-verification" diff --git a/apps/api/tests/test_executor_trust_boundary_readback.py b/apps/api/tests/test_executor_trust_boundary_readback.py index 869b33bcd..f45087216 100644 --- a/apps/api/tests/test_executor_trust_boundary_readback.py +++ b/apps/api/tests/test_executor_trust_boundary_readback.py @@ -1,3 +1,5 @@ +from datetime import UTC, datetime + from src.services.executor_trust_boundary_readback import ( RECEIPT_SCHEMA_VERSION, build_executor_trust_boundary_readback, @@ -8,7 +10,7 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]: return { "schema_version": RECEIPT_SCHEMA_VERSION, "verified_source_sha": source_sha, - "verified_at": "2026-07-11T01:00:00Z", + "verified_at": datetime.now(UTC).isoformat().replace("+00:00", "Z"), "workflow_run_id": "4870", "verifier": "kubectl_auth_can_i_and_live_mount_readback", "api_service_account": "awoooi-api", @@ -54,8 +56,8 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]: "trust_failure_fail_closed": "true", "learning_receipts_public_safe": "true", "db_identity_verifier": ( - "runtime_role_fingerprint_secret_projection_and_" - "representative_preflight_v2" + "runtime_role_fingerprint_role_limit_null_pool_http_concurrency_" + "and_representative_preflight_v4" ), "api_database_secret_ref": "awoooi-api-db", "worker_database_secret_ref": "awoooi-worker-db", @@ -69,9 +71,15 @@ def _verified_receipt(source_sha: str = "abc123") -> dict[str, str]: "api_required_connection_budget": "8", "worker_required_connection_budget": "5", "broker_required_connection_budget": "2", - "api_concurrent_connection_probe_count": "8", - "worker_concurrent_connection_probe_count": "5", - "broker_concurrent_connection_probe_count": "2", + "api_active_role_connection_count": "2", + "worker_active_role_connection_count": "1", + "broker_active_role_connection_count": "1", + "api_available_connection_headroom": "10", + "worker_available_connection_headroom": "7", + "broker_available_connection_headroom": "3", + "api_representative_connection_probe_count": "1", + "worker_representative_connection_probe_count": "1", + "broker_representative_connection_probe_count": "1", "api_representative_db_preflight_passed": "true", "worker_representative_db_preflight_passed": "true", "broker_representative_db_preflight_passed": "true", @@ -114,12 +122,16 @@ def test_executor_trust_boundary_requires_live_controls_and_matching_source() -> assert database_boundary["connection_budget"] == { "observed_total": 24, "required_total": 15, + "active_role_connection_total": 4, + "available_headroom_total": 20, } assert database_boundary["controls"][ "representative_db_preflights_verified" ] is True assert database_boundary["secret_values_read"] is False assert database_boundary["role_names_exposed"] is False + assert database_boundary["signal_freshness"]["fresh"] is True + assert database_boundary["signal_freshness"]["max_age_seconds"] == 900 source_boundary = readback["autonomous_single_writer_source_boundary"] assert source_boundary["source_boundary_verified"] is True assert source_boundary["active_blockers"] == [] @@ -203,6 +215,30 @@ def test_executor_trust_boundary_rejects_stale_or_incomplete_receipt() -> None: assert "worker_ssh_egress_denied_not_verified" in incomplete["active_blockers"] +def test_executor_trust_boundary_expires_database_capacity_snapshot() -> None: + receipt = _verified_receipt() + receipt["verified_at"] = "2026-07-11T01:00:00Z" + + readback = build_executor_trust_boundary_readback( + receipt, + deployed_source_sha="abc123", + now_utc=datetime(2026, 7, 14, 1, 0, tzinfo=UTC), + ) + + assert readback["production_boundary_verified"] is True + assert readback["full_workload_boundary_verified"] is False + boundary = readback["database_identity_boundary"] + assert boundary["connection_budget_verified"] is False + assert boundary["signal_freshness"] == { + "verified_at": "2026-07-11T01:00:00Z", + "age_seconds": 259200, + "max_age_seconds": 900, + "future_clock_skew_seconds": 300, + "fresh": False, + } + assert "database_capacity_receipt_stale" in boundary["active_blockers"] + + def test_executor_trust_boundary_keeps_shared_database_identity_in_progress() -> None: shared = _verified_receipt() for workload_id in ("api", "worker", "broker"): @@ -258,9 +294,9 @@ def test_executor_trust_boundary_requires_representative_database_preflight() -> ) -def test_executor_trust_boundary_rejects_static_budget_without_concurrency_probe() -> None: +def test_executor_trust_boundary_rejects_missing_representative_probe() -> None: receipt = _verified_receipt() - receipt["api_concurrent_connection_probe_count"] = "4" + receipt["api_representative_connection_probe_count"] = "0" readback = build_executor_trust_boundary_readback( receipt, @@ -288,3 +324,62 @@ def test_executor_trust_boundary_requires_api_http_concurrency_probe() -> None: assert "api_http_concurrency_probe_not_verified" in ( database_boundary["active_blockers"] ) + + +def test_executor_trust_boundary_requires_broker_secret_isolation() -> None: + receipt = _verified_receipt() + receipt["broker_monolithic_secret_env_from_absent"] = "false" + + readback = build_executor_trust_boundary_readback( + receipt, + deployed_source_sha="abc123", + ) + + database_boundary = readback["database_identity_boundary"] + assert readback["full_workload_boundary_verified"] is False + assert database_boundary["db_identity_isolated"] is False + assert "broker_monolithic_secret_env_from_present" in ( + database_boundary["active_blockers"] + ) + + +def test_executor_trust_boundary_requires_live_connection_headroom() -> None: + receipt = _verified_receipt() + receipt["worker_active_role_connection_count"] = "7" + receipt["worker_available_connection_headroom"] = "1" + + readback = build_executor_trust_boundary_readback( + receipt, + deployed_source_sha="abc123", + ) + + database_boundary = readback["database_identity_boundary"] + assert readback["full_workload_boundary_verified"] is False + assert database_boundary["connection_budget_verified"] is False + assert "worker_connection_budget_not_verified" in ( + database_boundary["active_blockers"] + ) + + +def test_executor_trust_boundary_binds_budget_and_verifier_contract() -> None: + wrong_budget = _verified_receipt() + wrong_budget["worker_required_connection_budget"] = "4" + wrong_budget_readback = build_executor_trust_boundary_readback( + wrong_budget, + deployed_source_sha="abc123", + ) + assert wrong_budget_readback["full_workload_boundary_verified"] is False + assert "worker_connection_budget_not_verified" in ( + wrong_budget_readback["database_identity_boundary"]["active_blockers"] + ) + + wrong_verifier = _verified_receipt() + wrong_verifier["db_identity_verifier"] = "unknown" + wrong_verifier_readback = build_executor_trust_boundary_readback( + wrong_verifier, + deployed_source_sha="abc123", + ) + assert wrong_verifier_readback["full_workload_boundary_verified"] is False + assert "database_identity_verifier_not_verified" in ( + wrong_verifier_readback["database_identity_boundary"]["active_blockers"] + ) diff --git a/apps/api/tests/test_signal_worker_ansible_executor_binding.py b/apps/api/tests/test_signal_worker_ansible_executor_binding.py index dc14e22fd..6fd5df564 100644 --- a/apps/api/tests/test_signal_worker_ansible_executor_binding.py +++ b/apps/api/tests/test_signal_worker_ansible_executor_binding.py @@ -188,7 +188,7 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "repair-ssh-key|repair-known-hosts|ssh-mcp-key" in workflow assert "legacy cluster-wide executor binding still exists" in workflow assert "awoooi-executor-boundary-verification" in workflow - assert "awoooi_executor_boundary_verification_v3" in workflow + assert "awoooi_executor_boundary_verification_v4" in workflow assert "single_writer_source_verifier=cd_tests_and_production_source_sha_v1" in workflow assert "decision_manager_queue_only=true" in workflow assert "webhook_router_queue_only=true" in workflow @@ -234,8 +234,8 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert 'database_boundary.get("connection_budget_verified") is True' in workflow assert "verified_source_sha" in workflow assert ( - "runtime_role_fingerprint_secret_projection_and_" - "representative_preflight_v2" + "runtime_role_fingerprint_role_limit_null_pool_http_concurrency_and_" + "representative_preflight_v4" ) in workflow assert "db_identity_isolated" in workflow assert "connection_budget_verified" in workflow @@ -245,12 +245,27 @@ def test_cd_prunes_and_verifies_api_executor_boundary_in_production() -> None: assert "current_user AS role_name" in workflow assert "hashlib.sha256" in workflow assert "representative_db_preflights_verified" in workflow - assert "api_required_connection_budget=8" in workflow - assert "worker_required_connection_budget=5" in workflow - assert "broker_required_connection_budget=2" in workflow - assert "api_concurrent_connection_probe_count" in workflow - assert "worker_concurrent_connection_probe_count" in workflow - assert "broker_concurrent_connection_probe_count" in workflow + assert 'api_required_connection_budget="$API_DB_REQUIRED_CONNECTION_BUDGET"' in workflow + assert 'worker_required_connection_budget="$WORKER_DB_REQUIRED_CONNECTION_BUDGET"' in workflow + assert 'broker_required_connection_budget="$BROKER_DB_REQUIRED_CONNECTION_BUDGET"' in workflow + assert "api_representative_connection_probe_count" in workflow + assert "worker_representative_connection_probe_count" in workflow + assert "broker_representative_connection_probe_count" in workflow + assert "api_active_role_connection_count" in workflow + assert "worker_active_role_connection_count" in workflow + assert "broker_active_role_connection_count" in workflow + assert "api_available_connection_headroom" in workflow + assert "worker_available_connection_headroom" in workflow + assert "broker_available_connection_headroom" in workflow + assert '"$BROKER_MONOLITHIC_SECRET_ENVFROM_ABSENT" = "true"' in workflow + assert '"$WORKER_DB_REQUIRED_CONNECTION_BUDGET" -eq 5' in workflow + assert "FROM pg_stat_activity" in workflow + assert "pid <> pg_backend_pid()" in workflow + assert "read_workload_db_identity awoooi-api api 1 8" in workflow + assert "read_workload_db_identity awoooi-worker worker 1 5" in workflow + assert "awoooi-ansible-executor-broker broker 1 2" in workflow + assert "AWOOOI_DB_REPRESENTATIVE_PROBE_COUNT" in workflow + assert "AWOOOI_DB_REQUIRED_CONNECTION_BUDGET" in workflow assert "api_http_concurrency_probe_passed" in workflow assert "ThreadPoolExecutor" in workflow assert "max_workers=8" in workflow diff --git a/apps/api/tests/test_workload_database_identity_projection.py b/apps/api/tests/test_workload_database_identity_projection.py index 9fb230ff0..1ffde1910 100644 --- a/apps/api/tests/test_workload_database_identity_projection.py +++ b/apps/api/tests/test_workload_database_identity_projection.py @@ -102,12 +102,14 @@ def test_workload_database_bootstrap_keeps_secret_values_off_output() -> None: assert "docker exec -i -u postgres" in script assert "table_privilege_gap_count" in script assert "sequence_privilege_gap_count" in script - assert "REQUIRED_CONCURRENT_CONNECTIONS" in script - assert "concurrent_connection_probe_count" in script - assert "existing_role_connection_count" in script - assert "additional_connection_probe_count" in script + assert "REPRESENTATIVE_CONNECTION_PROBE_COUNT" in script + assert "REQUIRED_CONCURRENT_CONNECTIONS" not in script + assert "representative_connection_probe_count" in script + assert "active_role_connection_count" in script + assert "available_connection_headroom" in script + assert "connection_headroom_below_required" in script assert "FROM pg_stat_activity" in script - assert "required_concurrency\n - existing_role_connection_count" in script + assert "pid <> pg_backend_pid()" in script assert "for preflight_attempt in 1 2 3" in script assert "preflight_attempts_exhausted=3" in script assert 'text("SELECT pg_sleep(0.25)")' in script diff --git a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh index f5d9152a4..904131826 100755 --- a/scripts/ops/awoooi-workload-db-identity-bootstrap.sh +++ b/scripts/ops/awoooi-workload-db-identity-bootstrap.sh @@ -77,6 +77,14 @@ workload_required_connection_budget() { esac } +workload_representative_probe_count() { + # Capacity is proven by the PostgreSQL role limit and production HTTP + # concurrency. The isolated pod opens one representative connection for + # role, privilege, and query verification without consuming the live + # workload's entire role budget. + printf '1' +} + workload_deployment() { case "$1" in api) printf 'awoooi-api|api' ;; @@ -222,6 +230,7 @@ preflight_workload_once() { secret="$2" connection_limit="$3" required_connection_budget="$4" + representative_probe_count="$5" deployment_spec="$(workload_deployment "$workload")" deployment="${deployment_spec%%|*}" container="${deployment_spec#*|}" @@ -262,7 +271,9 @@ spec: key: DATABASE_URL - name: EXPECTED_CONNECTION_LIMIT value: "${connection_limit}" - - name: REQUIRED_CONCURRENT_CONNECTIONS + - name: REPRESENTATIVE_CONNECTION_PROBE_COUNT + value: "${representative_probe_count}" + - name: REQUIRED_CONNECTION_BUDGET value: "${required_connection_budget}" - name: WORKLOAD_ID value: "${workload}" @@ -287,7 +298,15 @@ spec: async with engine.connect() as conn: identity = ( await conn.execute(text(""" - SELECT current_user AS role_name, rolconnlimit + SELECT + current_user AS role_name, + rolconnlimit, + ( + SELECT count(*) + FROM pg_stat_activity + WHERE usename = current_user + AND pid <> pg_backend_pid() + ) AS active_role_connection_count FROM pg_roles WHERE rolname = current_user """)) ).mappings().one() @@ -333,39 +352,38 @@ spec: AND NOT has_sequence_privilege(current_user, sequence.oid, 'UPDATE')) ) """))).scalar_one()) - required_concurrency = int( - os.environ["REQUIRED_CONCURRENT_CONNECTIONS"] - ) - existing_role_connection_count = int(( - await conn.execute(text(""" - SELECT count(*) - FROM pg_stat_activity - WHERE usename = current_user - """)) - ).scalar_one()) - additional_probe_count = max( - 0, - required_concurrency - - existing_role_connection_count, - ) + representative_probe_count = int( + os.environ["REPRESENTATIVE_CONNECTION_PROBE_COUNT"] + ) + if representative_probe_count < 1: + raise RuntimeError("representative_probe_count_invalid") - async def probe_connection() -> None: - async with engine.connect() as probe: - await probe.execute( - text("SELECT pg_sleep(0.25)") - ) + async def probe_connection() -> None: + async with engine.connect() as probe: + await probe.execute(text("SELECT pg_sleep(0.25)")) - await asyncio.gather(*( - probe_connection() - for _ in range(additional_probe_count) - )) + await asyncio.gather(*( + probe_connection() + for _ in range(representative_probe_count) + )) finally: await engine.dispose() expected_limit = int(os.environ["EXPECTED_CONNECTION_LIMIT"]) + required_budget = int(os.environ["REQUIRED_CONNECTION_BUDGET"]) observed_limit = int(identity["rolconnlimit"]) + active_role_connection_count = int( + identity["active_role_connection_count"] + ) + available_connection_headroom = ( + observed_limit - active_role_connection_count + ) if observed_limit != expected_limit: raise RuntimeError("connection_limit_mismatch") + if observed_limit < required_budget: + raise RuntimeError("connection_budget_below_required") + if available_connection_headroom < required_budget: + raise RuntimeError("connection_headroom_below_required") if table_gap_count or sequence_gap_count: raise RuntimeError("database_privilege_gap") print(json.dumps({ @@ -375,12 +393,11 @@ spec: str(identity["role_name"]).encode("utf-8") ).hexdigest(), "connection_limit": observed_limit, - "concurrent_connection_probe_count": required_concurrency, - "existing_role_connection_count": ( - existing_role_connection_count - ), - "additional_connection_probe_count": ( - additional_probe_count + "required_connection_budget": required_budget, + "active_role_connection_count": active_role_connection_count, + "available_connection_headroom": available_connection_headroom, + "representative_connection_probe_count": ( + representative_probe_count ), "representative_read_count": 3, "table_privilege_gap_count": table_gap_count, @@ -474,6 +491,7 @@ for workload in "${selected_workloads[@]}"; do secret="$(workload_secret "$workload")" connection_limit="$(workload_connection_limit "$workload")" required_connection_budget="$(workload_required_connection_budget "$workload")" + representative_probe_count="$(workload_representative_probe_count "$workload")" secret_present=false if kube get secret "$secret" -n "$NAMESPACE" >/dev/null 2>&1; then secret_present=true @@ -507,11 +525,11 @@ for workload in "${selected_workloads[@]}"; do else ready=false fi - echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget" + echo "workload=$workload secret_ref=$secret secret_present=$secret_present role_present=$role_present contract_ready=$state_ready connection_limit=$connection_limit required_connection_budget=$required_connection_budget representative_probe_count=$representative_probe_count" if [ "$MODE" = "apply" ] || [ "$MODE" = "verify" ]; then preflight_workload "$workload" "$secret" "$connection_limit" \ - "$required_connection_budget" + "$required_connection_budget" "$representative_probe_count" fi done diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index 7f053bea3..20f8954e1 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -7,6 +7,89 @@ param( $ErrorActionPreference = "Stop" +function Get-AgentLivePreflightDecision { + param( + [bool]$AgentRootPresent, + [object]$Manifest, + [int]$ExpectedRuntimeFileCount, + [int]$TaskFailureCount, + [int]$RelayListenerCount, + [int]$RequiredEvidenceStaleCount, + [int]$RequiredEvidenceParseFailureCount, + [int]$RequiredEvidenceContentFailureCount, + [string]$SelfHealthSeverity, + [int]$SelfHealthPerformanceIssueCount, + [int]$StaleAlertExecutionCount, + [int]$AlertIncomingCount, + [int]$PendingCommandCount + ) + + $blockingReasons = @() + $warningReasons = @() + $manifestExists = [bool]($Manifest -and $Manifest.exists) + $manifestParseReady = [bool]( + $manifestExists -and + -not ([string]$Manifest.parseError) + ) + $manifestIdentityReady = [bool]( + $manifestParseReady -and + [string]$Manifest.sourceRevision -match "^[a-fA-F0-9]{40}$" -and + [int]$Manifest.fileCount -eq $ExpectedRuntimeFileCount + ) + $manifestContentReady = [bool]( + $manifestParseReady -and + $Manifest.runtimeMatched -eq $true -and + [int]$Manifest.mismatchCount -eq 0 + ) + + if (-not $AgentRootPresent) { $blockingReasons += "agent_root_missing" } + if (-not $manifestExists) { + $blockingReasons += "runtime_manifest_missing" + } elseif (-not $manifestParseReady) { + $blockingReasons += "runtime_manifest_parse_failed" + } else { + if (-not $manifestIdentityReady) { + $blockingReasons += "runtime_manifest_identity_invalid" + } + if (-not $manifestContentReady) { + $blockingReasons += "runtime_manifest_mismatch" + } + } + if ($TaskFailureCount -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } + if ($RelayListenerCount -lt 1) { $blockingReasons += "sre_alert_relay_not_listening" } + if ($RequiredEvidenceStaleCount -gt 0) { $blockingReasons += "required_evidence_stale" } + if ($RequiredEvidenceParseFailureCount -gt 0) { $blockingReasons += "required_evidence_parse_failed" } + if ($RequiredEvidenceContentFailureCount -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" } + + $normalizedSelfHealth = ([string]$SelfHealthSeverity).Trim().ToLowerInvariant() + switch ($normalizedSelfHealth) { + "ok" {} + "warning" { + $warningReasons += "self_health_warning" + if ($SelfHealthPerformanceIssueCount -gt 0) { + $warningReasons += "performance_warning" + } + } + "critical" { $blockingReasons += "self_health_critical" } + default { $blockingReasons += "self_health_unclassified" } + } + + if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" } + if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) { + $blockingReasons += "pending_work_before_reboot" + } + + return [pscustomobject]@{ + deploymentEligible = [bool]($blockingReasons.Count -eq 0) + blockingReasons = @($blockingReasons) + warningReasons = @($warningReasons) + warningCount = [int]$warningReasons.Count + manifestIdentityReady = $manifestIdentityReady + manifestContentReady = $manifestContentReady + selfHealthClassification = $normalizedSelfHealth + } +} + function Get-TaskReadback { param([string]$TaskName) @@ -103,6 +186,38 @@ function Get-EvidenceReadback { $publicRows = if ($rawEvidence.PSObject.Properties["public"]) { @($rawEvidence.public) } else { @() } $performanceRows = if ($rawEvidence.PSObject.Properties["performance"]) { @($rawEvidence.performance) } else { @() } $telegramRows = if ($rawEvidence.PSObject.Properties["telegram"]) { @($rawEvidence.telegram) } else { @() } + $selfHealthPerformance = if ( + $rawEvidence.PSObject.Properties["selfHealth"] -and + $rawEvidence.selfHealth -and + $rawEvidence.selfHealth.PSObject.Properties["latestPerformance"] + ) { $rawEvidence.selfHealth.latestPerformance } else { $null } + $selfHealthPerformanceSeverity = if ($selfHealthPerformance) { + ([string]$selfHealthPerformance.severity).Trim().ToLowerInvariant() + } else { "" } + if ($selfHealthPerformanceSeverity -notin @("ok", "warning", "critical")) { + $selfHealthPerformanceSeverity = "unknown" + } + $selfHealthPerformanceRows = if ($selfHealthPerformance) { + @($selfHealthPerformance.hosts | ForEach-Object { + $safeHost = ([string]$_.host).Trim() + if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$") { + $safeHost = "unknown" + } + $safeSeverity = ([string]$_.severity).Trim().ToLowerInvariant() + if ($safeSeverity -notin @("ok", "warning", "critical", "missing")) { + $safeSeverity = "unknown" + } + $safeReasons = @($_.reasons | ForEach-Object { + $reason = ([string]$_).Trim().ToLowerInvariant() + if ($reason -match "^[a-z0-9_]{1,80}$") { $reason } + }) + [pscustomobject]@{ + host = $safeHost + severity = $safeSeverity + reasons = $safeReasons + } + }) + } else { @() } $safeSummary = if ($Name -eq "sre_alert_inbox") { [pscustomobject]@{ ok = if ($rawEvidence.PSObject.Properties["ok"]) { [bool]$rawEvidence.ok } else { $false } @@ -152,6 +267,9 @@ function Get-EvidenceReadback { performanceCount = $performanceRows.Count performanceIssueCount = [int]@($performanceRows | Where-Object { $_.severity -in @("warning", "critical") }).Count selfHealthSeverity = if ($rawEvidence.PSObject.Properties["selfHealth"] -and $rawEvidence.selfHealth) { [string]$rawEvidence.selfHealth.severity } else { "" } + selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity + selfHealthPerformanceIssueCount = [int]@($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" }).Count + selfHealthPerformanceIssues = @($selfHealthPerformanceRows | Where-Object { $_.severity -ne "ok" }) backupHealthSeverity = if ($rawEvidence.PSObject.Properties["backupHealth"] -and $rawEvidence.backupHealth) { [string]$rawEvidence.backupHealth.severity } else { "" } controlProcessedCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.processedCount } else { 0 } controlPendingCount = if ($rawEvidence.PSObject.Properties["controlTick"] -and $rawEvidence.controlTick) { [int]$rawEvidence.controlTick.pendingCount } else { 0 } @@ -292,21 +410,27 @@ $requiredEvidenceParseFailures = @($evidence | Where-Object { $_.required -and $ $requiredEvidenceContentFailures = @($evidence | Where-Object { $_.required -and $_.fresh -and $_.parsed -and -not $_.contentHealthy }) $selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 $selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" } +$selfHealthPerformanceSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthPerformanceSeverity } else { "unknown" } +$selfHealthPerformanceIssueCount = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount } else { 0 } $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 $alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 -$blockingReasons = @() -if (-not (Test-Path $AgentRoot)) { $blockingReasons += "agent_root_missing" } -if (-not $manifest.exists) { $blockingReasons += "runtime_manifest_missing" } -if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blockingReasons += "runtime_manifest_mismatch" } -if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } -if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } -if ($requiredEvidenceStale.Count -gt 0) { $blockingReasons += "required_evidence_stale" } -if ($requiredEvidenceParseFailures.Count -gt 0) { $blockingReasons += "required_evidence_parse_failed" } -if ($requiredEvidenceContentFailures.Count -gt 0) { $blockingReasons += "required_evidence_content_unhealthy" } -if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" } -if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } -if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } +$decision = Get-AgentLivePreflightDecision ` + -AgentRootPresent ([bool](Test-Path $AgentRoot)) ` + -Manifest $manifest ` + -ExpectedRuntimeFileCount 14 ` + -TaskFailureCount $taskFailures.Count ` + -RelayListenerCount $relayListeners.Count ` + -RequiredEvidenceStaleCount $requiredEvidenceStale.Count ` + -RequiredEvidenceParseFailureCount $requiredEvidenceParseFailures.Count ` + -RequiredEvidenceContentFailureCount $requiredEvidenceContentFailures.Count ` + -SelfHealthSeverity $selfHealthSeverity ` + -SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount ` + -StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes ` + -AlertIncomingCount $alertIncoming.count ` + -PendingCommandCount $commandQueue.count +$blockingReasons = @($decision.blockingReasons) +$warningReasons = @($decision.warningReasons) $result = [pscustomobject]@{ schemaVersion = "agent99_live_preflight_v1" @@ -332,16 +456,24 @@ $result = [pscustomobject]@{ requiredEvidenceParseFailureCount = $requiredEvidenceParseFailures.Count requiredEvidenceContentFailureCount = $requiredEvidenceContentFailures.Count selfHealthSeverity = $selfHealthSeverity + selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity + selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount alertIncomingCount = [int]$alertIncoming.count alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count relayListenerCount = $relayListeners.Count - preflightGreen = [bool]($blockingReasons.Count -eq 0) + preflightGreen = [bool]$decision.deploymentEligible + deploymentEligible = [bool]$decision.deploymentEligible + manifestIdentityReady = [bool]$decision.manifestIdentityReady + manifestContentReady = [bool]$decision.manifestContentReady blockingReasons = $blockingReasons + warningCount = [int]$decision.warningCount + warningReasons = $warningReasons } } Write-Output "AGENT99_LIVE_PREFLIGHT=$([int]$result.summary.preflightGreen)" +Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)" Write-Output "MODE=$Mode" Write-Output "SOURCE_REVISION=$($manifest.sourceRevision)" Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)" @@ -351,13 +483,14 @@ Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)" Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" +Write-Output "WARNING_REASONS=$($warningReasons -join ',')" Write-Output "SECRET_VALUE_READ=0" Write-Output "REMOTE_WRITE_PERFORMED=0" Write-Output "JSON_BEGIN" $result | ConvertTo-Json -Depth 8 -Compress Write-Output "JSON_END" -if ($result.summary.preflightGreen) { +if ($result.summary.deploymentEligible) { exit 0 } exit 2 diff --git a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 index 1508e9fdc..f73a18702 100644 --- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 +++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1 @@ -33,6 +33,21 @@ $FixedRuntimeFiles = @( "agent99.config.99.example.json" ) $script:DeployLock = $null +$script:ReceiverRemoteWritePerformed = $false +$script:ReceiverLiveRuntimeWritePerformed = $false +$script:ReceiverLivePromotionAttempted = $false +$script:ReceiverLivePromotionPerformed = $false +$script:ReceiverScheduledTaskRestartPerformed = $false +$script:ReceiverRollbackAttempted = $false +$script:ReceiverRollbackVerified = $false +$script:ReceiverOperationPhase = "envelope_validation" +$script:ReceiverAttemptReceiptPath = "" +$script:ReceiverCanonicalReceiptPath = "" +$script:ReceiverTraceId = "" +$script:ReceiverRunId = "" +$script:ReceiverWorkItemId = "" +$script:ReceiverSourceRevision = "" +$script:ReceiverManifestDigest = "" function Get-AgentSha256Bytes { param([byte[]]$Bytes) @@ -503,9 +518,26 @@ function Restore-AgentBundle { function Write-AgentRemoteDeployReceipt { param([string]$Path, [object]$Receipt) - $temporary = "$Path.tmp.$PID" - $Receipt | ConvertTo-Json -Depth 12 | Set-Content -LiteralPath $temporary -Encoding UTF8 - Move-Item -LiteralPath $temporary -Destination $Path -Force + if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" } + $temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))" + try { + $json = $Receipt | ConvertTo-Json -Depth 12 + $null = $json | ConvertFrom-Json + $utf8NoBom = New-Object System.Text.UTF8Encoding($false) + $script:ReceiverRemoteWritePerformed = $true + [IO.File]::WriteAllText($temporary, $json, $utf8NoBom) + $storedJson = [IO.File]::ReadAllText($temporary, [Text.Encoding]::UTF8) + $null = $storedJson | ConvertFrom-Json + if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" } + # File.Move is the final operation: on Windows PowerShell/.NET it is atomic + # within this volume and refuses to replace an existing destination. + [IO.File]::Move($temporary, $Path) + } catch { + if (Test-Path -LiteralPath $temporary) { + Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue + } + throw + } } function Write-AgentResultAndExit { @@ -576,6 +608,19 @@ try { $stageToken = (Get-AgentSha256Bytes ([Text.Encoding]::UTF8.GetBytes($stageIdentity))).Substring(0, 20) $stagePath = Join-Path $DeployRoot "remote-source-$stageToken" $receiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken.json" + $attemptStamp = [DateTime]::UtcNow.ToString( + "yyyyMMdd'T'HHmmssfffffff'Z'", + [Globalization.CultureInfo]::InvariantCulture + ) + $attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString('N'))" + $attemptReceiptPath = Join-Path $EvidenceDir "agent99-remote-deploy-$stageToken-attempt-$attemptToken.json" + $script:ReceiverAttemptReceiptPath = $attemptReceiptPath + $script:ReceiverCanonicalReceiptPath = $receiptPath + $script:ReceiverTraceId = $traceId + $script:ReceiverRunId = $runId + $script:ReceiverWorkItemId = $workItemId + $script:ReceiverSourceRevision = $sourceRevision + $script:ReceiverManifestDigest = $manifestDigest $currentManifest = Get-AgentSafeRuntimeManifest if ($mode -eq "check") { @@ -591,6 +636,9 @@ try { livePreflight = [pscustomobject]@{ executed = $false; reason = "apply_only_after_validate_only" } operationBoundaries = [pscustomobject]@{ remoteWritePerformed = $false + liveRuntimeWritePerformed = $false + livePromotionAttempted = $false + livePromotionPerformed = $false secretValueRead = $false privateKeyValueRead = $false tokenValueRead = $false @@ -606,6 +654,8 @@ try { }) 0 } + $script:ReceiverOperationPhase = "apply_state_initialization" + $script:ReceiverRemoteWritePerformed = $true New-Item -ItemType Directory -Force -Path $StateDir, $EvidenceDir, $DeployRoot | Out-Null $lockPath = Join-Path $StateDir "remote-atomic-deploy.lock" try { @@ -619,18 +669,29 @@ try { runId = $runId workItemId = $workItemId sourceRevision = $sourceRevision - remoteWritePerformed = $false + operationPhase = $script:ReceiverOperationPhase + remoteWritePerformed = $script:ReceiverRemoteWritePerformed liveRuntimeWritePerformed = $false + livePromotionAttempted = $false livePromotionPerformed = $false }) 75 } + $canonicalReceiptExists = $false $prior = $null + $priorParseErrorType = "" if (Test-Path -LiteralPath $receiptPath -PathType Leaf) { - try { $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json } catch { $prior = $null } + $canonicalReceiptExists = $true + try { + $prior = Get-Content -LiteralPath $receiptPath -Raw | ConvertFrom-Json + if (-not $prior) { $priorParseErrorType = "NullCanonicalReceipt" } + } catch { + $prior = $null + $priorParseErrorType = $_.Exception.GetType().Name + } } - $stageWrittenThisRun = $false + $script:ReceiverOperationPhase = "staging" if (-not (Test-Path -LiteralPath $stagePath -PathType Container)) { $stageBuildPath = "$stagePath.partial.$PID.$([Guid]::NewGuid().ToString('N'))" New-Item -ItemType Directory -Path $stageBuildPath | Out-Null @@ -640,7 +701,7 @@ try { [IO.File]::WriteAllBytes((Join-Path $stageBuildPath "agent99-live-preflight.ps1"), $preflightBytes) $sourceManifest | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath (Join-Path $stageBuildPath "source-manifest.json") -Encoding UTF8 Move-Item -LiteralPath $stageBuildPath -Destination $stagePath - $stageWrittenThisRun = $true + $script:ReceiverRemoteWritePerformed = $true } else { foreach ($name in $FixedRuntimeFiles) { if ((Get-AgentSha256File (Join-Path $stagePath $name)) -ne ([string]($filesByName[$name].sha256)).ToLowerInvariant()) { @@ -678,6 +739,7 @@ try { $live.fileCount -eq 14 -and $live.mismatchCount -eq 0 ) { + $script:ReceiverOperationPhase = "idempotent_replay_post_verifier" $replayPreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") if ($replayPreflight.ok -and $replayPreflight.sourceRevision -eq $sourceRevision -and $replayPreflight.runtimeMatched) { $originalOperationBoundaries = $prior.operationBoundaries @@ -686,8 +748,9 @@ try { $prior | Add-Member -NotePropertyName livePreflight -NotePropertyValue $replayPreflight -Force $prior | Add-Member -NotePropertyName originalOperationBoundaries -NotePropertyValue $originalOperationBoundaries -Force $prior | Add-Member -NotePropertyName operationBoundaries -NotePropertyValue ([pscustomobject]@{ - remoteWritePerformed = $stageWrittenThisRun + remoteWritePerformed = $script:ReceiverRemoteWritePerformed liveRuntimeWritePerformed = $false + livePromotionAttempted = $false livePromotionPerformed = $false secretValueRead = $false privateKeyValueRead = $false @@ -702,7 +765,7 @@ try { }) -Force Write-AgentResultAndExit $prior 0 } - Write-AgentResultAndExit ([pscustomobject]@{ + $receipt = [pscustomobject]@{ schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" status = "idempotent_replay_post_verifier_failed_no_promotion" mode = "apply" @@ -712,15 +775,66 @@ try { sourceRevision = $sourceRevision manifestSha256 = $manifestDigest stagingPath = $stagePath + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath runtimeManifest = $live livePreflight = $replayPreflight - remoteWritePerformed = $stageWrittenThisRun + durableReceiptWritten = $true + remoteWritePerformed = $script:ReceiverRemoteWritePerformed liveRuntimeWritePerformed = $false + livePromotionAttempted = $false livePromotionPerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false nextSafeAction = "repair_reported_live_preflight_blocker_then_retry_same_identity" - }) 2 + } + $script:ReceiverOperationPhase = "idempotent_replay_failure_receipt_write" + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt + Write-AgentResultAndExit $receipt 2 } + if ($canonicalReceiptExists) { + $canonicalStatus = if (-not $prior) { + "blocked_canonical_receipt_invalid_no_promotion" + } elseif ([string]$prior.status -eq "deployed_verified") { + "blocked_canonical_success_receipt_runtime_drift_no_promotion" + } else { + "blocked_canonical_receipt_reserved_no_promotion" + } + $receipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = $canonicalStatus + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath + canonicalReceiptStatus = if ($prior) { [string]$prior.status } else { "unparseable" } + canonicalReceiptParseErrorType = $priorParseErrorType + runtimeManifest = $live + durableReceiptWritten = $true + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $false + livePromotionAttempted = $false + livePromotionPerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_trace_run_work_item_identity" + } + $script:ReceiverOperationPhase = "canonical_receipt_reserved_failure_receipt_write" + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt + Write-AgentResultAndExit $receipt 2 + } + + $script:ReceiverOperationPhase = "validate_only" $validate = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision -ValidateOnly if (-not $validate.ok -or -not $validate.payload -or [string]$validate.payload.status -ne "validated") { $receipt = [pscustomobject]@{ @@ -733,12 +847,22 @@ try { sourceRevision = $sourceRevision manifestSha256 = $manifestDigest stagingPath = $stagePath + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } - remoteWritePerformed = $true + durableReceiptWritten = $true + remoteWritePerformed = $script:ReceiverRemoteWritePerformed liveRuntimeWritePerformed = $false + livePromotionAttempted = $false livePromotionPerformed = $false + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + nextSafeAction = "repair_validate_only_blocker_then_retry_same_identity" } - Write-AgentRemoteDeployReceipt $receiptPath $receipt + $script:ReceiverOperationPhase = "validate_only_failure_receipt_write" + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt Write-AgentResultAndExit $receipt 2 } @@ -773,10 +897,14 @@ try { sourceRevision = $sourceRevision manifestSha256 = $manifestDigest stagingPath = $stagePath + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut } relayRuntime = $beforeRelayRuntime - remoteWritePerformed = $true + durableReceiptWritten = $true + remoteWritePerformed = $script:ReceiverRemoteWritePerformed liveRuntimeWritePerformed = $false + livePromotionAttempted = $false livePromotionPerformed = $false secretValueRead = $false privateKeyValueRead = $false @@ -790,7 +918,8 @@ try { scheduledTaskModification = $false nextSafeAction = "repair_exact_existing_relay_task_runtime_then_retry_same_identity" } - Write-AgentRemoteDeployReceipt $receiptPath $receipt + $script:ReceiverOperationPhase = "relay_precondition_failure_receipt_write" + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt Write-AgentResultAndExit $receipt 2 } @@ -803,6 +932,9 @@ try { $failureType = "" $backupPath = "" try { + $script:ReceiverOperationPhase = "live_deploy" + $script:ReceiverLiveRuntimeWritePerformed = $true + $script:ReceiverLivePromotionAttempted = $true $deploy = Invoke-AgentDeployChild -SourceRoot $stagePath -SourceRevision $sourceRevision if ($deploy.payload -and $deploy.payload.PSObject.Properties["backupDir"]) { $backupPath = [string]$deploy.payload.backupDir @@ -811,11 +943,15 @@ try { $failurePhase = "deploy" $failureType = "deploy_child_failed_or_invalid" } else { + $script:ReceiverLivePromotionPerformed = $true + $script:ReceiverOperationPhase = "relay_reload" + $script:ReceiverScheduledTaskRestartPerformed = $true $relayReload = Invoke-AgentRelayTaskRestart -ForbiddenGenerations @($beforeRelayRuntime.listenerGenerations) if (-not $relayReload.ok) { $failurePhase = "relay_reload" $failureType = "relay_reload_failed" } else { + $script:ReceiverOperationPhase = "independent_post_verifier" $runtimeManifest = Get-AgentSafeRuntimeManifest $livePreflight = Invoke-AgentLivePreflight (Join-Path $stagePath "agent99-live-preflight.ps1") $postVerified = [bool]( @@ -845,6 +981,55 @@ try { $failureType = $_.Exception.GetType().Name } + $successReceipt = $null + if (-not $failurePhase) { + try { + $script:ReceiverOperationPhase = "success_receipt_construction" + $successReceipt = [pscustomobject]@{ + schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" + status = "deployed_verified" + mode = "apply" + traceId = $traceId + runId = $runId + workItemId = $workItemId + sourceRevision = $sourceRevision + manifestSha256 = $manifestDigest + stagingPath = $stagePath + backupPath = $backupPath + canonicalReceiptPath = $receiptPath + validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status } + deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } + relayReload = $relayReload + runtimeManifest = $runtimeManifest + livePreflight = $livePreflight + durableReceiptWritten = $true + idempotentReplay = $false + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $true + liveRuntimeWritePerformed = $true + livePromotionAttempted = $true + livePromotionPerformed = $true + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskRestart = $true + scheduledTaskModification = $false + } + nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback" + } + $script:ReceiverOperationPhase = "success_receipt_write" + Write-AgentRemoteDeployReceipt $receiptPath $successReceipt + } catch { + $failurePhase = $script:ReceiverOperationPhase + $failureType = $_.Exception.GetType().Name + } + } + if ($failurePhase) { if (-not $backupPath) { $newBackup = Get-ChildItem -LiteralPath $DeployRoot -Directory -Filter "backup-*" -ErrorAction SilentlyContinue | @@ -853,6 +1038,8 @@ try { Select-Object -First 1 if ($newBackup) { $backupPath = $newBackup.FullName } } + $script:ReceiverOperationPhase = "runtime_rollback" + $script:ReceiverRollbackAttempted = $true $rollback = Restore-AgentBundle $backupPath $beforeFilePresence $beforeConfigPresent $beforeManifestPresent $rollbackForbiddenGenerations = @($beforeRelayRuntime.listenerGenerations) if ($relayReload) { @@ -902,6 +1089,7 @@ try { $rollbackRelayRestart.after.taskRunning -and $rollbackRelayRestart.after.listenerCount -gt 0 ) + $script:ReceiverRollbackVerified = $rollbackVerified } catch { $rollbackVerified = $false $rollbackVerifierErrorType = $_.Exception.GetType().Name @@ -911,6 +1099,8 @@ try { schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" status = if ($failurePhase -eq "deploy") { if ($rollbackVerified) { "deploy_failed_rollback_verified" } else { "deploy_failed_rollback_unverified" } + } elseif ($failurePhase -in @("success_receipt_construction", "success_receipt_write")) { + if ($rollbackVerified) { "rolled_back_success_receipt_write_failed" } else { "rollback_failed_after_success_receipt_write_failure" } } else { if ($rollbackVerified) { "rolled_back_post_verifier_failed" } else { "rollback_failed_after_post_verifier" } } @@ -926,6 +1116,8 @@ try { deploy = if ($deploy) { [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut } } else { [pscustomobject]@{ ok = $false; exitCode = -1; timedOut = $false } } failurePhase = $failurePhase failureType = $failureType + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath relayReload = $relayReload failedRuntimeManifest = $runtimeManifest failedLivePreflight = $livePreflight @@ -935,6 +1127,24 @@ try { rollbackVerifierErrorType = $rollbackVerifierErrorType runtimeManifest = $afterManifest durableReceiptWritten = $true + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed + livePromotionAttempted = $script:ReceiverLivePromotionAttempted + livePromotionPerformed = $script:ReceiverLivePromotionPerformed + rollbackAttempted = $script:ReceiverRollbackAttempted + rollbackVerified = $rollbackVerified + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskRestart = [bool](($relayReload -and $relayReload.attempted) -or $rollbackRelayRestart.attempted) + scheduledTaskModification = $false + } secretValueRead = $false privateKeyValueRead = $false tokenValueRead = $false @@ -957,10 +1167,22 @@ try { sourceRevision = $sourceRevision failurePhase = $failurePhase failureType = $failureType + canonicalReceiptPath = $receiptPath + attemptReceiptPath = $attemptReceiptPath rollbackVerified = $false runtimeRollbackWasVerifiedBeforeReceiptFailure = $rollbackVerified receiptConstructionErrorType = $_.Exception.GetType().Name durableReceiptWritten = $true + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed + livePromotionAttempted = $script:ReceiverLivePromotionAttempted + livePromotionPerformed = $script:ReceiverLivePromotionPerformed + rollbackAttempted = $script:ReceiverRollbackAttempted + rollbackVerified = $false + scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed + scheduledTaskModification = $false + } secretValueRead = $false environmentSecretRead = $false scheduledTaskModification = $false @@ -968,7 +1190,7 @@ try { $rollbackVerified = $false } try { - Write-AgentRemoteDeployReceipt $receiptPath $receipt + Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt } catch { $receipt.status = "rollback_receipt_write_failed" $receipt.durableReceiptWritten = $false @@ -978,45 +1200,29 @@ try { Write-AgentResultAndExit $receipt $(if ($rollbackVerified) { 2 } else { 3 }) } - $receipt = [pscustomobject]@{ - schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" - status = "deployed_verified" - mode = "apply" - traceId = $traceId - runId = $runId - workItemId = $workItemId - sourceRevision = $sourceRevision - manifestSha256 = $manifestDigest - stagingPath = $stagePath - backupPath = $backupPath - validateOnly = [pscustomobject]@{ ok = $validate.ok; exitCode = $validate.exitCode; timedOut = $validate.timedOut; status = [string]$validate.payload.status } - deploy = [pscustomobject]@{ ok = $deploy.ok; exitCode = $deploy.exitCode; timedOut = $deploy.timedOut; status = [string]$deploy.payload.status } - relayReload = $relayReload - runtimeManifest = $runtimeManifest - livePreflight = $livePreflight - idempotentReplay = $false - operationBoundaries = [pscustomobject]@{ - remoteWritePerformed = $true - secretValueRead = $false - privateKeyValueRead = $false - tokenValueRead = $false - environmentSecretRead = $false - uiInteraction = $false - vmPowerChange = $false - hostReboot = $false - serviceRestart = $false - scheduledTaskRestart = $true - scheduledTaskModification = $false - } - nextSafeAction = "run_existing_production_principal_verifier_and_same_trace_completion_readback" - } - Write-AgentRemoteDeployReceipt $receiptPath $receipt - Write-AgentResultAndExit $receipt 0 + $script:ReceiverOperationPhase = "success_result_output" + Write-AgentResultAndExit $successReceipt 0 } catch { - Write-AgentResultAndExit ([pscustomobject]@{ + $outerErrorType = $_.Exception.GetType().Name + $outerFailure = [pscustomobject]@{ schemaVersion = "agent99_remote_atomic_deploy_receipt_v1" status = "transport_receiver_failed" - errorType = $_.Exception.GetType().Name + errorType = $outerErrorType + operationPhase = $script:ReceiverOperationPhase + traceId = $script:ReceiverTraceId + runId = $script:ReceiverRunId + workItemId = $script:ReceiverWorkItemId + sourceRevision = $script:ReceiverSourceRevision + manifestSha256 = $script:ReceiverManifestDigest + canonicalReceiptPath = if ($script:ReceiverCanonicalReceiptPath) { $script:ReceiverCanonicalReceiptPath } else { "unavailable" } + attemptReceiptPath = if ($script:ReceiverAttemptReceiptPath) { $script:ReceiverAttemptReceiptPath } else { "unavailable" } + durableReceiptWritten = $false + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed + livePromotionAttempted = $script:ReceiverLivePromotionAttempted + livePromotionPerformed = $script:ReceiverLivePromotionPerformed + rollbackAttempted = $script:ReceiverRollbackAttempted + rollbackVerified = $script:ReceiverRollbackVerified secretValueRead = $false privateKeyValueRead = $false tokenValueRead = $false @@ -1026,9 +1232,40 @@ try { vmPowerChange = $false hostReboot = $false serviceRestart = $false - scheduledTaskRestart = $false + scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed scheduledTaskModification = $false - }) 70 + operationBoundaries = [pscustomobject]@{ + remoteWritePerformed = $script:ReceiverRemoteWritePerformed + liveRuntimeWritePerformed = $script:ReceiverLiveRuntimeWritePerformed + livePromotionAttempted = $script:ReceiverLivePromotionAttempted + livePromotionPerformed = $script:ReceiverLivePromotionPerformed + rollbackAttempted = $script:ReceiverRollbackAttempted + rollbackVerified = $script:ReceiverRollbackVerified + secretValueRead = $false + privateKeyValueRead = $false + tokenValueRead = $false + environmentSecretRead = $false + uiInteraction = $false + vmPowerChange = $false + hostReboot = $false + serviceRestart = $false + scheduledTaskRestart = $script:ReceiverScheduledTaskRestartPerformed + scheduledTaskModification = $false + } + } + if ($script:ReceiverAttemptReceiptPath) { + try { + $script:ReceiverRemoteWritePerformed = $true + $outerFailure.remoteWritePerformed = $true + $outerFailure.operationBoundaries.remoteWritePerformed = $true + $outerFailure.durableReceiptWritten = $true + Write-AgentRemoteDeployReceipt $script:ReceiverAttemptReceiptPath $outerFailure + } catch { + $outerFailure.durableReceiptWritten = $false + $outerFailure | Add-Member -NotePropertyName receiptWriteErrorType -NotePropertyValue $_.Exception.GetType().Name -Force + } + } + Write-AgentResultAndExit $outerFailure 70 } finally { if ($script:DeployLock) { try { $script:DeployLock.Dispose() } catch {} diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh index 387adb9ca..897744eb0 100755 --- a/scripts/reboot-recovery/host112-guest-readiness.sh +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -1,11 +1,10 @@ #!/usr/bin/env bash # Host 112 guest readiness and bounded recovery executor. -# Version: 2.0 -# Last modified: 2026-07-14 11:00 (Asia/Taipei) +# Version: 2.1 +# Last modified: 2026-07-14 13:15 (Asia/Taipei) # Last modified by: Codex -# Change: add correlated, cooldown-guarded Wazuh manager recovery and an -# independent process/port verifier without reboot, VM power, firewall, or -# secret access. +# Change: guard every runtime mutation with a fixed durable artifact WAL and +# require independently verified receipt/readback persistence before success. set -uo pipefail @@ -174,6 +173,18 @@ signal_readback_status="not_requested" signal_readback_path="none" signal_readback_sha256="none" action_failure_observed=0 +ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt" +artifact_transaction_status="not_requested" +artifact_transaction_phase="not_started" +artifact_transaction_started=0 +artifact_transaction_blocked=0 +artifact_transaction_write_performed=0 +artifact_transaction_pair_verified=0 +artifact_transaction_prior_pair_verified=0 +artifact_expected_receipt_path="none" +artifact_expected_readback_path="none" +artifact_chain_failed=0 +artifact_receipt_ready=0 is_uint() { [[ "$1" =~ ^[0-9]+$ ]] @@ -274,6 +285,12 @@ start_if_inactive() { if [ "$state" = "active" ] || [ "$state" = "activating" ]; then return 0 fi + if ! authorize_runtime_mutation "start_${unit//[^A-Za-z0-9._-]/_}"; then + record_action "${action}_artifact_transaction_blocked_failed" + # Keep the executor on its fail-closed evidence path. The recorded + # failure drives a degraded terminal without allowing this mutation. + return 0 + fi # A mutating systemctl invocation may partially change runtime state before # returning non-zero or timing out. Record the attempt before execution so # those paths can never be classified as no-write timer observations. @@ -291,6 +308,10 @@ enable_if_disabled() { if [ "$state" = "enabled" ] || [ "$state" = "static" ]; then return 0 fi + if ! authorize_runtime_mutation "enable_${unit//[^A-Za-z0-9._-]/_}"; then + record_action "${action}_artifact_transaction_blocked_failed" + return 0 + fi runtime_write_performed=1 state_write_performed=1 if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl enable "$unit" >/dev/null; then @@ -507,8 +528,268 @@ timer_slot_for_identity() { printf '%03d' "$((checksum % timer_ledger_capacity))" } +reset_durable_receipt_verdict() { + durable_receipt_present=0 + durable_receipt_identity_match=0 + durable_receipt_boot_match=0 + durable_receipt_terminal="none" +} + +reset_durable_readback_verdict() { + durable_readback_present=0 + durable_readback_identity_match=0 + durable_readback_boot_match=0 + durable_readback_receipt_link_match=0 + durable_readback_terminal_match=0 +} + +artifact_paths_for_current_identity() { + local receipt_name timer_slot + if [ "$MODE" = "timer_apply" ]; then + timer_slot="$(timer_slot_for_identity)" || return 1 + timer_ledger_slot="$timer_slot" + receipt_name="timer-slot-${timer_slot}" + artifact_expected_receipt_path="$STATE_DIR/timer-ledger/${receipt_name}.receipt.txt" + artifact_expected_readback_path="$STATE_DIR/timer-ledger/${receipt_name}.readback.txt" + else + receipt_name="$(receipt_name_for_identity)" + artifact_expected_receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" + artifact_expected_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" + fi + case "$artifact_expected_receipt_path" in *[[:space:]]*) return 1 ;; esac + case "$artifact_expected_readback_path" in *[[:space:]]*) return 1 ;; esac +} + +controlled_artifact_paths() { + local candidate_receipt="$1" candidate_readback="$2" + case "$candidate_receipt" in + "$STATE_DIR"/receipts/*.txt|"$STATE_DIR"/timer-ledger/*.receipt.txt) ;; + *) return 1 ;; + esac + case "$candidate_readback" in + "$STATE_DIR"/readbacks/*.txt|"$STATE_DIR"/timer-ledger/*.readback.txt) ;; + *) return 1 ;; + esac +} + +verify_artifact_pair_paths() { + local expected_trace="$1" expected_run="$2" expected_work_item="$3" + local expected_boot="$4" candidate_receipt="$5" candidate_readback="$6" + local receipt_schema receipt_trace receipt_run receipt_work_item receipt_boot receipt_terminal receipt_digest + local readback_schema + local readback_trace readback_run readback_work_item readback_boot + local readback_receipt_path readback_receipt_sha readback_terminal + controlled_artifact_paths "$candidate_receipt" "$candidate_readback" || return 1 + [ -f "$candidate_receipt" ] && [ -f "$candidate_readback" ] || return 1 + receipt_schema="$(receipt_token "$candidate_receipt" schema_version)" + receipt_trace="$(receipt_token "$candidate_receipt" trace_id)" + receipt_run="$(receipt_token "$candidate_receipt" run_id)" + receipt_work_item="$(receipt_token "$candidate_receipt" work_item_id)" + receipt_boot="$(receipt_token "$candidate_receipt" boot_id)" + receipt_terminal="$(receipt_token "$candidate_receipt" terminal)" + [ "$receipt_schema" = "host112_wazuh_manager_recovery_receipt_v2" ] \ + && [ -n "$receipt_terminal" ] && [ "$receipt_terminal" != "none" ] \ + && [ "$receipt_trace" = "$expected_trace" ] \ + && [ "$receipt_run" = "$expected_run" ] \ + && [ "$receipt_work_item" = "$expected_work_item" ] \ + && [ "$receipt_boot" = "$expected_boot" ] || return 1 + receipt_digest="$(sha256sum "$candidate_receipt" | awk '{print $1}')" + [ -n "$receipt_digest" ] || return 1 + readback_schema="$(receipt_token "$candidate_readback" schema_version)" + readback_trace="$(receipt_token "$candidate_readback" trace_id)" + readback_run="$(receipt_token "$candidate_readback" run_id)" + readback_work_item="$(receipt_token "$candidate_readback" work_item_id)" + readback_boot="$(receipt_token "$candidate_readback" boot_id)" + readback_receipt_path="$(receipt_token "$candidate_readback" receipt_path)" + readback_receipt_sha="$(receipt_token "$candidate_readback" receipt_sha256)" + readback_terminal="$(receipt_token "$candidate_readback" durable_receipt_terminal)" + [ "$readback_schema" = "host112_guest_recovery_v2" ] \ + && [ "$readback_trace" = "$expected_trace" ] \ + && [ "$readback_run" = "$expected_run" ] \ + && [ "$readback_work_item" = "$expected_work_item" ] \ + && [ "$readback_boot" = "$expected_boot" ] \ + && [ "$readback_receipt_path" = "$candidate_receipt" ] \ + && [ "$readback_receipt_sha" = "$receipt_digest" ] \ + && [ "$readback_terminal" = "$receipt_terminal" ] +} + +write_artifact_transaction_record() { + local requested_status="$1" requested_phase="$2" pair_verified="$3" + local tmp stored_schema stored_status stored_phase stored_trace stored_run + local stored_work_item stored_boot stored_receipt stored_readback stored_pair + artifact_paths_for_current_identity || return 1 + install -d -m 0750 "$STATE_DIR" || return 1 + tmp="${ARTIFACT_TRANSACTION_PATH}.tmp.$$" + if ! printf '%s\n' \ + "schema_version=host112_active_artifact_transaction_v1 updated_at=$(date --iso-8601=seconds) status=$requested_status phase=$requested_phase trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID boot_id=$boot_id receipt_path=$artifact_expected_receipt_path readback_path=$artifact_expected_readback_path pair_verified=$pair_verified" \ + >"$tmp" \ + || ! chmod 0640 "$tmp" \ + || ! mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"; then + rm -f "$tmp" >/dev/null 2>&1 || true + return 1 + fi + stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)" + stored_status="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" status)" + stored_phase="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" phase)" + stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)" + stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)" + stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)" + stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)" + stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)" + stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)" + stored_pair="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" pair_verified)" + if [ "$stored_schema" != "host112_active_artifact_transaction_v1" ] \ + || [ "$stored_status" != "$requested_status" ] \ + || [ "$stored_phase" != "$requested_phase" ] \ + || [ "$stored_trace" != "$TRACE_ID" ] \ + || [ "$stored_run" != "$RUN_ID" ] \ + || [ "$stored_work_item" != "$WORK_ITEM_ID" ] \ + || [ "$stored_boot" != "$boot_id" ] \ + || [ "$stored_receipt" != "$artifact_expected_receipt_path" ] \ + || [ "$stored_readback" != "$artifact_expected_readback_path" ] \ + || [ "$stored_pair" != "$pair_verified" ]; then + return 1 + fi + artifact_transaction_status="$requested_status" + artifact_transaction_phase="$requested_phase" + artifact_transaction_write_performed=1 + state_write_performed=1 + return 0 +} + +inspect_active_artifact_transaction() { + local stored_schema stored_trace stored_run stored_work_item stored_boot + local stored_receipt stored_readback + [ -f "$ARTIFACT_TRANSACTION_PATH" ] || { + artifact_transaction_status="none" + artifact_transaction_phase="no_prior_transaction" + return 0 + } + stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)" + stored_trace="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" trace_id)" + stored_run="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" run_id)" + stored_work_item="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" work_item_id)" + stored_boot="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" boot_id)" + stored_receipt="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" receipt_path)" + stored_readback="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" readback_path)" + if [ "$stored_schema" = "host112_active_artifact_transaction_v1" ] \ + && safe_identity "$stored_trace" \ + && safe_identity "$stored_run" \ + && safe_identity "$stored_work_item" \ + && [ -n "$stored_boot" ] \ + && verify_artifact_pair_paths "$stored_trace" "$stored_run" "$stored_work_item" \ + "$stored_boot" "$stored_receipt" "$stored_readback"; then + artifact_transaction_prior_pair_verified=1 + artifact_transaction_status="prior_pair_verified" + artifact_transaction_phase="prior_transaction_resolved" + return 0 + fi + artifact_transaction_blocked=1 + artifact_transaction_status="blocked_unresolved_prior_transaction" + artifact_transaction_phase="mutation_denied" + return 1 +} + +authorize_runtime_mutation() { + local phase="${1:-runtime_mutation}" + [ "$artifact_transaction_started" -eq 0 ] || return 0 + [ "$artifact_transaction_blocked" -eq 0 ] || return 1 + if write_artifact_transaction_record "pending_artifact_commit" "$phase" 0; then + artifact_transaction_started=1 + return 0 + fi + artifact_transaction_blocked=1 + artifact_transaction_status="wal_write_or_readback_failed" + artifact_transaction_phase="mutation_denied" + return 1 +} + +mark_artifact_transaction_partial() { + local phase="$1" + [ "$artifact_transaction_started" -eq 1 ] || return 0 + if ! write_artifact_transaction_record "partial_pending_public_artifact_commit" "$phase" 0; then + artifact_transaction_status="pending_artifact_commit_partial_update_failed" + artifact_transaction_phase="$phase" + fi +} + +verify_current_receipt_from_disk() { + local expected_terminal="${1:-}" stored_trace stored_run stored_work_item stored_boot stored_terminal + reset_durable_receipt_verdict + [ -f "$receipt_path" ] || return 1 + durable_receipt_present=1 + stored_trace="$(receipt_token "$receipt_path" trace_id)" + stored_run="$(receipt_token "$receipt_path" run_id)" + stored_work_item="$(receipt_token "$receipt_path" work_item_id)" + stored_boot="$(receipt_token "$receipt_path" boot_id)" + stored_terminal="$(receipt_token "$receipt_path" terminal)" + receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" + durable_receipt_terminal="$stored_terminal" + if [ "$stored_trace" = "$TRACE_ID" ] \ + && [ "$stored_run" = "$RUN_ID" ] \ + && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then + durable_receipt_identity_match=1 + fi + [ "$stored_boot" = "$boot_id" ] && durable_receipt_boot_match=1 + [ "$durable_receipt_identity_match" -eq 1 ] \ + && [ "$durable_receipt_boot_match" -eq 1 ] \ + && [ -n "$stored_terminal" ] \ + && { [ -z "$expected_terminal" ] || [ "$stored_terminal" = "$expected_terminal" ]; } +} + +verify_current_readback_from_disk() { + local stored_trace stored_run stored_work_item stored_boot + local stored_receipt_path stored_receipt_sha stored_terminal + reset_durable_readback_verdict + [ -f "$durable_readback_path" ] || return 1 + durable_readback_present=1 + stored_trace="$(receipt_token "$durable_readback_path" trace_id)" + stored_run="$(receipt_token "$durable_readback_path" run_id)" + stored_work_item="$(receipt_token "$durable_readback_path" work_item_id)" + stored_boot="$(receipt_token "$durable_readback_path" boot_id)" + stored_receipt_path="$(receipt_token "$durable_readback_path" receipt_path)" + stored_receipt_sha="$(receipt_token "$durable_readback_path" receipt_sha256)" + stored_terminal="$(receipt_token "$durable_readback_path" durable_receipt_terminal)" + durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + if [ "$stored_trace" = "$TRACE_ID" ] \ + && [ "$stored_run" = "$RUN_ID" ] \ + && [ "$stored_work_item" = "$WORK_ITEM_ID" ]; then + durable_readback_identity_match=1 + fi + [ "$stored_boot" = "$boot_id" ] && durable_readback_boot_match=1 + if [ "$stored_receipt_path" = "$receipt_path" ] \ + && [ "$stored_receipt_sha" = "$receipt_sha256" ]; then + durable_readback_receipt_link_match=1 + fi + [ "$stored_terminal" = "$durable_receipt_terminal" ] \ + && durable_readback_terminal_match=1 + [ "$durable_readback_identity_match" -eq 1 ] \ + && [ "$durable_readback_boot_match" -eq 1 ] \ + && [ "$durable_readback_receipt_link_match" -eq 1 ] \ + && [ "$durable_readback_terminal_match" -eq 1 ] +} + +commit_current_artifact_transaction() { + [ "$artifact_transaction_blocked" -eq 0 ] || return 1 + artifact_paths_for_current_identity || return 1 + if ! verify_artifact_pair_paths "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$boot_id" \ + "$artifact_expected_receipt_path" "$artifact_expected_readback_path"; then + mark_artifact_transaction_partial "independent_pair_verifier_failed" + return 1 + fi + artifact_transaction_pair_verified=1 + if [ "$artifact_transaction_started" -eq 1 ]; then + write_artifact_transaction_record "committed_verified_pair" \ + "independent_pair_verifier_passed" 1 || return 1 + else + artifact_transaction_status="existing_pair_verified" + artifact_transaction_phase="independent_pair_verifier_passed" + fi +} + load_durable_receipt() { local receipt_name stored_trace stored_run stored_work_item stored_boot + reset_durable_receipt_verdict receipt_name="$(receipt_name_for_identity)" receipt_path="$STATE_DIR/receipts/${receipt_name}.txt" [ -f "$receipt_path" ] || return 1 @@ -539,6 +820,7 @@ load_durable_receipt() { load_durable_readback() { local receipt_name stored_trace stored_run stored_work_item stored_boot local stored_receipt_path stored_receipt_sha stored_terminal + reset_durable_readback_verdict receipt_name="$(receipt_name_for_identity)" durable_readback_path="$STATE_DIR/readbacks/${receipt_name}.txt" [ -f "$durable_readback_path" ] || { @@ -618,13 +900,10 @@ write_public_receipt() { && mv "$tmp" "$receipt_path"; then receipt_write_performed=1 receipt_status="written_bounded_timer_slot" - receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" receipt_ref="host112-timer-ledger:${receipt_name}" - durable_receipt_present=1 - durable_receipt_identity_match=1 - durable_receipt_boot_match=1 - durable_receipt_terminal="$run_terminal" - return 0 + if verify_current_receipt_from_disk "$run_terminal"; then + return 0 + fi elif [ "$MODE" != "timer_apply" ] \ && printf '%s\n' "$receipt_line" >"$tmp" \ && chmod 0640 "$tmp" \ @@ -632,13 +911,10 @@ write_public_receipt() { rm -f "$tmp" receipt_write_performed=1 receipt_status="written_immutable" - receipt_sha256="$(sha256sum "$receipt_path" | awk '{print $1}')" receipt_ref="host112-manager-recovery:${receipt_name}" - durable_receipt_present=1 - durable_receipt_identity_match=1 - durable_receipt_boot_match=1 - durable_receipt_terminal="$run_terminal" - return 0 + if verify_current_receipt_from_disk "$run_terminal"; then + return 0 + fi fi rm -f "$tmp" >/dev/null 2>&1 || true if [ "$MODE" != "timer_apply" ] && load_durable_receipt; then @@ -720,19 +996,15 @@ write_bounded_timer_mutation_readback() { return 1 } tmp="${durable_readback_path}.tmp" - if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite durable_readback_status=written_bounded_timer_slot durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ + if printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s timer_ledger_slot=%s timer_ledger_capacity=%s retention=rolling_atomic_overwrite stored_readback_kind=bounded_timer_slot durable_readback_path=%s\n' \ "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" \ "$timer_ledger_slot" "$timer_ledger_capacity" "$durable_readback_path" >"$tmp" \ && chmod 0640 "$tmp" \ && mv "$tmp" "$durable_readback_path"; then - durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" - durable_readback_status="written_bounded_timer_slot" - durable_readback_present=1 - durable_readback_identity_match=1 - durable_readback_boot_match=1 - durable_readback_receipt_link_match=1 - durable_readback_terminal_match=1 - return 0 + if verify_current_readback_from_disk; then + durable_readback_status="written_bounded_timer_slot" + return 0 + fi fi rm -f "$tmp" >/dev/null 2>&1 || true durable_readback_status="write_failed" @@ -806,7 +1078,7 @@ write_immutable_readback() { fi install -d -m 0750 "$(dirname "$durable_readback_path")" || return 1 tmp="${durable_readback_path}.tmp" - if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s durable_readback_status=written_immutable durable_readback_path=%s durable_readback_present=1 durable_readback_identity_match=1 durable_readback_boot_match=1 durable_readback_receipt_link_match=1 durable_readback_terminal_match=1\n' \ + if ! printf 'generated_at=%s %s receipt_path=%s receipt_sha256=%s stored_readback_kind=immutable durable_readback_path=%s\n' \ "$(date --iso-8601=seconds)" "$(canonical_readback_base)" "$receipt_path" "$receipt_sha256" "$durable_readback_path" >"$tmp" \ || ! chmod 0640 "$tmp" \ || ! ln "$tmp" "$durable_readback_path" 2>/dev/null; then @@ -815,13 +1087,11 @@ write_immutable_readback() { return 1 fi rm -f "$tmp" - durable_readback_sha256="$(sha256sum "$durable_readback_path" | awk '{print $1}')" + if ! verify_current_readback_from_disk; then + durable_readback_status="written_immutable_disk_verification_failed" + return 1 + fi durable_readback_status="written_immutable" - durable_readback_present=1 - durable_readback_identity_match=1 - durable_readback_boot_match=1 - durable_readback_receipt_link_match=1 - durable_readback_terminal_match=1 # Preserve the historical convenience readback as an explicitly mutable # pointer. It is never used as execution proof; the immutable path/hash is. @@ -834,8 +1104,11 @@ write_immutable_readback() { return 0 fi rm -f "$latest_tmp" >/dev/null 2>&1 || true - durable_readback_status="immutable_written_latest_pointer_failed" - return 1 + # The immutable artifact remains verified even when this auxiliary mutable + # convenience pointer cannot be refreshed. Keep the stable consumer status; + # latest_readback_pointer_written=0 carries the narrower pointer failure. + durable_readback_status="written_immutable" + return 0 } boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)" @@ -875,6 +1148,41 @@ if [ "$durable_receipt_present" -eq 1 ]; then fi fi +# Every dry-run first proves that no unresolved fixed WAL makes the planned +# apply impossible. When a WAL exists, hold a shared lock on the already- +# existing lock file so an apply cannot race this read-only verifier. Only a +# post-apply dry-run with a current immutable pair performs the extra in-memory +# pair verification; neither path creates or mutates state. +if [ "$MODE" = "dry_run" ]; then + if [ -f "$ARTIFACT_TRANSACTION_PATH" ]; then + if ! { [ -f "$MANAGER_LOCK_FILE" ] \ + && exec 8<"$MANAGER_LOCK_FILE" \ + && flock -s -n 8 \ + && inspect_active_artifact_transaction; }; then + artifact_chain_failed=1 + artifact_transaction_blocked=1 + if [ "$artifact_transaction_status" = "not_requested" ]; then + artifact_transaction_status="blocked_artifact_transaction_verifier_lock_unavailable" + artifact_transaction_phase="read_only_verifier_denied" + fi + fi + else + inspect_active_artifact_transaction + fi + if [ "$artifact_chain_failed" -eq 0 ] \ + && [ "$durable_receipt_present" -eq 1 ]; then + if [ "$artifact_transaction_prior_pair_verified" -ne 1 ]; then + artifact_chain_failed=1 + artifact_transaction_blocked=1 + artifact_transaction_status="blocked_fixed_wal_not_verified_for_durable_pair" + artifact_transaction_phase="read_only_pair_verifier_denied" + elif ! commit_current_artifact_transaction; then + artifact_chain_failed=1 + artifact_transaction_blocked=1 + fi + fi +fi + # Capture a pre-mutation baseline so a TERM/INT/HUP receipt never relies on # unset state, even when interruption lands during the general guest repair. capture_manager_state @@ -901,16 +1209,18 @@ handle_executor_signal() { fi capture_manager_state || true if [ "$runtime_write_performed" -eq 1 ]; then - if write_public_receipt && write_signal_readback; then - exit 75 + if write_public_receipt; then + mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}" + write_signal_readback || true + else + mark_artifact_transaction_partial "receipt_write_failed_after_signal_${signal_name}" fi - exit 74 + exit 1 fi exit 75 } -if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ - && [ "$immutable_replay" -eq 0 ]; then +if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then if [ "${EUID:-$(id -u)}" -ne 0 ]; then echo "BLOCKER host112_apply_requires_root" >&2 exit 77 @@ -920,18 +1230,30 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ echo "BLOCKER host112_manager_single_flight_busy" >&2 exit 75 fi + if ! inspect_active_artifact_transaction; then + record_action "artifact_transaction_unresolved_prior_run_failed" + artifact_chain_failed=1 + fi trap 'handle_executor_signal TERM' TERM trap 'handle_executor_signal INT' INT trap 'handle_executor_signal HUP' HUP +fi +if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ + && [ "$immutable_replay" -eq 0 ] \ + && [ "$artifact_transaction_blocked" -eq 0 ]; then current_default="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" if [ "$current_default" != "graphical.target" ]; then - runtime_write_performed=1 - state_write_performed=1 - if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then - record_action "set_default_graphical_target" + if authorize_runtime_mutation "set_default_graphical_target"; then + runtime_write_performed=1 + state_write_performed=1 + if bounded_timeout "$SERVICE_START_TIMEOUT_SECONDS" systemctl set-default graphical.target >/dev/null; then + record_action "set_default_graphical_target" + else + record_action "set_default_graphical_target_failed" + fi else - record_action "set_default_graphical_target_failed" + record_action "set_default_graphical_target_artifact_transaction_blocked_failed" fi fi @@ -964,18 +1286,23 @@ if { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ fi is_uint "$last_retry_epoch" || last_retry_epoch=0 if [ $((now_epoch - last_retry_epoch)) -ge "$INDEXER_RETRY_COOLDOWN_SECONDS" ]; then - if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then + if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then + # A marker write can partially mutate state before a non-zero return. runtime_write_performed=1 state_write_performed=1 - bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \ - systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true - if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then - record_action "retry_wazuh_indexer" + if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then + bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" \ + systemctl reset-failed wazuh-indexer.service >/dev/null 2>&1 || true + if bounded_timeout "$INDEXER_START_TIMEOUT_SECONDS" systemctl start wazuh-indexer.service; then + record_action "retry_wazuh_indexer" + else + record_action "retry_wazuh_indexer_failed" + fi else - record_action "retry_wazuh_indexer_failed" + record_action "wazuh_indexer_marker_write_failed" fi else - record_action "wazuh_indexer_marker_write_failed_no_write" + record_action "wazuh_indexer_artifact_transaction_blocked_failed" fi else record_action "wazuh_indexer_retry_cooldown" @@ -988,11 +1315,15 @@ now_epoch="$(date +%s)" evaluate_manager_preflight "$now_epoch" "$uptime_seconds" if [ "$MODE" = "dry_run" ]; then - case "$manager_preflight_status" in - ready) manager_terminal="dry_run_ready" ;; - already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;; - *) manager_terminal="dry_run_${manager_preflight_status}" ;; - esac + if [ "$artifact_chain_failed" -eq 1 ]; then + manager_terminal="dry_run_artifact_transaction_blocked" + else + case "$manager_preflight_status" in + ready) manager_terminal="dry_run_ready" ;; + already_verified_healthy) manager_terminal="dry_run_idempotent_already_healthy" ;; + *) manager_terminal="dry_run_${manager_preflight_status}" ;; + esac + fi elif { [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; } \ && [ "$immutable_replay" -eq 1 ]; then manager_terminal="immutable_receipt_replay" @@ -1002,48 +1333,52 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then manager_terminal="idempotent_already_verified_healthy" ;; ready) - if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then + if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then # The cooldown marker is a durable runtime state mutation even if the - # following systemctl operation fails. Account for it immediately so - # no failure path can emit a false no-write receipt. + # marker writer returns non-zero. Account for the attempt immediately. runtime_write_performed=1 state_write_performed=1 - manager_marker_write_performed=1 - manager_attempt_count=1 - if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then - record_action "reset_failed_wazuh_manager" - runtime_write_performed=1 - manager_runtime_write_performed=1 - if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then - manager_start_exit=0 - else - manager_start_exit=$? - fi - if wait_for_manager_verifier; then - if [ "$manager_start_exit" = "0" ]; then - record_action "start_wazuh_manager_verified" - manager_terminal="verified_healthy" + if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then + manager_marker_write_performed=1 + manager_attempt_count=1 + if bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service >/dev/null 2>&1; then + record_action "reset_failed_wazuh_manager" + runtime_write_performed=1 + manager_runtime_write_performed=1 + if bounded_timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service; then + manager_start_exit=0 else - record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed" - manager_terminal="manager_start_nonzero_verifier_healthy_unattributed" + manager_start_exit=$? + fi + if wait_for_manager_verifier; then + if [ "$manager_start_exit" = "0" ]; then + record_action "start_wazuh_manager_verified" + manager_terminal="verified_healthy" + else + record_action "wazuh_manager_verifier_healthy_after_nonzero_start_unattributed" + manager_terminal="manager_start_nonzero_verifier_healthy_unattributed" + fi + else + record_action "start_wazuh_manager_verifier_failed" + rollback_manager_to_inactive + if [ "$rollback_verified" -eq 1 ]; then + manager_terminal="rolled_back_verifier_failed" + else + manager_terminal="rollback_failed_degraded" + fi fi else - record_action "start_wazuh_manager_verifier_failed" - rollback_manager_to_inactive - if [ "$rollback_verified" -eq 1 ]; then - manager_terminal="rolled_back_verifier_failed" - else - manager_terminal="rollback_failed_degraded" - fi + manager_start_exit="reset_failed_no_start" + record_action "reset_failed_wazuh_manager_failed_no_start" + manager_terminal="no_start_reset_failed" fi else - manager_start_exit="reset_failed_no_start" - record_action "reset_failed_wazuh_manager_failed_no_start" - manager_terminal="no_start_reset_failed" + record_action "wazuh_manager_marker_write_failed" + manager_terminal="cooldown_marker_write_failed_degraded" fi else - record_action "wazuh_manager_marker_write_failed_no_write" - manager_terminal="no_write_cooldown_marker_write_failed" + record_action "wazuh_manager_artifact_transaction_blocked_failed" + manager_terminal="blocked_unresolved_artifact_transaction" fi ;; *) @@ -1159,13 +1494,22 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then if [ "$MODE" = "timer_apply" ]; then timer_artifact_policy="bounded_ring_128_for_runtime_mutation" fi - if ! write_public_receipt; then + if ! authorize_runtime_mutation "public_artifact_commit"; then + artifact_chain_failed=1 + run_terminal="${run_terminal}_artifact_transaction_blocked" + elif write_public_receipt; then + artifact_receipt_ready=1 + else + artifact_chain_failed=1 + mark_artifact_transaction_partial "receipt_write_or_disk_verification_failed" run_terminal="${run_terminal}_receipt_failed" fi else receipt_status="timer_no_runtime_mutation_aggregated" timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact" fi + elif [ "$durable_receipt_present" -eq 1 ]; then + artifact_receipt_ready=1 fi if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then if ! write_bounded_timer_observation; then @@ -1178,7 +1522,7 @@ fi trace_output="${TRACE_ID:-none}" run_output="${RUN_ID:-none}" work_item_output="${WORK_ITEM_ID:-none}" -readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" +readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed artifact_transaction_status=$artifact_transaction_status artifact_transaction_phase=$artifact_transaction_phase artifact_transaction_path=$ARTIFACT_TRANSACTION_PATH artifact_transaction_started=$artifact_transaction_started artifact_transaction_blocked=$artifact_transaction_blocked artifact_transaction_write_performed=$artifact_transaction_write_performed artifact_transaction_pair_verified=$artifact_transaction_pair_verified artifact_transaction_prior_pair_verified=$artifact_transaction_prior_pair_verified receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" canonical_readback_base() { printf '%s\n' "$readback" | awk ' @@ -1188,6 +1532,9 @@ canonical_readback_base() { if ($i ~ /^durable_readback_(present|identity_match|boot_match|receipt_link_match|terminal_match)=/) { continue } + if ($i ~ /^artifact_transaction_/) { + continue + } printf "%s%s", separator, $i separator = " " } @@ -1197,8 +1544,12 @@ canonical_readback_base() { } emit_readback() { - printf '%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \ - "$(canonical_readback_base)" "$durable_readback_status" "$durable_readback_path" \ + printf '%s artifact_transaction_status=%s artifact_transaction_phase=%s artifact_transaction_path=%s artifact_transaction_started=%s artifact_transaction_blocked=%s artifact_transaction_write_performed=%s artifact_transaction_pair_verified=%s artifact_transaction_prior_pair_verified=%s durable_readback_status=%s durable_readback_path=%s durable_readback_sha256=%s durable_readback_present=%s durable_readback_identity_match=%s durable_readback_boot_match=%s durable_readback_receipt_link_match=%s durable_readback_terminal_match=%s latest_readback_pointer_written=%s\n' \ + "$(canonical_readback_base)" "$artifact_transaction_status" "$artifact_transaction_phase" \ + "$ARTIFACT_TRANSACTION_PATH" "$artifact_transaction_started" \ + "$artifact_transaction_blocked" "$artifact_transaction_write_performed" \ + "$artifact_transaction_pair_verified" "$artifact_transaction_prior_pair_verified" \ + "$durable_readback_status" "$durable_readback_path" \ "$durable_readback_sha256" "$durable_readback_present" \ "$durable_readback_identity_match" "$durable_readback_boot_match" \ "$durable_readback_receipt_link_match" "$durable_readback_terminal_match" \ @@ -1206,24 +1557,39 @@ emit_readback() { } if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then - if [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then + if [ "$artifact_chain_failed" -eq 1 ]; then + emit_readback + exit 1 + elif [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ]; then if [ "$timer_observation_failed" -eq 1 ]; then emit_readback - exit 74 + exit 1 fi elif [ "$MODE" = "timer_apply" ]; then if ! write_bounded_timer_mutation_readback; then + mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed" emit_readback - exit 74 + exit 1 fi elif ! write_immutable_readback; then + mark_artifact_transaction_partial "durable_readback_write_or_disk_verification_failed" emit_readback - exit 74 + exit 1 + fi + if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then + if ! commit_current_artifact_transaction; then + mark_artifact_transaction_partial "artifact_pair_commit_failed" + emit_readback + exit 1 + fi fi fi emit_readback if [ "$MODE" = "dry_run" ]; then + if [ "$artifact_chain_failed" -eq 1 ]; then + exit 1 + fi case "$manager_preflight_status" in ready|already_verified_healthy) exit 0 ;; *) exit 1 ;; @@ -1237,6 +1603,8 @@ if [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then fi [ "$guest_ready" -eq 1 ] \ && [ "$receipt_status" != "write_failed" ] \ + && { [ "$MODE" = "timer_apply" ] && [ "$runtime_write_performed" -eq 0 ] \ + || [ "$artifact_transaction_pair_verified" -eq 1 ]; } \ && { [ "$successful_terminal" = "verified_healthy" ] \ || [ "$successful_terminal" = "idempotent_already_verified_healthy" ]; } exit $? diff --git a/scripts/reboot-recovery/install-host112-guest-recovery.sh b/scripts/reboot-recovery/install-host112-guest-recovery.sh index 8793dc0e5..b7cef9c8d 100755 --- a/scripts/reboot-recovery/install-host112-guest-recovery.sh +++ b/scripts/reboot-recovery/install-host112-guest-recovery.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Host 112 guest recovery installer. # Version: 2.1 -# Last modified: 2026-07-14 12:10 (Asia/Taipei) +# Last modified: 2026-07-14 13:15 (Asia/Taipei) # Last modified by: Codex -# Change: make installation transactional, verify the Windows99 direct route -# and exact sudo policy, and keep installation separate from recovery runs. +# Change: make timer shutdown state-aware on clean installs while preserving +# transactional rollback, exact sudo policy, and recovery-run separation. set -Eeuo pipefail @@ -66,6 +66,9 @@ exact correlated apply/dry-run sudo rules, the host-110 forced readback key, and the existing Windows99 direct Agent99 authorization. It starts only the timer after all static gates pass; it never synchronously starts the recovery oneshot, reboots the host, or changes VM power. + +HOST112_CONTROL_PUBLIC_KEY_PATH must point to an explicitly pre-staged public +key file. The installer never discovers, reads, copies, or derives a private key. USAGE } @@ -487,7 +490,9 @@ static_source_preflight() { for tool in bash cksum cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; } done - for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do + [ -f "$CONTROL_PUBLIC_KEY_PATH" ] \ + || { echo "BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH" >&2; return 1; } + for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do [ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; return 1; } done bash -n "$READINESS_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh" @@ -565,6 +570,25 @@ wait_for_recovery_service_idle() { done } +stop_recovery_timer_if_running() { + local state + state="$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + case "${state:-unknown}" in + active|activating|reloading|deactivating) + timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1 + ;; + inactive|failed|not-found|unknown) + # A clean install has no timer unit yet. Treat that as an idempotent + # no-op instead of asking systemd to stop a unit which does not exist. + return 0 + ;; + *) + echo "BLOCKER unexpected_recovery_timer_state=${state}" >&2 + return 1 + ;; + esac +} + resolve_target_user if [ "$MODE" = "check" ]; then @@ -612,7 +636,7 @@ TRANSACTION_ACTIVE=1 trap 'handle_apply_failure $?' ERR trap 'handle_apply_failure 130' INT TERM -timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1 +stop_recovery_timer_if_running wait_for_recovery_service_idle install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET" diff --git a/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py new file mode 100644 index 000000000..cd0ead9d1 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py @@ -0,0 +1,203 @@ +from __future__ import annotations + +import json +import shutil +import subprocess +from copy import deepcopy +from pathlib import Path +from typing import Any + +import pytest + + +ROOT = Path(__file__).resolve().parents[3] +PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1" + + +def _function_source() -> str: + source = PREFLIGHT.read_text(encoding="utf-8") + start = source.index("function Get-AgentLivePreflightDecision") + end = source.index("function Get-TaskReadback", start) + return source[start:end] + + +def _base_case() -> dict[str, Any]: + return { + "agentRootPresent": True, + "manifest": { + "exists": True, + "sourceRevision": "a" * 40, + "runtimeMatched": True, + "fileCount": 14, + "mismatchCount": 0, + "parseError": "", + }, + "taskFailureCount": 0, + "relayListenerCount": 1, + "requiredEvidenceStaleCount": 0, + "requiredEvidenceParseFailureCount": 0, + "requiredEvidenceContentFailureCount": 0, + "selfHealthSeverity": "ok", + "selfHealthPerformanceIssueCount": 0, + "staleAlertExecutionCount": 0, + "alertIncomingCount": 0, + "pendingCommandCount": 0, + } + + +def _with_overrides(**overrides: Any) -> dict[str, Any]: + case = deepcopy(_base_case()) + manifest_overrides = overrides.pop("manifest", None) + case.update(overrides) + if manifest_overrides: + case["manifest"].update(manifest_overrides) + return case + + +def _as_list(value: Any) -> list[str]: + if value is None: + return [] + if isinstance(value, list): + return [str(item) for item in value] + return [str(value)] + + +def _run_decision(powershell: str, case: dict[str, Any]) -> dict[str, Any]: + fixture = json.dumps(case, separators=(",", ":")) + command = f""" +{_function_source()} +$case = @' +{fixture} +'@ | ConvertFrom-Json +$parameters = @{{ + AgentRootPresent = [bool]$case.agentRootPresent + Manifest = $case.manifest + ExpectedRuntimeFileCount = 14 + TaskFailureCount = [int]$case.taskFailureCount + RelayListenerCount = [int]$case.relayListenerCount + RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount + RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount + RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount + SelfHealthSeverity = [string]$case.selfHealthSeverity + SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount + StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount + AlertIncomingCount = [int]$case.alertIncomingCount + PendingCommandCount = [int]$case.pendingCommandCount +}} +Get-AgentLivePreflightDecision @parameters | + ConvertTo-Json -Depth 6 -Compress +""" + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + payload = next( + line for line in reversed(result.stdout.splitlines()) if line.startswith("{") + ) + return json.loads(payload) + + +def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> None: + source = PREFLIGHT.read_text(encoding="utf-8") + decision = _function_source() + + assert '$warningReasons += "self_health_warning"' in decision + assert '$warningReasons += "performance_warning"' in decision + assert '"critical" { $blockingReasons += "self_health_critical" }' in decision + assert 'default { $blockingReasons += "self_health_unclassified" }' in decision + assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision + assert '$blockingReasons += "runtime_manifest_mismatch"' in decision + assert '$blockingReasons += "scheduled_task_unhealthy"' in decision + assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision + assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision + assert "preflightGreen = [bool]$decision.deploymentEligible" in source + assert "warningReasons = $warningReasons" in source + assert "selfHealthPerformanceIssues = @(" in source + assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source + assert 'if ($reason -match "^[a-z0-9_]{1,80}$")' in source + assert ( + "[int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount" + in source + ) + assert 'if ($result.summary.deploymentEligible)' in source + assert 'if ($selfHealthSeverity -ne "ok")' not in source + + +def test_decision_behavior_when_powershell_is_available() -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + cases = ( + (_base_case(), True, (), ()), + ( + _with_overrides( + selfHealthSeverity="warning", + selfHealthPerformanceIssueCount=1, + ), + True, + (), + ("self_health_warning", "performance_warning"), + ), + ( + _with_overrides(selfHealthSeverity="critical"), + False, + ("self_health_critical",), + (), + ), + ( + _with_overrides(taskFailureCount=1), + False, + ("scheduled_task_unhealthy",), + (), + ), + ( + _with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}), + False, + ("runtime_manifest_mismatch",), + (), + ), + ( + _with_overrides(manifest={"sourceRevision": "not-a-full-sha"}), + False, + ("runtime_manifest_identity_invalid",), + (), + ), + ( + _with_overrides(relayListenerCount=0), + False, + ("sre_alert_relay_not_listening",), + (), + ), + ) + + for case, expected_eligible, expected_blockers, expected_warnings in cases: + result = _run_decision(powershell, case) + assert result["deploymentEligible"] is expected_eligible + assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers) + assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings) + + preflight_path = str(PREFLIGHT).replace("'", "''") + parser_command = ( + "$tokens=$null; $errors=$null; " + "[System.Management.Automation.Language.Parser]::ParseFile(" + f"'{preflight_path}', [ref]$tokens, [ref]$errors) | Out-Null; " + "if ($errors.Count -gt 0) { " + "$errors | ForEach-Object { $_.Message }; exit 1 }" + ) + parsed = subprocess.run( + [ + powershell, + "-NoProfile", + "-NonInteractive", + "-Command", + parser_command, + ], + check=False, + capture_output=True, + text=True, + ) + assert parsed.returncode == 0, parsed.stdout + parsed.stderr diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index c37f08512..1c409f433 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -109,7 +109,15 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None: assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight assert 'Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"' in preflight - assert 'if ($selfHealthSeverity -ne "ok")' in preflight + assert "function Get-AgentLivePreflightDecision" in preflight + assert '"warning" {' in preflight + assert '$warningReasons += "self_health_warning"' in preflight + assert '$warningReasons += "performance_warning"' in preflight + assert '"critical" { $blockingReasons += "self_health_critical" }' in preflight + assert 'default { $blockingReasons += "self_health_unclassified" }' in preflight + assert 'if ($selfHealthSeverity -ne "ok")' not in preflight + assert 'Write-Output "DEPLOYMENT_ELIGIBLE=$([int]$result.summary.deploymentEligible)"' in preflight + assert 'Write-Output "WARNING_REASONS=$($warningReasons -join \',\')"' in preflight assert "runtime-manifest.json" in preflight assert "Get-ScheduledTask" in preflight assert "Get-NetTCPConnection -LocalPort 8787" in preflight @@ -143,6 +151,9 @@ def test_live_preflight_adapts_inbox_evidence_without_raw_message_readback() -> "updatesSeen", "processedCount", "alertedCount", + "selfHealthPerformanceSeverity", + "selfHealthPerformanceIssueCount", + "selfHealthPerformanceIssues", ): assert safe_field in preflight for forbidden in ("messageText", "rawUpdate", "botToken", "chatId"): diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 6f9e48923..a56407313 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -7,6 +7,8 @@ import shutil import subprocess from pathlib import Path +import pytest + ROOT = Path(__file__).resolve().parents[3] SENDER = ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh" RECEIVER = ( @@ -259,6 +261,178 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary() assert "[bool]$prior.relayReload.newGenerationVerified" in source +def test_receiver_creates_canonical_success_receipt_once_with_atomic_no_replace() -> None: + source = RECEIVER.read_text(encoding="utf-8") + writer = source[ + source.index("function Write-AgentRemoteDeployReceipt") : source.index( + "function Write-AgentResultAndExit" + ) + ] + + assert ( + source.count( + "Write-AgentRemoteDeployReceipt $receiptPath $successReceipt" + ) + == 1 + ) + assert 'throw "immutable_receipt_path_already_exists"' in writer + assert writer.count("[IO.File]::Move($temporary, $Path)") == 1 + assert "Move-Item -LiteralPath $temporary -Destination $Path -Force" not in writer + assert "$json = $Receipt | ConvertTo-Json -Depth 12" in writer + assert "$null = $storedJson | ConvertFrom-Json" in writer + + atomic_move = writer.index("[IO.File]::Move($temporary, $Path)") + catch_boundary = writer.index("} catch {", atomic_move) + assert writer[atomic_move:catch_boundary].strip().endswith( + "[IO.File]::Move($temporary, $Path)" + ) + + +def test_receiver_uses_unique_filesystem_safe_attempt_failure_receipts() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + assert '"yyyyMMdd\'T\'HHmmssfffffff\'Z\'"' in source + assert "[Globalization.CultureInfo]::InvariantCulture" in source + assert ( + '$attemptToken = "$attemptStamp-$PID-$([Guid]::NewGuid().ToString(\'N\'))"' + in source + ) + assert ( + '"agent99-remote-deploy-$stageToken-attempt-$attemptToken.json"' + in source + ) + assert source.count( + "Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt" + ) >= 5 + assert "yyyyMMddTHHmmssfffffffK" not in source + assert 'ToString("K")' not in source + + +def test_success_receipt_failure_rolls_back_before_durable_attempt_receipt() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + success_construction = source.index( + '$script:ReceiverOperationPhase = "success_receipt_construction"' + ) + canonical_write = source.index( + "Write-AgentRemoteDeployReceipt $receiptPath $successReceipt" + ) + failure_branch = source.index("if ($failurePhase)", canonical_write) + rollback = source.index( + "$rollback = Restore-AgentBundle $backupPath $beforeFilePresence", + failure_branch, + ) + rollback_restart = source.index( + "Invoke-AgentRelayTaskRestart -ForbiddenGenerations " + "$rollbackForbiddenGenerations", + rollback, + ) + failure_receipt = source.index( + "Write-AgentRemoteDeployReceipt $attemptReceiptPath $receipt", + rollback_restart, + ) + + assert success_construction < canonical_write < failure_branch + assert failure_branch < rollback < rollback_restart < failure_receipt + assert ( + '$failurePhase = $script:ReceiverOperationPhase' in source + ) + assert ( + '$failurePhase -in @("success_receipt_construction", ' + '"success_receipt_write")' + in source + ) + assert '"rolled_back_success_receipt_write_failed"' in source + assert '"rollback_failed_after_success_receipt_write_failure"' in source + assert "canonicalReceiptPath = $receiptPath" in source + assert "attemptReceiptPath = $attemptReceiptPath" in source + assert "durableReceiptWritten = $true" in source + + +def test_existing_canonical_receipt_blocks_live_promotion_without_overwrite() -> None: + source = RECEIVER.read_text(encoding="utf-8") + + canonical_guard = source.index("if ($canonicalReceiptExists)") + validate_only = source.index( + "$validate = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision -ValidateOnly" + ) + live_deploy = source.index( + "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath " + "-SourceRevision $sourceRevision" + ) + assert canonical_guard < validate_only < live_deploy + assert '"blocked_canonical_receipt_invalid_no_promotion"' in source + assert ( + '"blocked_canonical_success_receipt_runtime_drift_no_promotion"' + in source + ) + assert '"blocked_canonical_receipt_reserved_no_promotion"' in source + assert ( + 'nextSafeAction = "preserve_immutable_canonical_receipt_and_retry_with_new_' + 'trace_run_work_item_identity"' + in source + ) + + +def test_outer_catch_reports_conservative_operation_boundaries() -> None: + source = RECEIVER.read_text(encoding="utf-8") + transport_failure = source.index('status = "transport_receiver_failed"') + outer_catch = source[ + source.rfind("} catch {", 0, transport_failure) : source.index( + "} finally {", transport_failure + ) + ] + + for tracker in ( + "$script:ReceiverRemoteWritePerformed", + "$script:ReceiverLiveRuntimeWritePerformed", + "$script:ReceiverLivePromotionAttempted", + "$script:ReceiverLivePromotionPerformed", + "$script:ReceiverRollbackAttempted", + "$script:ReceiverRollbackVerified", + "$script:ReceiverScheduledTaskRestartPerformed", + ): + assert tracker in outer_catch + assert "operationBoundaries = [pscustomobject]@{" in outer_catch + assert "secretValueRead = $false" in outer_catch + assert "privateKeyValueRead = $false" in outer_catch + assert "tokenValueRead = $false" in outer_catch + assert "environmentSecretRead = $false" in outer_catch + assert "rawEnvelopeStored = $false" in outer_catch + assert "$outerErrorType = $_.Exception.GetType().Name" in outer_catch + assert "$script:ReceiverAttemptReceiptPath" in outer_catch + assert ( + "Write-AgentRemoteDeployReceipt " + "$script:ReceiverAttemptReceiptPath $outerFailure" + ) in outer_catch + assert "durableReceiptWritten = $false" in outer_catch + assert "$outerFailure.durableReceiptWritten = $true" in outer_catch + assert "$outerFailure.durableReceiptWritten = $false" in outer_catch + + +def test_receiver_parses_with_powershell_ast_when_runtime_is_available() -> None: + powershell = shutil.which("pwsh") or shutil.which("powershell") + if powershell is None: + pytest.skip("PowerShell runtime is not installed on this macOS verifier") + + receiver_path = str(RECEIVER).replace("'", "''") + parser_command = ( + "$tokens=$null; $errors=$null; " + "[System.Management.Automation.Language.Parser]::ParseFile(" + f"'{receiver_path}', [ref]$tokens, [ref]$errors) | Out-Null; " + "if ($errors.Count -gt 0) { " + "$errors | ForEach-Object { $_.Message }; exit 1 }" + ) + result = subprocess.run( + [powershell, "-NoProfile", "-NonInteractive", "-Command", parser_command], + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stdout + result.stderr + + def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generation() -> None: source = RECEIVER.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index 8410443b4..97b8f61ed 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -1,6 +1,7 @@ from __future__ import annotations import hashlib +import json import os import shutil import subprocess @@ -180,37 +181,41 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None assert "return 1" in marker_function manager_start = source.index( - ' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then' + ' if authorize_runtime_mutation "wazuh_manager_cooldown_marker"; then' ) manager_end = source.index(" ;;", manager_start) manager_branch = source[manager_start:manager_end] marker_failure_at = manager_branch.index( - ' else\n record_action "wazuh_manager_marker_write_failed_no_write"' + ' else\n record_action "wazuh_manager_marker_write_failed"' ) success = manager_branch[:marker_failure_at] failure = manager_branch[marker_failure_at:] + authorization_at = success.index("authorize_runtime_mutation") + marker_write_at = success.index( + 'write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"' + ) marker_success_at = success.index( - ' manager_marker_write_performed=1' + ' manager_marker_write_performed=1' ) reset_at = success.index("systemctl reset-failed wazuh-manager.service") - assert marker_success_at < reset_at - assert "runtime_write_performed=1" in success[:reset_at] - assert "state_write_performed=1" in success[:reset_at] + assert authorization_at < marker_write_at < marker_success_at < reset_at + assert "runtime_write_performed=1" in success[:marker_write_at] + assert "state_write_performed=1" in success[:marker_write_at] assert "systemctl reset-failed wazuh-manager.service" in success assert "systemctl start wazuh-manager.service" in success assert 'bounded_timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success assert "manager_attempt_count=1" in success assert "systemctl reset-failed wazuh-manager.service" not in failure assert "systemctl start wazuh-manager.service" not in failure - assert "wazuh_manager_marker_write_failed_no_write" in failure - assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure + assert "wazuh_manager_marker_write_failed" in failure + assert 'manager_terminal="cooldown_marker_write_failed_degraded"' in failure assert "manager_marker_write_performed=1" not in failure assert "state_write_performed=$state_write_performed" in source assert "manager_marker_write_performed=$manager_marker_write_performed" in source indexer_start = source.index( - ' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then' + ' if authorize_runtime_mutation "wazuh_indexer_cooldown_marker"; then' ) indexer_end = source.index(" else\n record_action", indexer_start) indexer_branch = source[indexer_start:indexer_end] @@ -219,7 +224,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None assert "systemctl start wazuh-indexer.service" in indexer_success assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure assert "systemctl start wazuh-indexer.service" not in indexer_failure - assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure + assert "wazuh_indexer_marker_write_failed" in indexer_failure def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None: @@ -271,6 +276,46 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No assert "latest_readback_pointer_written=%s" in source +def test_runtime_mutations_are_guarded_by_a_fixed_durable_artifact_wal() -> None: + source = read(READINESS) + transaction = source[ + source.index("artifact_paths_for_current_identity() {") : source.index( + "load_durable_receipt() {" + ) + ] + + assert 'ARTIFACT_TRANSACTION_PATH="${STATE_DIR}/active-artifact-transaction.txt"' in source + assert "schema_version=host112_active_artifact_transaction_v1" in transaction + assert 'mv "$tmp" "$ARTIFACT_TRANSACTION_PATH"' in transaction + assert 'stored_schema="$(receipt_token "$ARTIFACT_TRANSACTION_PATH" schema_version)"' in transaction + assert 'artifact_transaction_status="blocked_unresolved_prior_transaction"' in transaction + assert 'write_artifact_transaction_record "pending_artifact_commit"' in transaction + assert 'write_artifact_transaction_record "partial_pending_public_artifact_commit"' in transaction + assert 'write_artifact_transaction_record "committed_verified_pair"' in transaction + assert "verify_artifact_pair_paths" in transaction + + apply = source[source.index(' exec 9>"$MANAGER_LOCK_FILE"') :] + inspect_at = apply.index("inspect_active_artifact_transaction") + first_mutation_at = apply.index("systemctl set-default graphical.target") + assert inspect_at < first_mutation_at + for phase in ( + "set_default_graphical_target", + "wazuh_indexer_cooldown_marker", + "wazuh_manager_cooldown_marker", + "public_artifact_commit", + ): + assert f'authorize_runtime_mutation "{phase}"' in source + + immutable_writer = source[ + source.index("write_immutable_readback() {") : source.index( + 'boot_id="$(cat /proc/sys/kernel/random/boot_id' + ) + ] + assert "durable_readback_present=1" not in immutable_writer + assert "durable_readback_identity_match=1" not in immutable_writer + assert "verify_current_readback_from_disk" in immutable_writer + + def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch( tmp_path: Path, ) -> None: @@ -352,7 +397,9 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No assert "trap 'handle_executor_signal TERM' TERM" in source assert "trap 'handle_executor_signal INT' INT" in source assert "trap 'handle_executor_signal HUP' HUP" in source - assert "if write_public_receipt && write_signal_readback; then" in source + assert "if write_public_receipt; then" in source + assert 'mark_artifact_transaction_partial "interrupted_before_durable_readback_${signal_name}"' in source + assert "write_signal_readback || true" in source assert "action_count=${#actions[@]} actions=$receipt_action_text" in source assert "schema_version=host112_guest_recovery_signal_readback_v1" in source assert 'ln "$tmp" "$signal_readback_path"' in source @@ -394,8 +441,9 @@ def test_apply_success_is_bound_to_same_run_durable_terminal() -> None: assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]') - receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier) + receipt_commit = source.index("elif write_public_receipt; then", guest_verifier) assert guest_verifier < receipt_commit + assert "commit_current_artifact_transaction" in source[receipt_commit:] assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source assert 'action_failure_observed=1' in source @@ -471,7 +519,13 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None: assert 'reason = "controlled_dispatch_identity_unsafe"' in control assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function assert '$dryRun.managerPreflightStatus -eq "ready"' in function + assert "-not $dryRun.artifactTransactionBlocked" in function + assert '$dryRun.artifactTransactionStatus -eq "prior_pair_verified"' in function assert 'receiptStatus -eq "written_immutable"' in function + assert '$applyReceipt.artifactTransactionStatus -eq "committed_verified_pair"' in function + assert "$applyReceipt.artifactTransactionPairVerified" in function + assert "$durableReceiptReadback.artifactTransactionPriorPairVerified" in function + assert '$durableReceiptReadback.artifactTransactionStatus -eq "existing_pair_verified"' in function assert '$after.managerVerified' in function assert 'name = "wazuh_manager_service_active"' in function assert 'name = "wazuh_analysisd_process_present"' in function @@ -570,11 +624,56 @@ def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None: assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy assert 'name = "host112_canonical_direct_route"' in deploy - assert 'Host112 canonical direct SSH config post-verifier failed.' in deploy + assert 'Host112 canonical SSH and VMX config post-verifier failed.' in deploy assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap - assert 'Host112 canonical direct SSH config bootstrap verifier failed.' in bootstrap + assert 'Host112 canonical SSH and VMX config bootstrap verifier failed.' in bootstrap for config in (generic_config, host_config): assert '"192.168.0.112": "kali"' in config assert '"directHosts"' in config assert '"192.168.0.112"' in config + + +def test_deploy_and_bootstrap_canonicalize_host112_ssd_vmx_identity() -> None: + deploy = read(ROOT / "agent99-deploy.ps1") + bootstrap = read(BOOTSTRAP) + contract = read(ROOT / "agent99-contract-check.ps1") + synthetic = read(ROOT / "agent99-synthetic-tests.ps1") + host_config = read(ROOT / "agent99.config.99.example.json") + canonical_ps = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" + canonical_json = r"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx" + policy = json.loads(host_config) + host112_rows = [ + row + for row in policy["vms"] + if row.get("host") == "192.168.0.112" or row.get("name") == "host112-kali" + ] + + assert canonical_json in host_config + assert host112_rows == [ + { + "name": "host112-kali", + "host": "192.168.0.112", + "vmx": canonical_ps, + "priority": 30, + } + ] + assert canonical_ps not in deploy + assert canonical_ps not in bootstrap + assert "Get-AgentHost112CanonicalVm $policyConfigPath" in deploy + assert "Get-AgentHost112CanonicalVm $PolicyPath" in bootstrap + bootstrap_preflight = ( + '$null = Get-AgentHost112CanonicalVm $host112BootstrapPolicyPath' + ) + bootstrap_first_write = "foreach ($dir in @($AgentRoot, $BinDir, $ConfigDir" + assert bootstrap.index(bootstrap_preflight) < bootstrap.index(bootstrap_first_write) + assert 'name = "host112_canonical_vmx_path"' in deploy + assert "Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf" in deploy + assert "Host112 canonical SSH and VMX config post-verifier failed." in deploy + assert "Host112 canonical SSH and VMX config bootstrap verifier failed." in bootstrap + assert '$persistedHost112Vm.Count -ne 1' in deploy + assert '$persistedHost112Vm.Count -ne 1' in bootstrap + assert 'Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in deploy + assert 'Set-AgentBootstrapProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in bootstrap + assert 'recovery:host112_canonical_vmx' in contract + assert 'recovery:host112_canonical_vmx' in synthetic diff --git a/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py b/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py index 4754e4d7f..eaa23d694 100644 --- a/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py +++ b/scripts/reboot-recovery/tests/test_host112_recovery_runtime_harness.py @@ -81,6 +81,21 @@ if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]] sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}" fi /bin/mv "$@" +""", + ) + _write_executable( + fake_bin / "ln", + """#!/usr/bin/env bash +set -u +destination="" +for argument in "$@"; do destination="$argument"; done +if [ "${FAKE_RECEIPT_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */receipts/*.txt ]]; then + exit 1 +fi +if [ "${FAKE_READBACK_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */readbacks/*.txt ]] && [[ "$destination" != *.signal.txt ]]; then + exit 1 +fi +/bin/ln "$@" """, ) _write_executable( @@ -118,6 +133,11 @@ printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /' """#!/usr/bin/env bash set -u state="${FAKE_HOST_STATE_DIR:?}" +require_wal() { + if [ ! -f "${HOST112_RECOVERY_STATE_DIR:?}/active-artifact-transaction.txt" ]; then + : >"$state/mutation_without_wal" + fi +} cmd="${1:-}" [ "$#" -gt 0 ] && shift case "$cmd" in @@ -154,10 +174,12 @@ case "$cmd" in ;; get-default) printf '%s\n' 'graphical.target' ;; set-default) + require_wal printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log" exit "${FAKE_SET_DEFAULT_RC:-0}" ;; start) + require_wal unit="${1:-}" printf 'start:%s\n' "$unit" >>"$state/mutations.log" if [ "$unit" = "lightdm.service" ]; then @@ -168,9 +190,10 @@ case "$cmd" in fi ;; enable) + require_wal printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log" ;; - reset-failed|stop) ;; + reset-failed|stop) require_wal ;; *) exit 1 ;; esac """, @@ -211,6 +234,81 @@ def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str ) +def _dry_run(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + "bash", + str(script), + "--dry-run", + "--trace-id", + "trace:runtime-harness", + "--run-id", + "run:runtime-harness", + "--work-item-id", + "HOST112-RUNTIME-HARNESS", + ], + text=True, + capture_output=True, + env=env, + check=False, + timeout=15, + ) + + +def test_post_apply_dry_run_read_only_verifies_fixed_wal_and_immutable_pair( + tmp_path: Path, +) -> None: + script, env, _ = _runtime_harness(tmp_path) + + applied = _apply(script, env) + assert applied.returncode == 0, applied.stdout + applied.stderr + transaction = tmp_path / "state" / "active-artifact-transaction.txt" + transaction_before = transaction.read_bytes() + + verified = _dry_run(script, env) + output = verified.stdout + verified.stderr + assert verified.returncode == 0, output + assert "durable_readback_status=loaded_immutable_verified " in output + assert "artifact_transaction_status=existing_pair_verified" in output + assert "artifact_transaction_started=0" in output + assert "artifact_transaction_blocked=0" in output + assert "artifact_transaction_write_performed=0" in output + assert "artifact_transaction_pair_verified=1" in output + assert "artifact_transaction_prior_pair_verified=1" in output + assert transaction.read_bytes() == transaction_before + + +def test_pre_apply_dry_run_blocks_unresolved_prior_transaction( + tmp_path: Path, +) -> None: + script, env, fake_state = _runtime_harness(tmp_path) + state = tmp_path / "state" + state.mkdir() + transaction = state / "active-artifact-transaction.txt" + unresolved = ( + "schema_version=host112_active_artifact_transaction_v1 " + "updated_at=2026-07-14T12:01:00+08:00 " + "status=pending_artifact_commit phase=prior_runtime_mutation " + "trace_id=trace:prior run_id=run:prior " + "work_item_id=HOST112-PRIOR boot_id=boot:prior " + f"receipt_path={state / 'receipts' / 'prior.txt'} " + f"readback_path={state / 'readbacks' / 'prior.txt'} " + "pair_verified=0\n" + ) + transaction.write_text(unresolved, encoding="utf-8") + Path(env["HOST112_MANAGER_LOCK_FILE"]).touch() + + dry_run = _dry_run(script, env) + output = dry_run.stdout + dry_run.stderr + assert dry_run.returncode == 1, output + assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output + assert "artifact_transaction_blocked=1" in output + assert "runtime_write_performed=0" in output + assert "receipt_write_performed=0" in output + assert transaction.read_text(encoding="utf-8") == unresolved + assert not (fake_state / "mutations.log").exists() + + def test_failed_general_run_cannot_be_washed_green_by_external_recovery( tmp_path: Path, ) -> None: @@ -223,16 +321,28 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery( assert failed.returncode == 1 assert "runtime_write_performed=1" in failed_output assert "receipt_status=written_immutable" in failed_output - assert "durable_readback_status=written_immutable" in failed_output + assert "durable_readback_status=written_immutable " in failed_output assert ( "durable_receipt_terminal=execution_failed_or_unattributed_" "idempotent_already_verified_healthy" ) in failed_output assert "guest_ready=1" in failed_output assert "action_failure_observed=1" in failed_output + assert "artifact_transaction_status=committed_verified_pair" in failed_output + assert "artifact_transaction_pair_verified=1" in failed_output + assert not (fake_state / "mutation_without_wal").exists() receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) readback = next((tmp_path / "state" / "readbacks").glob("*.txt")) + stored_readback = readback.read_text(encoding="utf-8") + for derived in ( + "durable_readback_present=1", + "durable_readback_identity_match=1", + "durable_readback_boot_match=1", + "durable_readback_receipt_link_match=1", + "durable_readback_terminal_match=1", + ): + assert derived not in stored_readback receipt_before = receipt.read_bytes() readback_before = readback.read_bytes() @@ -250,6 +360,114 @@ def test_failed_general_run_cannot_be_washed_green_by_external_recovery( assert readback.read_bytes() == readback_before +def test_immutable_replay_cannot_ignore_newer_unresolved_fixed_transaction( + tmp_path: Path, +) -> None: + script, env, fake_state = _runtime_harness(tmp_path) + + first = _apply(script, env) + assert first.returncode == 0, first.stdout + first.stderr + state = tmp_path / "state" + receipt = next((state / "receipts").glob("*.txt")) + readback = next((state / "readbacks").glob("*.txt")) + receipt_before = receipt.read_bytes() + readback_before = readback.read_bytes() + transaction = state / "active-artifact-transaction.txt" + unresolved = ( + "schema_version=host112_active_artifact_transaction_v1 " + "updated_at=2026-07-14T12:01:00+08:00 " + "status=pending_artifact_commit phase=newer_runtime_mutation " + "trace_id=trace:newer run_id=run:newer " + "work_item_id=HOST112-NEWER boot_id=boot:newer " + f"receipt_path={state / 'receipts' / 'newer.txt'} " + f"readback_path={state / 'readbacks' / 'newer.txt'} " + "pair_verified=0\n" + ) + transaction.write_text(unresolved, encoding="utf-8") + mutations = fake_state / "mutations.log" + mutations_before = mutations.read_bytes() + + replay = _apply(script, env) + output = replay.stdout + replay.stderr + assert replay.returncode == 1, output + assert "receipt_status=reused_immutable" in output + assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output + assert "artifact_transaction_blocked=1" in output + assert "artifact_transaction_pair_verified=0" in output + assert transaction.read_text(encoding="utf-8") == unresolved + assert receipt.read_bytes() == receipt_before + assert readback.read_bytes() == readback_before + assert mutations.read_bytes() == mutations_before + + +def test_receipt_failure_leaves_durable_partial_without_readback_and_blocks_retry( + tmp_path: Path, +) -> None: + script, env, fake_state = _runtime_harness(tmp_path) + env.update( + { + "FAKE_LIGHTDM_START_RC": "124", + "FAKE_RECEIPT_LINK_FAIL": "1", + } + ) + + failed = _apply(script, env) + output = failed.stdout + failed.stderr + assert failed.returncode == 1 + assert "runtime_write_performed=1" in output + assert "receipt_status=write_failed" in output + assert "artifact_transaction_status=partial_pending_public_artifact_commit" in output + transaction = tmp_path / "state" / "active-artifact-transaction.txt" + stored = transaction.read_text(encoding="utf-8") + assert "status=partial_pending_public_artifact_commit" in stored + assert "phase=receipt_write_or_disk_verification_failed" in stored + assert not (tmp_path / "state" / "readbacks").exists() + assert not (fake_state / "mutation_without_wal").exists() + + mutations = fake_state / "mutations.log" + mutations.write_text("", encoding="utf-8") + retry_env = env.copy() + retry_env.pop("FAKE_RECEIPT_LINK_FAIL") + retry = _apply(script, retry_env) + retry_output = retry.stdout + retry.stderr + assert retry.returncode == 1 + assert "schema_version=host112_guest_recovery_v2" in retry_output + assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in ( + retry_output + ) + assert "terminal=execution_failed_or_unattributed_" in retry_output + assert "artifact_transaction_blocked" in retry_output + assert "receipt_write_performed=0" in retry_output + assert mutations.read_text(encoding="utf-8") == "" + + +def test_readback_failure_keeps_receipt_and_durable_partial_not_false_success( + tmp_path: Path, +) -> None: + script, env, fake_state = _runtime_harness(tmp_path) + env.update( + { + "FAKE_LIGHTDM_START_RC": "124", + "FAKE_READBACK_LINK_FAIL": "1", + } + ) + + failed = _apply(script, env) + output = failed.stdout + failed.stderr + assert failed.returncode == 1 + assert "receipt_status=written_immutable" in output + assert "durable_readback_status=write_failed" in output + assert "artifact_transaction_pair_verified=0" in output + receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) + assert "terminal=" in receipt.read_text(encoding="utf-8") + assert not list((tmp_path / "state" / "readbacks").glob("*.txt")) + transaction = tmp_path / "state" / "active-artifact-transaction.txt" + stored = transaction.read_text(encoding="utf-8") + assert "status=partial_pending_public_artifact_commit" in stored + assert "phase=durable_readback_write_or_disk_verification_failed" in stored + assert not (fake_state / "mutation_without_wal").exists() + + def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( tmp_path: Path, ) -> None: @@ -282,8 +500,12 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( mutation_root.mkdir() script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4) env["FAKE_LIGHTDM_START_RC"] = "124" - for index in range(12): - run_env = env | {"INVOCATION_ID": f"mutation-{index}"} + mutation_slots: list[str] = [] + # These five identities cover every four-slot ring position and then reuse + # one slot. That proves both capacity coverage and rollover without twelve + # expensive full shell-harness invocations. + for index in range(5): + run_env = env | {"INVOCATION_ID": f"slot-{index}"} result = subprocess.run( ["bash", str(script), "--apply-timer"], text=True, @@ -296,7 +518,14 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( assert result.returncode == 1 assert "runtime_write_performed=1" in output assert "receipt_status=written_bounded_timer_slot" in output - assert "durable_readback_status=written_bounded_timer_slot" in output + assert "durable_readback_status=written_bounded_timer_slot " in output + mutation_slots.append( + next( + token.split("=", 1)[1] + for token in output.split() + if token.startswith("timer_ledger_slot=") + ) + ) for field in ( "durable_readback_present", "durable_readback_identity_match", @@ -308,12 +537,15 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( assert f"{field}=0" not in output assert "timer_no_runtime_mutation_aggregated" not in output + assert len(set(mutation_slots)) == 4 + assert len(mutation_slots) > len(set(mutation_slots)) + ledger_files = [ path for path in (mutation_root / "state" / "timer-ledger").iterdir() if path.is_file() ] - assert len(ledger_files) <= 8 + assert len(ledger_files) == 8 assert not (mutation_root / "state" / "receipts").exists() assert not (mutation_root / "state" / "readbacks").exists() for readback in (mutation_root / "state" / "timer-ledger").glob( @@ -327,8 +559,10 @@ def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( "durable_readback_receipt_link_match", "durable_readback_terminal_match", ): - assert stored.count(f"{field}=1") == 1 + assert f"{field}=1" not in stored assert f"{field}=0" not in stored + transaction = mutation_root / "state" / "active-artifact-transaction.txt" + assert "status=committed_verified_pair" in transaction.read_text(encoding="utf-8") def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps( @@ -337,33 +571,50 @@ def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps( script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4) env["FAKE_LIGHTDM_START_RC"] = "124" - for index in range(12): - marker = tmp_path / f"receipt-mv-{index}.entered" - run_env = env | { - "INVOCATION_ID": f"sigkill-{index}", - "FAKE_RECEIPT_MV_MARKER": str(marker), - "FAKE_RECEIPT_MV_SLEEP_SECONDS": "5", - } - process = subprocess.Popen( - ["bash", str(script), "--apply-timer"], - text=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=run_env, - start_new_session=True, - ) - deadline = time.monotonic() + 8 - while not marker.exists() and time.monotonic() < deadline: - time.sleep(0.01) - assert marker.exists() - os.killpg(process.pid, signal.SIGKILL) - process.communicate(timeout=5) - assert process.returncode == -signal.SIGKILL + marker = tmp_path / "receipt-mv.entered" + run_env = env | { + "INVOCATION_ID": "sigkill-0", + "FAKE_RECEIPT_MV_MARKER": str(marker), + "FAKE_RECEIPT_MV_SLEEP_SECONDS": "5", + } + process = subprocess.Popen( + ["bash", str(script), "--apply-timer"], + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=run_env, + start_new_session=True, + ) + deadline = time.monotonic() + 8 + while not marker.exists() and time.monotonic() < deadline: + time.sleep(0.01) + assert marker.exists() + os.killpg(process.pid, signal.SIGKILL) + process.communicate(timeout=5) + assert process.returncode == -signal.SIGKILL ledger = tmp_path / "state" / "timer-ledger" files = [path for path in ledger.iterdir() if path.is_file()] - assert len(files) <= 4 + assert len(files) == 1 assert all(path.name.endswith(".receipt.txt.tmp") for path in files) + transaction = tmp_path / "state" / "active-artifact-transaction.txt" + assert "status=pending_artifact_commit" in transaction.read_text(encoding="utf-8") + + mutations = tmp_path / "fake-host" / "mutations.log" + before = mutations.read_bytes() + retry = subprocess.run( + ["bash", str(script), "--apply-timer"], + text=True, + capture_output=True, + env=env | {"INVOCATION_ID": "sigkill-retry"}, + check=False, + timeout=15, + ) + assert retry.returncode == 1 + assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in ( + retry.stdout + retry.stderr + ) + assert mutations.read_bytes() == before def test_signal_during_final_verifier_writes_correlated_failure_chain( @@ -400,7 +651,7 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain( assert marker.exists() os.kill(process.pid, signal.SIGTERM) stdout, stderr = process.communicate(timeout=12) - assert process.returncode == 75, stdout + stderr + assert process.returncode == 1, stdout + stderr receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt")) @@ -411,6 +662,15 @@ def test_signal_during_final_verifier_writes_correlated_failure_chain( assert f"receipt_path={receipt}" in signal_text assert "terminal=interrupted_signal_TERM" in signal_text assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text + transaction = tmp_path / "state" / "active-artifact-transaction.txt" + transaction_text = transaction.read_text(encoding="utf-8") + assert "status=partial_pending_public_artifact_commit" in transaction_text + assert "phase=interrupted_before_durable_readback_TERM" in transaction_text + assert not [ + path + for path in (tmp_path / "state" / "readbacks").glob("*.txt") + if not path.name.endswith(".signal.txt") + ] replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""}) assert replay.returncode == 65 diff --git a/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py index ce0a92738..7924d7c9e 100644 --- a/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import subprocess from pathlib import Path @@ -67,6 +68,8 @@ def test_windows99_direct_route_is_fingerprint_bound_and_not_forced() -> None: assert 'cat "$AUTH_FILE"' not in text assert 'from="%s",restrict,command="%s --check"' in text assert 'CONTROL_SOURCE_ADDRESS:-192.168.0.110' in text + assert 'BLOCKER control_public_key_not_staged=$CONTROL_PUBLIC_KEY_PATH' in text + assert "never discovers, reads, copies, or derives a private key" in text def test_apply_is_transactional_and_has_a_verified_rollback() -> None: @@ -149,3 +152,73 @@ def test_check_gate_requires_policy_route_timer_and_no_write_readback() -> None: assert 'runtime_write_performed=0' in text assert 'manager_runtime_write_performed=0' in text assert 'receipt_write_performed=0' in text + + +def test_clean_install_timer_stop_is_state_aware_and_idempotent( + tmp_path: Path, +) -> None: + text = source() + function = text[ + text.index("stop_recovery_timer_if_running() {") : text.index( + "\n\nresolve_target_user", + text.index("stop_recovery_timer_if_running() {"), + ) + ] + runner = tmp_path / "run-stop-contract.sh" + runner.write_text( + "#!/usr/bin/env bash\nset -u\n" + function + "\nstop_recovery_timer_if_running\n", + encoding="utf-8", + ) + runner.chmod(0o755) + fake_bin = tmp_path / "bin" + fake_bin.mkdir() + log = tmp_path / "systemctl.log" + systemctl = fake_bin / "systemctl" + systemctl.write_text( + """#!/usr/bin/env bash +printf '%s\n' "$*" >>"${SYSTEMCTL_LOG:?}" +if [ "${1:-}" = "is-active" ]; then + printf '%s\n' "${FAKE_TIMER_STATE:-not-found}" + [ "${FAKE_TIMER_STATE:-not-found}" = "active" ] +fi +""", + encoding="utf-8", + ) + systemctl.chmod(0o755) + timeout = fake_bin / "timeout" + timeout.write_text( + "#!/usr/bin/env bash\nshift\n\"$@\"\n", + encoding="utf-8", + ) + timeout.chmod(0o755) + env = os.environ | { + "PATH": f"{fake_bin}:{os.environ['PATH']}", + "SYSTEMCTL_LOG": str(log), + } + + clean = subprocess.run( + ["bash", str(runner)], + text=True, + capture_output=True, + env=env | {"FAKE_TIMER_STATE": "not-found"}, + check=False, + timeout=10, + ) + assert clean.returncode == 0, clean.stdout + clean.stderr + assert "stop awoooi-host112-guest-recovery.timer" not in log.read_text( + encoding="utf-8" + ) + + log.write_text("", encoding="utf-8") + active = subprocess.run( + ["bash", str(runner)], + text=True, + capture_output=True, + env=env | {"FAKE_TIMER_STATE": "active"}, + check=False, + timeout=10, + ) + assert active.returncode == 0, active.stdout + active.stderr + assert "stop awoooi-host112-guest-recovery.timer" in log.read_text( + encoding="utf-8" + ) diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 97d814440..c06dbcbb8 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -88,6 +88,22 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> for field in ["console_ready", "services_ready", "recovery_timer_ready", "guest_ready"]: assert field in host_probe + canonical_host112_vmx = r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" + assert canonical_host112_vmx in windows99 + assert r"D:\Downloads\kali-linux-2025.4-vmware-amd64" not in windows99 + assert r'"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"' in read( + "agent99.config.99.example.json" + ) + assert canonical_host112_vmx in read( + "apps/api/src/services/reboot_auto_recovery_slo_scorecard.py" + ) + for locator in ( + "scripts/reboot-recovery/windows99-vmx-source-locator.ps1", + "scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1", + "scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1", + ): + assert r"S:\VMs" in read(locator) + host112_readiness = read("scripts/reboot-recovery/host112-guest-readiness.sh") host112_installer = read("scripts/reboot-recovery/install-host112-guest-recovery.sh") host112_service = read( diff --git a/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 index af0a4d12d..c828441f9 100644 --- a/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 @@ -1,6 +1,6 @@ param( [string[]]$VmNameAllowList = @("110", "188", "120", "121", "112"), - [string[]]$SearchRoots = @("C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"), + [string[]]$SearchRoots = @("S:\VMs", "C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"), [string]$TaskName = "AWOOOI VMware VM AutoStart", [switch]$WhatIfOnly ) diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index a876d6cdf..c58722b4c 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -12,6 +12,7 @@ param( [string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112"), [switch]$EnableRecursiveDiscovery, [string[]]$DiscoveryRoot = @( + "S:\VMs", "D:\Documents\Virtual Machines", "D:\Downloads", "D:\VMs", @@ -41,7 +42,7 @@ $KnownVmxCandidates = @{ "188" = @("D:\Documents\Virtual Machines\Ollama_Ubuntu_64-bit\Ollama_Ubuntu_64-bit.vmx") "120" = @("D:\Documents\Virtual Machines\192.168.0.120_Ubuntu_64-bit\192.168.0.120_Ubuntu_64-bit.vmx") "121" = @("D:\Documents\Virtual Machines\192.168.0.121_Ubuntu_64-bit\192.168.0.121_Ubuntu_64-bit.vmx") - "112" = @("D:\Downloads\kali-linux-2025.4-vmware-amd64\kali-linux-2025.4-vmware-amd64.vmwarevm\kali-linux-2025.4-vmware-amd64.vmx") + "112" = @("S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx") } function Resolve-VmxPath { diff --git a/scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1 b/scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1 index fb972ffd1..8344e0478 100644 --- a/scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1 +++ b/scripts/reboot-recovery/windows99-vmx-source-locate-check.ps1 @@ -2,6 +2,7 @@ param( [string]$Alias = "111", [string]$ExpectedVmxPath = "D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx", [string[]]$DiscoveryRoot = @( + "S:\VMs", "D:\Documents\Virtual Machines", "D:\Downloads", "D:\VMs", diff --git a/scripts/reboot-recovery/windows99-vmx-source-locator.ps1 b/scripts/reboot-recovery/windows99-vmx-source-locator.ps1 index 0cb4fa6e7..7f2d36909 100644 --- a/scripts/reboot-recovery/windows99-vmx-source-locator.ps1 +++ b/scripts/reboot-recovery/windows99-vmx-source-locator.ps1 @@ -6,6 +6,7 @@ param( "D:\Documents\Virtual Machines\192.168.0.111_Ubuntu_64-bit\192.168.0.111_Ubuntu_64-bit.vmx" ), [string[]]$DiscoveryRoot = @( + "S:\VMs", "D:\Documents\Virtual Machines", "D:\Downloads", "D:\VMs",