diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index f711bdb49..1b58a9991 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1629,7 +1629,7 @@ function Invoke-AgentRecoveryCoordinatorReadback { } $readbackCommand = @' -latest="$(find /home/wooo/reboot-recovery -maxdepth 1 -type d -name 'reboot-auto-recovery-slo-*' -printf '%T@ %p\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)" +latest="$(find /home/wooo/reboot-recovery -mindepth 2 -maxdepth 2 -type f -name 'scorecard.json' -printf '%T@ %h\n' 2>/dev/null | sort -nr | head -n 1 | cut -d' ' -f2-)" if [ -z "$latest" ]; then printf 'RECOVERY_READBACK=0\n' exit 0 @@ -1641,7 +1641,7 @@ grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true -python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); r=d.get("required_checks",{}); o=d.get("rollups",{}); b=lambda v:"1" if v else "0"; print("SCORECARD_STATUS="+str(d.get("status", ""))); print("SCORECARD_BLOCKER_COUNT="+str(o.get("active_blocker_count", -1))); print("SCORECARD_READINESS_PERCENT="+str(o.get("readiness_percent", -1))); print("SCORECARD_PRIMARY_BLOCKER="+str(d.get("primary_blocker", ""))); print("SCORECARD_CAN_CLAIM="+b(d.get("can_claim_all_services_recovered_within_target"))); print("SCORECARD_FRESH_REBOOT_WINDOW="+b(r.get("fresh_reboot_window_observed"))); print("SCORECARD_SOURCE_CONTROLS_READY="+b(r.get("source_controls_present"))); print("SCORECARD_PUBLIC_FALLBACK_READY="+b(r.get("public_maintenance_fallback_runtime_ready")))' "$latest/scorecard.json" 2>/dev/null || true +printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX1NPVVJDRV9DT05UUk9MU19SRUFEWT0iK2Ioci5nZXQoInNvdXJjZV9jb250cm9sc19wcmVzZW50IikpKQpwcmludCgiU0NPUkVDQVJEX1BVQkxJQ19GQUxMQkFDS19SRUFEWT0iK2Ioci5nZXQoInB1YmxpY19tYWludGVuYW5jZV9mYWxsYmFja19ydW50aW1lX3JlYWR5IikpKQ== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true '@ $deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 })) diff --git a/scripts/reboot-recovery/agent99-live-preflight.ps1 b/scripts/reboot-recovery/agent99-live-preflight.ps1 index defdaf3c1..a6a4bf66a 100644 --- a/scripts/reboot-recovery/agent99-live-preflight.ps1 +++ b/scripts/reboot-recovery/agent99-live-preflight.ps1 @@ -144,6 +144,17 @@ function Get-EvidenceReadback { }) } } else { $null } + recoveryCoordinator = if ($rawEvidence.recoveryCoordinator) { + [pscustomobject]@{ + verified = [bool]$rawEvidence.recoveryCoordinator.verified + requireFreshRebootWindow = [bool]$rawEvidence.recoveryCoordinator.requireFreshRebootWindow + rebootSloClaimed = [bool]$rawEvidence.recoveryCoordinator.rebootSloClaimed + scorecardStatus = [string]$rawEvidence.recoveryCoordinator.scorecardStatus + scorecardBlockerCount = [int]$rawEvidence.recoveryCoordinator.scorecardBlockerCount + readinessPercent = [int]$rawEvidence.recoveryCoordinator.readinessPercent + failedChecks = @($rawEvidence.recoveryCoordinator.failedChecks) + } + } else { $null } } } catch { $parseError = $_.Exception.GetType().Name @@ -242,6 +253,8 @@ $evidence = @( $operatingSystem = Get-CimInstance Win32_OperatingSystem $taskFailures = @($tasks | Where-Object { -not $_.healthy }) $requiredEvidenceFailures = @($evidence | Where-Object { $_.required -and -not $_.fresh }) +$selfCheckEvidence = $evidence | Where-Object { $_.name -eq "self_check" } | Select-Object -First 1 +$selfHealthSeverity = if ($selfCheckEvidence -and $selfCheckEvidence.safeSummary) { [string]$selfCheckEvidence.safeSummary.selfHealthSeverity } else { "unknown" } $alertIncoming = $queues | Where-Object { $_.relativePath -eq "alerts\incoming" } | Select-Object -First 1 $alertRunning = $queues | Where-Object { $_.relativePath -eq "alerts\running" } | Select-Object -First 1 $commandQueue = $queues | Where-Object { $_.relativePath -eq "queue" } | Select-Object -First 1 @@ -252,6 +265,7 @@ if (-not $manifest.runtimeMatched -or $manifest.mismatchCount -ne 0) { $blocking if ($taskFailures.Count -gt 0) { $blockingReasons += "scheduled_task_unhealthy" } if ($relayListeners.Count -eq 0) { $blockingReasons += "sre_alert_relay_not_listening" } if ($requiredEvidenceFailures.Count -gt 0) { $blockingReasons += "required_evidence_stale" } +if ($selfHealthSeverity -ne "ok") { $blockingReasons += "self_health_not_ok" } if ($alertRunning.staleOverFiveMinutes -gt 0) { $blockingReasons += "stale_alert_execution" } if ($alertIncoming.count -gt 0 -or $commandQueue.count -gt 0) { $blockingReasons += "pending_work_before_reboot" } @@ -275,6 +289,7 @@ $result = [pscustomobject]@{ taskFailureCount = $taskFailures.Count freshRequiredEvidenceCount = [int]@($evidence | Where-Object { $_.required -and $_.fresh }).Count requiredEvidenceFailureCount = $requiredEvidenceFailures.Count + selfHealthSeverity = $selfHealthSeverity alertIncomingCount = [int]$alertIncoming.count alertRunningCount = [int]$alertRunning.count pendingCommandCount = [int]$commandQueue.count @@ -292,6 +307,7 @@ Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summar Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)" Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)" Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)" +Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)" Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')" Write-Output "SECRET_VALUE_READ=0" Write-Output "REMOTE_WRITE_PERFORMED=0" diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index ce5256d12..6b071fcc0 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -75,6 +75,8 @@ def test_coordinator_reads_only_bounded_summary_artifacts() -> None: assert "public-maintenance-edge-fallback.txt" in coordinator assert "windows99-vmware-verify.txt" in coordinator assert "scorecard.json" in coordinator + assert "base64 -d | python3 -" in coordinator + assert "-type f -name 'scorecard.json'" in coordinator assert "*.log" not in coordinator assert ".env" not in coordinator assert "auth.json" not in coordinator @@ -88,6 +90,8 @@ def test_live_preflight_is_no_secret_and_no_remote_write() -> None: assert 'remoteWritePerformed = $false' in preflight assert 'Write-Output "SECRET_VALUE_READ=0"' in preflight assert 'Write-Output "REMOTE_WRITE_PERFORMED=0"' in preflight + assert 'Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"' in preflight + assert 'if ($selfHealthSeverity -ne "ok")' in preflight assert "runtime-manifest.json" in preflight assert "Get-ScheduledTask" in preflight assert "Get-NetTCPConnection -LocalPort 8787" in preflight