From 152fcbca050bc1b4445a816e277d65f4e961a6f7 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 17:16:38 +0800 Subject: [PATCH] fix(recovery): preserve reboot evidence across delayed startup --- agent99-contract-check.ps1 | 3 + agent99-control-plane.ps1 | 101 ++++- agent99-register-tasks.ps1 | 8 +- agent99-synthetic-tests.ps1 | 10 +- ...ternal-l0-recovery-supervisor.example.json | 57 +++ ...oi-external-l0-recovery-supervisor.service | 21 ++ ...oooi-external-l0-recovery-supervisor.timer | 12 + .../external-l0-recovery-supervisor.py | 344 ++++++++++++++++++ .../full-host-reboot-drill-observer.sh | 17 +- ...t_agent99_recovery_coordinator_contract.py | 20 +- .../test_external_l0_recovery_supervisor.py | 105 ++++++ ...ull_host_reboot_drill_observer_contract.py | 4 + 12 files changed, 688 insertions(+), 14 deletions(-) create mode 100644 ops/reboot-recovery/external-l0-recovery-supervisor.example.json create mode 100644 scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service create mode 100644 scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer create mode 100644 scripts/reboot-recovery/external-l0-recovery-supervisor.py create mode 100644 scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index d63732ea6..e37427de6 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -69,12 +69,15 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane. $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8) $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8) $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8) +$taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8) Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced" Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present" Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded" Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized" Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue" +Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal" +Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence" Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present" Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard" Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present" diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 1b58a9991..2223b433b 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -1216,16 +1216,40 @@ function Get-AgentBootState { return [pscustomobject]@{ ok = $false; detected = $false; reason = "boot_time_unavailable"; statePath = $statePath } } + $observedAt = (Get-Date).ToString("o") $previous = $null if (Test-Path $statePath) { try { $previous = Get-Content $statePath -Raw | ConvertFrom-Json } catch { $previous = $null } } - $detected = [bool]($previous -and $previous.bootTime -and ([string]$previous.bootTime -ne $bootTime)) + $previousBootTime = if ($previous -and $previous.PSObject.Properties["bootTime"]) { [string]$previous.bootTime } else { $null } + $detected = [bool]($previousBootTime -and $previousBootTime -ne $bootTime) + $sameBoot = [bool]($previousBootTime -and $previousBootTime -eq $bootTime) + $previousPending = [bool]( + $sameBoot -and + $previous.PSObject.Properties["pendingRecovery"] -and + (Convert-AgentBool $previous.pendingRecovery) + ) + $pendingRecovery = [bool]($detected -or $previousPending) + $eventId = if ($detected) { + "windows99-boot-" + ($bootTime -replace '[^0-9]', '') + } elseif ($sameBoot -and $previous.PSObject.Properties["eventId"]) { + [string]$previous.eventId + } else { "" } $result = [pscustomobject]@{ ok = $true detected = $detected + pendingRecovery = $pendingRecovery + eventId = $eventId + detectedAt = if ($detected) { $observedAt } elseif ($sameBoot -and $previous.PSObject.Properties["detectedAt"]) { [string]$previous.detectedAt } else { "" } bootTime = $bootTime - previousBootTime = if ($previous) { [string]$previous.bootTime } else { $null } + previousBootTime = $previousBootTime + terminalState = if ($sameBoot -and $previous.PSObject.Properties["terminalState"]) { [string]$previous.terminalState } elseif ($pendingRecovery) { "pending" } else { "observed" } + terminalAt = if ($sameBoot -and $previous.PSObject.Properties["terminalAt"]) { [string]$previous.terminalAt } else { "" } + lastRecoveryRunId = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryRunId"]) { [string]$previous.lastRecoveryRunId } else { "" } + lastRecoveryCompletedAt = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompletedAt"]) { [string]$previous.lastRecoveryCompletedAt } else { "" } + lastRecoveryCompleted = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompleted"] -and (Convert-AgentBool $previous.lastRecoveryCompleted)) + lastRecoveryWithinSlo = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryWithinSlo"] -and (Convert-AgentBool $previous.lastRecoveryWithinSlo)) + lastObservedAt = $observedAt statePath = $statePath } $result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8 @@ -1235,6 +1259,40 @@ function Get-AgentBootState { $result } +function Update-AgentBootRecoveryState { + param([object]$RecoverySlo) + + $statePath = Join-Path (Join-Path $Config.agentRoot "state") "boot-state.json" + if (-not $RecoverySlo -or -not (Test-Path $statePath)) { return $null } + try { + $state = Get-Content $statePath -Raw | ConvertFrom-Json + if (-not $state.PSObject.Properties["pendingRecovery"] -or -not (Convert-AgentBool $state.pendingRecovery)) { + return $state + } + $state | Add-Member -NotePropertyName lastRecoveryRunId -NotePropertyValue ([string]$RecoverySlo.runId) -Force + $state | Add-Member -NotePropertyName lastRecoveryCompletedAt -NotePropertyValue ([string]$RecoverySlo.completedAt) -Force + $state | Add-Member -NotePropertyName lastRecoveryCompleted -NotePropertyValue ([bool]$RecoverySlo.completed) -Force + $state | Add-Member -NotePropertyName lastRecoveryWithinSlo -NotePropertyValue ([bool]$RecoverySlo.withinSlo) -Force + if ($RecoverySlo.completed) { + $state | Add-Member -NotePropertyName pendingRecovery -NotePropertyValue $false -Force + $terminalState = if ($RecoverySlo.withinSlo) { "recovered_within_slo" } else { "recovered_late" } + $state | Add-Member -NotePropertyName terminalState -NotePropertyValue $terminalState -Force + $state | Add-Member -NotePropertyName terminalAt -NotePropertyValue ((Get-Date).ToString("o")) -Force + } else { + $bootAgeSeconds = try { [math]::Max(0, [int64]((Get-Date) - ([datetime]$state.bootTime)).TotalSeconds) } catch { -1 } + if ($bootAgeSeconds -gt ([int64]$RecoverySlo.targetMinutes * 60)) { + $state | Add-Member -NotePropertyName terminalState -NotePropertyValue "slo_breached_recovery_pending" -Force + } + } + $state | Add-Member -NotePropertyName lastObservedAt -NotePropertyValue ((Get-Date).ToString("o")) -Force + $state | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8 + return $state + } catch { + Write-AgentLog "boot_recovery_state_write_failed error=$($_.Exception.GetType().Name)" + return $null + } +} + function Get-AgentAutoRecoveryConfig { $configured = if ($Config.PSObject.Properties["autoRecovery"]) { $Config.autoRecovery } else { $null } [pscustomobject]@{ @@ -1546,10 +1604,10 @@ function Convert-AgentRecoveryReadback { $artifactEpoch = & $asInt "RECOVERY_ARTIFACT_EPOCH" -1 $nowEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds() - $startedEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds() + $evidenceNotBeforeEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds() $artifactAgeSeconds = if ($artifactEpoch -gt 0) { [math]::Max(0, $nowEpoch - $artifactEpoch) } else { -1 } $artifactRecent = [bool]($artifactAgeSeconds -ge 0 -and $artifactAgeSeconds -le $CoordinatorConfig.maxArtifactAgeSeconds) - $artifactFreshForRecovery = [bool]($artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)) + $artifactFreshForRecovery = [bool]($artifactEpoch -ge ($evidenceNotBeforeEpoch - $CoordinatorConfig.artifactClockSkewSeconds)) $requiredHostFailures = @() foreach ($alias in @($CoordinatorConfig.requiredHostAliases)) { @@ -1574,9 +1632,19 @@ function Convert-AgentRecoveryReadback { [pscustomobject]@{ name = "scorecard_source_controls_ready"; passed = (& $asBool "SCORECARD_SOURCE_CONTROLS_READY") }, [pscustomobject]@{ name = "scorecard_public_fallback_ready"; passed = (& $asBool "SCORECARD_PUBLIC_FALLBACK_READY") } ) + $sloChecks = @() if ($RequireFreshRebootWindow) { $checks += @( [pscustomobject]@{ name = "artifact_fresh_for_recovery"; passed = $artifactFreshForRecovery }, + [pscustomobject]@{ + name = "scorecard_non_slo_blockers_clear" + passed = [bool]( + ((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) -or + (& $asBool "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") + ) + } + ) + $sloChecks = @( [pscustomobject]@{ name = "fresh_reboot_window_observed"; passed = (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW") }, [pscustomobject]@{ name = "scorecard_can_claim_slo"; passed = (& $asBool "SCORECARD_CAN_CLAIM") }, [pscustomobject]@{ name = "scorecard_has_no_blockers"; passed = [bool]((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) } @@ -1584,12 +1652,13 @@ function Convert-AgentRecoveryReadback { } $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) + $sloFailedChecks = @($sloChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) [pscustomobject]@{ schemaVersion = "agent99_recovery_coordinator_readback_v1" ok = [bool]($failedChecks.Count -eq 0) verified = [bool]($failedChecks.Count -eq 0) requireFreshRebootWindow = $RequireFreshRebootWindow - rebootSloClaimed = [bool]($RequireFreshRebootWindow -and (& $asBool "SCORECARD_CAN_CLAIM") -and (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW")) + rebootSloClaimed = [bool]($RequireFreshRebootWindow -and $sloFailedChecks.Count -eq 0) artifactDir = & $get "RECOVERY_ARTIFACT_DIR" artifactEpoch = $artifactEpoch artifactAgeSeconds = $artifactAgeSeconds @@ -1605,6 +1674,8 @@ function Convert-AgentRecoveryReadback { hosts = $hosts checks = $checks failedChecks = $failedChecks + sloChecks = $sloChecks + sloFailedChecks = $sloFailedChecks } } @@ -1641,7 +1712,7 @@ grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true -printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX1NPVVJDRV9DT05UUk9MU19SRUFEWT0iK2Ioci5nZXQoInNvdXJjZV9jb250cm9sc19wcmVzZW50IikpKQpwcmludCgiU0NPUkVDQVJEX1BVQkxJQ19GQUxMQkFDS19SRUFEWT0iK2Ioci5nZXQoInB1YmxpY19tYWludGVuYW5jZV9mYWxsYmFja19ydW50aW1lX3JlYWR5IikpKQ== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true + printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRURfQllfUkVCT09UX1dJTkRPV19PTkxZPSIrYihvLmdldCgiYmxvY2tlZF9ieV9mcmVzaF9yZWJvb3Rfd2luZG93X29ubHkiKSkpCnByaW50KCJTQ09SRUNBUkRfU09VUkNFX0NPTlRST0xTX1JFQURZPSIrYihyLmdldCgic291cmNlX2NvbnRyb2xzX3ByZXNlbnQiKSkpCnByaW50KCJTQ09SRUNBUkRfUFVCTElDX0ZBTExCQUNLX1JFQURZPSIrYihyLmdldCgicHVibGljX21haW50ZW5hbmNlX2ZhbGxiYWNrX3J1bnRpbWVfcmVhZHkiKSkpCg== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true '@ $deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 })) @@ -4319,8 +4390,13 @@ $recoveryConfig = Get-AgentRecoverySloConfig if ($Mode -in @("Status", "Recover")) { $bootState = Get-AgentBootState } -if ($Mode -eq "Status" -and $bootState -and $bootState.detected) { - $recoveryTrigger = Start-AgentRecoveryFromObservation "host_reboot_detected" $bootState +if ( + $Mode -eq "Status" -and + $bootState -and + ($bootState.detected -or ($bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery)) +) { + $triggerReason = if ($bootState.detected) { "host_reboot_detected" } else { "host_reboot_recovery_pending" } + $recoveryTrigger = Start-AgentRecoveryFromObservation $triggerReason $bootState } if ($Mode -in @("Recover", "StartVMs")) { @@ -4394,9 +4470,14 @@ if ($Mode -eq "Recover") { } $requireFreshRebootWindow = [bool]( ($bootState -and $bootState.detected) -or + ($bootState -and $bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery) -or ($windowsBootAgeSeconds -ge 0 -and $windowsBootAgeSeconds -le ($recoveryConfig.targetMinutes * 60)) ) - $recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $recoveryStartedAt $requireFreshRebootWindow + $coordinatorEvidenceNotBefore = $recoveryStartedAt + if ($requireFreshRebootWindow -and $bootState -and $bootState.bootTime) { + try { $coordinatorEvidenceNotBefore = [datetime]$bootState.bootTime } catch { $coordinatorEvidenceNotBefore = $recoveryStartedAt } + } + $recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $coordinatorEvidenceNotBefore $requireFreshRebootWindow $coordinatorStatus = if ($recoveryCoordinator.verified) { "ok" } else { "failed" } $coordinatorEvidence = if ($recoveryCoordinator.PSObject.Properties["artifactDir"]) { [string]$recoveryCoordinator.artifactDir } else { "" } $recoveryPhases += New-AgentRecoveryPhase "full_sop_scorecard" $coordinatorStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $coordinatorEvidence @@ -4479,6 +4560,8 @@ if ($Mode -eq "Recover") { rebootSloClaimed = $rebootSloClaimed phases = @($recoveryPhases) } + $updatedBootState = Update-AgentBootRecoveryState $recoverySlo + if ($updatedBootState) { $bootState = $updatedBootState } $sloSeverity = if ($withinSlo) { "info" } elseif ($recoveryCompleted) { "warning" } else { "critical" } $sloMessage = if ($withinSlo -and $requiresFreshWindow) { "全主機恢復完成，耗時 $elapsedSeconds 秒，fresh reboot scorecard 已證明符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。" diff --git a/agent99-register-tasks.ps1 b/agent99-register-tasks.ps1 index f54a62d15..f78f16fee 100644 --- a/agent99-register-tasks.ps1 +++ b/agent99-register-tasks.ps1 @@ -18,6 +18,12 @@ $settings = New-ScheduledTaskSettingsSet ` -StartWhenAvailable ` -MultipleInstances IgnoreNew ` -ExecutionTimeLimit (New-TimeSpan -Minutes 10) +$recoverySettings = New-ScheduledTaskSettingsSet ` + -AllowStartIfOnBatteries ` + -DontStopIfGoingOnBatteries ` + -StartWhenAvailable ` + -MultipleInstances IgnoreNew ` + -ExecutionTimeLimit (New-TimeSpan -Minutes 20) $serviceSettings = New-ScheduledTaskSettingsSet ` -AllowStartIfOnBatteries ` -DontStopIfGoingOnBatteries ` @@ -34,7 +40,7 @@ Register-ScheduledTask ` -Action $recoverAction ` -Trigger @($startupTrigger, $logonTrigger) ` -Principal $principal ` - -Settings $settings ` + -Settings $recoverySettings ` -Description "Agent99 startup recovery for VMs, Harbor, K3s, and public routes." ` -Force | Out-Null diff --git a/agent99-synthetic-tests.ps1 b/agent99-synthetic-tests.ps1 index 5219ed570..4c23dff99 100644 --- a/agent99-synthetic-tests.ps1 +++ b/agent99-synthetic-tests.ps1 @@ -95,6 +95,7 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { "SCORECARD_PRIMARY_BLOCKER=", "SCORECARD_CAN_CLAIM=1", "SCORECARD_FRESH_REBOOT_WINDOW=1", + "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_SOURCE_CONTROLS_READY=1", "SCORECARD_PUBLIC_FALLBACK_READY=1", "HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown", @@ -110,14 +111,19 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) { $blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3") $blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig - Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 2) "missing SLO claim and active blockers fail closed" + Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green" + + $lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1") + $lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig + Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach" } else { Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found" } $control = $controlSource Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated" -Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" +Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence" +Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure" Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present" $failures = @($checks | Where-Object { -not $_.ok }) diff --git a/ops/reboot-recovery/external-l0-recovery-supervisor.example.json b/ops/reboot-recovery/external-l0-recovery-supervisor.example.json new file mode 100644 index 000000000..98612b965 --- /dev/null +++ b/ops/reboot-recovery/external-l0-recovery-supervisor.example.json @@ -0,0 +1,57 @@ +{ + "schema_version": "external_l0_recovery_supervisor_config_v1", + "callback_allowlist_root": "/usr/local/libexec/awoooi-l0", + "thresholds": { + "outage_consecutive_samples": 2, + "recovery_consecutive_samples": 2, + "minimum_failed_targets": 3, + "minimum_healthy_targets": 4 + }, + "targets": [ + { + "id": "awoooi-api", + "url": "https://awoooi.wooo.work/api/v1/health", + "expected_statuses": [200], + "timeout_seconds": 5 + }, + { + "id": "awoooi-web", + "url": "https://awoooi.wooo.work/", + "expected_statuses": [200, 307, 308], + "timeout_seconds": 5 + }, + { + "id": "gitea", + "url": "https://gitea.wooo.work/", + "expected_statuses": [200], + "timeout_seconds": 5 + }, + { + "id": "stock-freshness", + "url": "https://stock.wooo.work/api/v1/system/freshness", + "expected_statuses": [200], + "timeout_seconds": 5 + } + ], + "required_callback_ids": { + "outage_confirmed": ["telegram_down", "maintenance_activate", "power_restore_99"], + "recovering": ["agent99_recover_enqueue", "telegram_recovering"], + "recovered": ["maintenance_clear", "telegram_recovered", "closure_writeback"] + }, + "callbacks": { + "outage_confirmed": [ + {"id": "telegram_down", "path": "/usr/local/libexec/awoooi-l0/telegram-down", "timeout_seconds": 15}, + {"id": "maintenance_activate", "path": "/usr/local/libexec/awoooi-l0/maintenance-activate", "timeout_seconds": 30}, + {"id": "power_restore_99", "path": "/usr/local/libexec/awoooi-l0/power-restore-99", "timeout_seconds": 30} + ], + "recovering": [ + {"id": "agent99_recover_enqueue", "path": "/usr/local/libexec/awoooi-l0/agent99-recover-enqueue", "timeout_seconds": 30}, + {"id": "telegram_recovering", "path": "/usr/local/libexec/awoooi-l0/telegram-recovering", "timeout_seconds": 15} + ], + "recovered": [ + {"id": "maintenance_clear", "path": "/usr/local/libexec/awoooi-l0/maintenance-clear", "timeout_seconds": 30}, + {"id": "telegram_recovered", "path": "/usr/local/libexec/awoooi-l0/telegram-recovered", "timeout_seconds": 15}, + {"id": "closure_writeback", "path": "/usr/local/libexec/awoooi-l0/closure-writeback", "timeout_seconds": 30} + ] + } +} diff --git a/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service new file mode 100644 index 000000000..fd8535e84 --- /dev/null +++ b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service @@ -0,0 +1,21 @@ +[Unit] +Description=AWOOOI external L0 outage and recovery supervisor +Documentation=file:/opt/awoooi/reboot-recovery/docs/runbooks/REBOOT-RECOVERY-SOP.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +User=awoooi-l0 +Group=awoooi-l0 +WorkingDirectory=/opt/awoooi/reboot-recovery +ExecStart=/usr/bin/python3 /opt/awoooi/reboot-recovery/scripts/reboot-recovery/external-l0-recovery-supervisor.py --apply --config /etc/awoooi/external-l0-recovery-supervisor.json --state-file /var/lib/awoooi/external-l0/state.json --artifact-dir /var/lib/awoooi/external-l0/evidence +NoNewPrivileges=true +PrivateTmp=true +ProtectHome=true +ProtectSystem=strict +ReadWritePaths=/var/lib/awoooi/external-l0 +TimeoutStartSec=55 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer new file mode 100644 index 000000000..6a221b761 --- /dev/null +++ b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Probe AWOOOI external L0 recovery state every 15 seconds + +[Timer] +OnBootSec=15s +OnUnitInactiveSec=15s +AccuracySec=2s +Persistent=true +Unit=awoooi-external-l0-recovery-supervisor.service + +[Install] +WantedBy=timers.target diff --git a/scripts/reboot-recovery/external-l0-recovery-supervisor.py b/scripts/reboot-recovery/external-l0-recovery-supervisor.py new file mode 100644 index 000000000..d0e68e853 --- /dev/null +++ b/scripts/reboot-recovery/external-l0-recovery-supervisor.py @@ -0,0 +1,344 @@ +#!/usr/bin/env python3 +"""Off-LAN outage state machine for the Windows 99 recovery dependency chain.""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +import urllib.error +import urllib.request +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + + +SCHEMA_VERSION = "external_l0_recovery_supervisor_v1" +STATE_SCHEMA_VERSION = "external_l0_recovery_state_v1" +VALID_LIFECYCLE_STATES = { + "healthy", + "suspected_outage", + "outage_confirmed", + "recovering", + "recovered", +} + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--config", required=True, type=Path) + parser.add_argument("--state-file", required=True, type=Path) + parser.add_argument("--artifact-dir", required=True, type=Path) + parser.add_argument("--samples-json", type=Path) + parser.add_argument("--now") + mode = parser.add_mutually_exclusive_group(required=True) + mode.add_argument("--check", action="store_true") + mode.add_argument("--apply", action="store_true") + return parser.parse_args() + + +def now_utc(value: str | None) -> datetime: + if not value: + return datetime.now(timezone.utc) + parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.astimezone(timezone.utc) + + +def isoformat(value: datetime) -> str: + return value.replace(microsecond=0).isoformat().replace("+00:00", "Z") + + +def read_json(path: Path) -> dict[str, Any]: + payload = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(payload, dict): + raise ValueError(f"JSON root must be an object: {path}") + return payload + + +def write_json(path: Path, payload: dict[str, Any]) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + temporary = path.with_suffix(path.suffix + ".tmp") + temporary.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") + os.replace(temporary, path) + + +def validate_config(config: dict[str, Any]) -> None: + if config.get("schema_version") != "external_l0_recovery_supervisor_config_v1": + raise ValueError("unsupported config schema_version") + targets = config.get("targets") + if not isinstance(targets, list) or len(targets) < 2: + raise ValueError("at least two external targets are required") + target_ids: set[str] = set() + for target in targets: + if not isinstance(target, dict): + raise ValueError("each target must be an object") + target_id = str(target.get("id") or "") + url = str(target.get("url") or "") + if not target_id or target_id in target_ids: + raise ValueError("target ids must be present and unique") + if not url.startswith("https://"): + raise ValueError("external targets must use https") + target_ids.add(target_id) + + thresholds = config.get("thresholds") or {} + for key in ( + "outage_consecutive_samples", + "recovery_consecutive_samples", + "minimum_failed_targets", + "minimum_healthy_targets", + ): + value = thresholds.get(key) + if not isinstance(value, int) or value < 1: + raise ValueError(f"threshold must be a positive integer: {key}") + if thresholds["minimum_failed_targets"] > len(targets): + raise ValueError("minimum_failed_targets exceeds target count") + if thresholds["minimum_healthy_targets"] > len(targets): + raise ValueError("minimum_healthy_targets exceeds target count") + + allow_root = Path(str(config.get("callback_allowlist_root") or "")) + if not allow_root.is_absolute(): + raise ValueError("callback_allowlist_root must be absolute") + callbacks = config.get("callbacks") or {} + if not isinstance(callbacks, dict): + raise ValueError("callbacks must be an object") + for lifecycle, actions in callbacks.items(): + if lifecycle not in VALID_LIFECYCLE_STATES: + raise ValueError(f"unsupported callback lifecycle: {lifecycle}") + if not isinstance(actions, list): + raise ValueError("callback lifecycle entries must be arrays") + for action in actions: + if not isinstance(action, dict) or not action.get("id") or not action.get("path"): + raise ValueError("callbacks require id and path") + action_path = Path(str(action["path"])) + if not action_path.is_absolute() or not action_path.is_relative_to(allow_root): + raise ValueError(f"callback path outside allowlist root: {action_path}") + + +def default_state(observed_at: str) -> dict[str, Any]: + return { + "schema_version": STATE_SCHEMA_VERSION, + "lifecycle_state": "healthy", + "incident_id": None, + "failure_streak": 0, + "success_streak": 0, + "first_failure_at": None, + "last_transition_at": observed_at, + "last_observed_at": observed_at, + } + + +def load_state(path: Path, observed_at: str) -> dict[str, Any]: + if not path.exists(): + return default_state(observed_at) + state = read_json(path) + if state.get("schema_version") != STATE_SCHEMA_VERSION: + raise ValueError("unsupported state schema_version") + if state.get("lifecycle_state") not in VALID_LIFECYCLE_STATES: + raise ValueError("invalid lifecycle state") + return state + + +def probe_target(target: dict[str, Any]) -> dict[str, Any]: + expected = {int(value) for value in target.get("expected_statuses", [200])} + timeout_seconds = float(target.get("timeout_seconds", 5)) + request = urllib.request.Request( + str(target["url"]), + headers={"User-Agent": "AWOOOI-External-L0-Recovery-Supervisor/1.0"}, + ) + status = 0 + error_class = "" + try: + with urllib.request.urlopen(request, timeout=timeout_seconds) as response: + status = int(response.status) + except urllib.error.HTTPError as exc: + status = int(exc.code) + error_class = "http_error" + except urllib.error.URLError: + error_class = "url_error" + except TimeoutError: + error_class = "timeout" + return { + "id": str(target["id"]), + "url": str(target["url"]), + "status": status, + "healthy": status in expected, + "error_class": error_class, + "response_body_read": False, + } + + +def collect_results(config: dict[str, Any], samples_json: Path | None) -> list[dict[str, Any]]: + if samples_json: + payload = read_json(samples_json) + results = payload.get("target_results") + if not isinstance(results, list): + raise ValueError("samples-json requires target_results array") + configured_ids = {str(target["id"]) for target in config["targets"]} + result_ids = {str(result.get("id") or "") for result in results if isinstance(result, dict)} + if configured_ids != result_ids: + raise ValueError("samples-json target ids do not match config") + return [dict(result) for result in results] + return [probe_target(target) for target in config["targets"]] + + +def advance_state( + state: dict[str, Any], + results: list[dict[str, Any]], + thresholds: dict[str, int], + observed_at: str, +) -> tuple[dict[str, Any], str | None, dict[str, Any]]: + previous = str(state["lifecycle_state"]) + failed = [str(result["id"]) for result in results if not bool(result.get("healthy"))] + healthy = [str(result["id"]) for result in results if bool(result.get("healthy"))] + outage_sample = len(failed) >= thresholds["minimum_failed_targets"] + recovery_sample = len(healthy) >= thresholds["minimum_healthy_targets"] + transition: str | None = None + + if previous in {"healthy", "recovered", "suspected_outage"}: + if outage_sample: + state["failure_streak"] = int(state.get("failure_streak", 0)) + 1 + state["success_streak"] = 0 + state["first_failure_at"] = state.get("first_failure_at") or observed_at + desired = "outage_confirmed" if state["failure_streak"] >= thresholds["outage_consecutive_samples"] else "suspected_outage" + else: + state["failure_streak"] = 0 + state["success_streak"] = 0 + state["first_failure_at"] = None + desired = "healthy" if previous != "recovered" else "recovered" + else: + if recovery_sample: + state["success_streak"] = int(state.get("success_streak", 0)) + 1 + state["failure_streak"] = 0 + desired = "recovered" if state["success_streak"] >= thresholds["recovery_consecutive_samples"] else "recovering" + else: + state["success_streak"] = 0 + desired = "outage_confirmed" + + if desired != previous: + transition = desired + state["lifecycle_state"] = desired + state["last_transition_at"] = observed_at + if desired == "outage_confirmed" and not state.get("incident_id"): + compact = observed_at.replace("-", "").replace(":", "").replace("T", "-").replace("Z", "") + state["incident_id"] = f"INC-L0-{compact}" + if desired == "healthy": + state["incident_id"] = None + state["last_observed_at"] = observed_at + sample = { + "outage_candidate": outage_sample, + "recovery_candidate": recovery_sample, + "failed_target_ids": failed, + "healthy_target_ids": healthy, + } + return state, transition, sample + + +def validate_required_callbacks(config: dict[str, Any], lifecycle: str) -> list[str]: + configured = {str(item["id"]) for item in (config.get("callbacks") or {}).get(lifecycle, [])} + required = {str(item) for item in (config.get("required_callback_ids") or {}).get(lifecycle, [])} + return sorted(required - configured) + + +def run_callbacks( + config: dict[str, Any], + lifecycle: str, + event_file: Path, + apply: bool, +) -> tuple[list[dict[str, Any]], list[str]]: + receipts: list[dict[str, Any]] = [] + blockers: list[str] = [] + missing = validate_required_callbacks(config, lifecycle) + blockers.extend(f"required_callback_missing:{callback_id}" for callback_id in missing) + for action in (config.get("callbacks") or {}).get(lifecycle, []): + action_id = str(action["id"]) + action_path = Path(str(action["path"])) + if not action_path.is_file() or not os.access(action_path, os.X_OK): + blockers.append(f"callback_not_executable:{action_id}") + receipts.append({"id": action_id, "status": "blocked_not_executable", "path": str(action_path)}) + continue + if not apply: + receipts.append({"id": action_id, "status": "check_only", "path": str(action_path)}) + continue + try: + completed = subprocess.run( + [str(action_path), "--event-file", str(event_file)], + stdin=subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=int(action.get("timeout_seconds", 30)), + check=False, + ) + status = "executed" if completed.returncode == 0 else "failed" + receipts.append({"id": action_id, "status": status, "exit_code": completed.returncode}) + if completed.returncode != 0: + blockers.append(f"callback_failed:{action_id}") + except subprocess.TimeoutExpired: + receipts.append({"id": action_id, "status": "timeout"}) + blockers.append(f"callback_timeout:{action_id}") + return receipts, blockers + + +def main() -> int: + args = parse_args() + observed = now_utc(args.now) + observed_at = isoformat(observed) + config = read_json(args.config) + validate_config(config) + state = load_state(args.state_file, observed_at) + previous_state = str(state["lifecycle_state"]) + results = collect_results(config, args.samples_json) + thresholds = {key: int(value) for key, value in config["thresholds"].items()} + state, transition, sample = advance_state(state, results, thresholds, observed_at) + run_id = f"external-l0-{observed.strftime('%Y%m%dT%H%M%SZ')}" + artifact_file = args.artifact_dir / f"{run_id}.json" + event: dict[str, Any] = { + "schema_version": SCHEMA_VERSION, + "run_id": run_id, + "observed_at": observed_at, + "mode": "apply" if args.apply else "check", + "previous_state": previous_state, + "lifecycle_state": state["lifecycle_state"], + "transition": transition, + "incident_id": state.get("incident_id"), + "sample": sample, + "target_results": results, + "callback_receipts": [], + "blockers": [], + "secret_value_read": False, + "arbitrary_shell_allowed": False, + "remote_write_performed": False, + } + write_json(artifact_file, event) + callback_receipts: list[dict[str, Any]] = [] + blockers: list[str] = [] + if transition: + callback_receipts, blockers = run_callbacks(config, transition, artifact_file, args.apply) + event["callback_receipts"] = callback_receipts + event["blockers"] = blockers + event["runtime_ready"] = not blockers + event["remote_write_performed"] = args.apply and any( + receipt.get("status") == "executed" for receipt in callback_receipts + ) + write_json(artifact_file, event) + write_json(args.state_file, state) + print(json.dumps({ + "run_id": run_id, + "state": state["lifecycle_state"], + "transition": transition, + "incident_id": state.get("incident_id"), + "runtime_ready": event["runtime_ready"], + "artifact": str(artifact_file), + }, ensure_ascii=True)) + return 2 if args.apply and blockers else 0 + + +if __name__ == "__main__": + try: + raise SystemExit(main()) + except (OSError, ValueError, json.JSONDecodeError) as exc: + print(f"external_l0_supervisor_error={exc}", file=sys.stderr) + raise SystemExit(2) diff --git a/scripts/reboot-recovery/full-host-reboot-drill-observer.sh b/scripts/reboot-recovery/full-host-reboot-drill-observer.sh index 7113d9043..5d7ae1ae7 100755 --- a/scripts/reboot-recovery/full-host-reboot-drill-observer.sh +++ b/scripts/reboot-recovery/full-host-reboot-drill-observer.sh @@ -4,6 +4,7 @@ set -u DURATION_SECONDS="${DURATION_SECONDS:-900}" INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}" CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-3}" +TCP_PROBE_PYTHON="${TCP_PROBE_PYTHON:-python3}" RUN_ID="${RUN_ID:-$(date -u '+%Y%m%dT%H%M%SZ')}" ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-full-host-reboot-drill/${RUN_ID}}" @@ -65,6 +66,11 @@ if ! is_positive_integer "$DURATION_SECONDS" || ! is_positive_integer "$INTERVAL exit 64 fi +if ! command -v "$TCP_PROBE_PYTHON" >/dev/null 2>&1; then + printf 'error=tcp_probe_python_not_found:%s\n' "$TCP_PROBE_PYTHON" >&2 + exit 69 +fi + mkdir -p "$ARTIFACT_DIR/state" SAMPLES_TSV="$ARTIFACT_DIR/samples.tsv" EVENTS_TSV="$ARTIFACT_DIR/events.tsv" @@ -78,7 +84,16 @@ sanitize_name() { probe_host() { local observed_at="$1" alias="$2" address="$3" port="$4" output="$5" - if nc -z -w "$CONNECT_TIMEOUT_SECONDS" "$address" "$port" >/dev/null 2>&1; then + if "$TCP_PROBE_PYTHON" - "$address" "$port" "$CONNECT_TIMEOUT_SECONDS" >/dev/null 2>&1 <<'PY'; then +import socket +import sys + +address = sys.argv[1] +port = int(sys.argv[2]) +timeout_seconds = float(sys.argv[3]) +with socket.create_connection((address, port), timeout=timeout_seconds): + pass +PY printf '%s\thost\t%s\tup\ttcp_%s_open\n' "$observed_at" "$alias" "$port" >"$output" else printf '%s\thost\t%s\tdown\ttcp_%s_closed_or_unreachable\n' "$observed_at" "$alias" "$port" >"$output" diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index 6b071fcc0..8ef748562 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -50,10 +50,28 @@ def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None: assert 'name = "fresh_reboot_window_observed"' in control assert 'name = "scorecard_can_claim_slo"' in control assert 'name = "scorecard_has_no_blockers"' in control - assert "$artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control + assert 'name = "scorecard_non_slo_blockers_clear"' in control + assert "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY" in control + assert "$artifactEpoch -ge ($evidenceNotBeforeEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control assert "-not $requiresFreshWindow -or $rebootSloClaimed" in control +def test_reboot_event_survives_a_killed_first_recovery_and_can_close_late() -> None: + control = read("agent99-control-plane.ps1") + tasks = read("agent99-register-tasks.ps1") + + assert "pendingRecovery = $pendingRecovery" in control + assert 'host_reboot_recovery_pending' in control + assert "Update-AgentBootRecoveryState" in control + assert '"slo_breached_recovery_pending"' in control + assert '"recovered_late"' in control + assert "$bootState.pendingRecovery" in control + assert "$coordinatorEvidenceNotBefore = [datetime]$bootState.bootTime" in control + assert "$recoverySettings" in tasks + assert "New-TimeSpan -Minutes 20" in tasks + assert '-Settings $recoverySettings' in tasks + + def test_runtime_deployer_migrates_recovery_coordinator_defaults() -> None: deployer = read("agent99-deploy.ps1") config = read("agent99.config.99.example.json") diff --git a/scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py b/scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py new file mode 100644 index 000000000..a8d76f078 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py @@ -0,0 +1,105 @@ +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "reboot-recovery" / "external-l0-recovery-supervisor.py" +EXAMPLE_CONFIG = ROOT / "ops" / "reboot-recovery" / "external-l0-recovery-supervisor.example.json" + + +def write_samples(path: Path, healthy: bool) -> None: + ids = ("awoooi-api", "awoooi-web", "gitea", "stock-freshness") + path.write_text( + json.dumps( + { + "target_results": [ + { + "id": target_id, + "url": f"https://{target_id}.example.invalid/", + "status": 200 if healthy else 0, + "healthy": healthy, + "error_class": "" if healthy else "timeout", + "response_body_read": False, + } + for target_id in ids + ] + } + ), + encoding="utf-8", + ) + + +def run_once(tmp_path: Path, samples: Path, observed_at: str) -> dict[str, object]: + completed = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--check", + "--config", + str(EXAMPLE_CONFIG), + "--state-file", + str(tmp_path / "state.json"), + "--artifact-dir", + str(tmp_path / "evidence"), + "--samples-json", + str(samples), + "--now", + observed_at, + ], + check=True, + capture_output=True, + text=True, + ) + return json.loads(completed.stdout) + + +def test_external_l0_state_machine_is_deduplicated_and_recovers(tmp_path: Path) -> None: + failed = tmp_path / "failed.json" + healthy = tmp_path / "healthy.json" + write_samples(failed, healthy=False) + write_samples(healthy, healthy=True) + + first = run_once(tmp_path, failed, "2026-07-11T08:00:00Z") + assert first["state"] == "suspected_outage" + assert first["transition"] == "suspected_outage" + + second = run_once(tmp_path, failed, "2026-07-11T08:00:15Z") + assert second["state"] == "outage_confirmed" + assert second["transition"] == "outage_confirmed" + assert str(second["incident_id"]).startswith("INC-L0-") + assert second["runtime_ready"] is False + + evidence = json.loads(Path(str(second["artifact"])).read_text(encoding="utf-8")) + assert sorted(evidence["blockers"]) == [ + "callback_not_executable:maintenance_activate", + "callback_not_executable:power_restore_99", + "callback_not_executable:telegram_down", + ] + + third = run_once(tmp_path, failed, "2026-07-11T08:00:30Z") + assert third["state"] == "outage_confirmed" + assert third["transition"] is None + + fourth = run_once(tmp_path, healthy, "2026-07-11T08:00:45Z") + assert fourth["state"] == "recovering" + assert fourth["transition"] == "recovering" + + fifth = run_once(tmp_path, healthy, "2026-07-11T08:01:00Z") + assert fifth["state"] == "recovered" + assert fifth["transition"] == "recovered" + + +def test_external_l0_contract_is_off_lan_and_never_reads_response_bodies() -> None: + config = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8")) + text = SCRIPT.read_text(encoding="utf-8") + + assert len(config["targets"]) == 4 + assert all(target["url"].startswith("https://") for target in config["targets"]) + assert "192.168.0." not in EXAMPLE_CONFIG.read_text(encoding="utf-8") + assert "response_body_read" in text + assert "TELEGRAM_BOT_TOKEN" not in text + assert "shell=True" not in text + assert "callback_allowlist_root" in text + assert '"power_restore_99"' in EXAMPLE_CONFIG.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py b/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py index c628e42f7..c6c6265a5 100644 --- a/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py +++ b/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py @@ -26,6 +26,10 @@ def test_observer_is_read_only_and_records_transitions() -> None: assert "probe_host" in text assert "probe_route" in text assert "record_transition" in text + assert "socket.create_connection" in text + assert 'TCP_PROBE_PYTHON="${TCP_PROBE_PYTHON:-python3}"' in text + assert "nc -z" not in text + assert "telnet://" not in text assert "samples.tsv" in text assert "events.tsv" in text assert "secret_value_read=false" in text