fix(recovery): preserve reboot evidence across delayed startup

2026-07-11 17:16:38 +08:00
parent 10df0e9c95
commit 152fcbca05
12 changed files with 688 additions and 14 deletions
--- a/agent99-contract-check.ps1
+++ b/agent99-contract-check.ps1
@@ -69,12 +69,15 @@ $control = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-control-plane.
 $inbox = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-sre-alert-inbox.ps1"), [Text.Encoding]::UTF8)
 $telegram = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-telegram-inbox.ps1"), [Text.Encoding]::UTF8)
 $bootstrap = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-bootstrap.ps1"), [Text.Encoding]::UTF8)
+$taskRegistration = [IO.File]::ReadAllText((Join-Path $SourceRoot "agent99-register-tasks.ps1"), [Text.Encoding]::UTF8)

 Add-Check "transport:dedicated_identity" ($control.Contains("sshIdentityFile") -and $control.Contains("IdentitiesOnly=yes")) "dedicated SSH identity is enforced"
 Add-Check "transport:jump_first" ($control.Contains("preferJumpHost") -and $control.Contains("ssh_jump_fallback")) "preferred jump with bounded fallback is present"
 Add-Check "transport:stale_process_guard" ($control.Contains("Invoke-AgentStaleSshProcessGuard") -and $control.Contains("taskkill.exe /PID") -and $control.Contains("ssh_stale_process_guard")) "timed-out and stale SSH clients are bounded"
 Add-Check "transport:cross_process_serialization" ($control.Contains("Enter-AgentSshTransportLock") -and $control.Contains("[IO.FileShare]::None") -and $control.Contains("ssh_transport_lock_failed")) "cross-process SSH sessions are serialized"
 Add-Check "recovery:auto_trigger" ($control.Contains("Get-AgentBootState") -and $control.Contains("Start-AgentRecoveryFromObservation") -and $control.Contains("recovery_already_queued")) "boot and host-down recovery use a single-flight queue"
+Add-Check "recovery:durable_boot_event" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("Update-AgentBootRecoveryState") -and $control.Contains("recovered_late")) "boot event remains pending until verified recovery terminal"
+Add-Check "recovery:terminal_evidence_budget" ($taskRegistration.Contains('$recoverySettings') -and $taskRegistration.Contains('New-TimeSpan -Minutes 20')) "startup recovery outlives the ten-minute SLO long enough to write terminal evidence"
 Add-Check "recovery:slo" ($control.Contains("recovery_slo_result") -and $control.Contains("withinSlo")) "10-minute recovery evidence is present"
 Add-Check "recovery:full_sop_coordinator" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_scorecard") -and $control.Contains("SCORECARD_FRESH_REBOOT_WINDOW") -and $control.Contains('scope = $recoveryScope')) "Agent99 cannot close reboot recovery without the 110 full-SOP scorecard"
 Add-Check "outcome:verified" ($control.Contains("Get-AgentOutcomeContract") -and $control.Contains('schemaVersion = "agent99_outcome_contract_v1"')) "verified outcome contract is present"
--- a/agent99-control-plane.ps1
+++ b/agent99-control-plane.ps1
@@ -1216,16 +1216,40 @@ function Get-AgentBootState {
    return [pscustomobject]@{ ok = $false; detected = $false; reason = "boot_time_unavailable"; statePath = $statePath }
  }

+  $observedAt = (Get-Date).ToString("o")
  $previous = $null
  if (Test-Path $statePath) {
    try { $previous = Get-Content $statePath -Raw | ConvertFrom-Json } catch { $previous = $null }
  }
-  $detected = [bool]($previous -and $previous.bootTime -and ([string]$previous.bootTime -ne $bootTime))
+  $previousBootTime = if ($previous -and $previous.PSObject.Properties["bootTime"]) { [string]$previous.bootTime } else { $null }
+  $detected = [bool]($previousBootTime -and $previousBootTime -ne $bootTime)
+  $sameBoot = [bool]($previousBootTime -and $previousBootTime -eq $bootTime)
+  $previousPending = [bool](
+    $sameBoot -and
+    $previous.PSObject.Properties["pendingRecovery"] -and
+    (Convert-AgentBool $previous.pendingRecovery)
+  )
+  $pendingRecovery = [bool]($detected -or $previousPending)
+  $eventId = if ($detected) {
+    "windows99-boot-" + ($bootTime -replace '[^0-9]', '')
+  } elseif ($sameBoot -and $previous.PSObject.Properties["eventId"]) {
+    [string]$previous.eventId
+  } else { "" }
  $result = [pscustomobject]@{
    ok = $true
    detected = $detected
+    pendingRecovery = $pendingRecovery
+    eventId = $eventId
+    detectedAt = if ($detected) { $observedAt } elseif ($sameBoot -and $previous.PSObject.Properties["detectedAt"]) { [string]$previous.detectedAt } else { "" }
    bootTime = $bootTime
-    previousBootTime = if ($previous) { [string]$previous.bootTime } else { $null }
+    previousBootTime = $previousBootTime
+    terminalState = if ($sameBoot -and $previous.PSObject.Properties["terminalState"]) { [string]$previous.terminalState } elseif ($pendingRecovery) { "pending" } else { "observed" }
+    terminalAt = if ($sameBoot -and $previous.PSObject.Properties["terminalAt"]) { [string]$previous.terminalAt } else { "" }
+    lastRecoveryRunId = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryRunId"]) { [string]$previous.lastRecoveryRunId } else { "" }
+    lastRecoveryCompletedAt = if ($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompletedAt"]) { [string]$previous.lastRecoveryCompletedAt } else { "" }
+    lastRecoveryCompleted = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryCompleted"] -and (Convert-AgentBool $previous.lastRecoveryCompleted))
+    lastRecoveryWithinSlo = [bool]($sameBoot -and $previous.PSObject.Properties["lastRecoveryWithinSlo"] -and (Convert-AgentBool $previous.lastRecoveryWithinSlo))
+    lastObservedAt = $observedAt
    statePath = $statePath
  }
  $result | ConvertTo-Json -Depth 5 | Set-Content -Path $statePath -Encoding UTF8
@@ -1235,6 +1259,40 @@ function Get-AgentBootState {
  $result
 }

+function Update-AgentBootRecoveryState {
+  param([object]$RecoverySlo)
+
+  $statePath = Join-Path (Join-Path $Config.agentRoot "state") "boot-state.json"
+  if (-not $RecoverySlo -or -not (Test-Path $statePath)) { return $null }
+  try {
+    $state = Get-Content $statePath -Raw | ConvertFrom-Json
+    if (-not $state.PSObject.Properties["pendingRecovery"] -or -not (Convert-AgentBool $state.pendingRecovery)) {
+      return $state
+    }
+    $state | Add-Member -NotePropertyName lastRecoveryRunId -NotePropertyValue ([string]$RecoverySlo.runId) -Force
+    $state | Add-Member -NotePropertyName lastRecoveryCompletedAt -NotePropertyValue ([string]$RecoverySlo.completedAt) -Force
+    $state | Add-Member -NotePropertyName lastRecoveryCompleted -NotePropertyValue ([bool]$RecoverySlo.completed) -Force
+    $state | Add-Member -NotePropertyName lastRecoveryWithinSlo -NotePropertyValue ([bool]$RecoverySlo.withinSlo) -Force
+    if ($RecoverySlo.completed) {
+      $state | Add-Member -NotePropertyName pendingRecovery -NotePropertyValue $false -Force
+      $terminalState = if ($RecoverySlo.withinSlo) { "recovered_within_slo" } else { "recovered_late" }
+      $state | Add-Member -NotePropertyName terminalState -NotePropertyValue $terminalState -Force
+      $state | Add-Member -NotePropertyName terminalAt -NotePropertyValue ((Get-Date).ToString("o")) -Force
+    } else {
+      $bootAgeSeconds = try { [math]::Max(0, [int64]((Get-Date) - ([datetime]$state.bootTime)).TotalSeconds) } catch { -1 }
+      if ($bootAgeSeconds -gt ([int64]$RecoverySlo.targetMinutes * 60)) {
+        $state | Add-Member -NotePropertyName terminalState -NotePropertyValue "slo_breached_recovery_pending" -Force
+      }
+    }
+    $state | Add-Member -NotePropertyName lastObservedAt -NotePropertyValue ((Get-Date).ToString("o")) -Force
+    $state | ConvertTo-Json -Depth 6 | Set-Content -Path $statePath -Encoding UTF8
+    return $state
+  } catch {
+    Write-AgentLog "boot_recovery_state_write_failed error=$($_.Exception.GetType().Name)"
+    return $null
+  }
+}
+
 function Get-AgentAutoRecoveryConfig {
  $configured = if ($Config.PSObject.Properties["autoRecovery"]) { $Config.autoRecovery } else { $null }
  [pscustomobject]@{
@@ -1546,10 +1604,10 @@ function Convert-AgentRecoveryReadback {

  $artifactEpoch = & $asInt "RECOVERY_ARTIFACT_EPOCH" -1
  $nowEpoch = [DateTimeOffset]::Now.ToUnixTimeSeconds()
-  $startedEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds()
+  $evidenceNotBeforeEpoch = ([DateTimeOffset]$RecoveryStartedAt).ToUnixTimeSeconds()
  $artifactAgeSeconds = if ($artifactEpoch -gt 0) { [math]::Max(0, $nowEpoch - $artifactEpoch) } else { -1 }
  $artifactRecent = [bool]($artifactAgeSeconds -ge 0 -and $artifactAgeSeconds -le $CoordinatorConfig.maxArtifactAgeSeconds)
-  $artifactFreshForRecovery = [bool]($artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds))
+  $artifactFreshForRecovery = [bool]($artifactEpoch -ge ($evidenceNotBeforeEpoch - $CoordinatorConfig.artifactClockSkewSeconds))

  $requiredHostFailures = @()
  foreach ($alias in @($CoordinatorConfig.requiredHostAliases)) {
@@ -1574,9 +1632,19 @@ function Convert-AgentRecoveryReadback {
    [pscustomobject]@{ name = "scorecard_source_controls_ready"; passed = (& $asBool "SCORECARD_SOURCE_CONTROLS_READY") },
    [pscustomobject]@{ name = "scorecard_public_fallback_ready"; passed = (& $asBool "SCORECARD_PUBLIC_FALLBACK_READY") }
  )
+  $sloChecks = @()
  if ($RequireFreshRebootWindow) {
    $checks += @(
      [pscustomobject]@{ name = "artifact_fresh_for_recovery"; passed = $artifactFreshForRecovery },
+      [pscustomobject]@{
+        name = "scorecard_non_slo_blockers_clear"
+        passed = [bool](
+          ((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) -or
+          (& $asBool "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY")
+        )
+      }
+    )
+    $sloChecks = @(
      [pscustomobject]@{ name = "fresh_reboot_window_observed"; passed = (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW") },
      [pscustomobject]@{ name = "scorecard_can_claim_slo"; passed = (& $asBool "SCORECARD_CAN_CLAIM") },
      [pscustomobject]@{ name = "scorecard_has_no_blockers"; passed = [bool]((& $asInt "SCORECARD_BLOCKER_COUNT" -1) -eq 0) }
@@ -1584,12 +1652,13 @@ function Convert-AgentRecoveryReadback {
  }

  $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
+  $sloFailedChecks = @($sloChecks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
  [pscustomobject]@{
    schemaVersion = "agent99_recovery_coordinator_readback_v1"
    ok = [bool]($failedChecks.Count -eq 0)
    verified = [bool]($failedChecks.Count -eq 0)
    requireFreshRebootWindow = $RequireFreshRebootWindow
-    rebootSloClaimed = [bool]($RequireFreshRebootWindow -and (& $asBool "SCORECARD_CAN_CLAIM") -and (& $asBool "SCORECARD_FRESH_REBOOT_WINDOW"))
+    rebootSloClaimed = [bool]($RequireFreshRebootWindow -and $sloFailedChecks.Count -eq 0)
    artifactDir = & $get "RECOVERY_ARTIFACT_DIR"
    artifactEpoch = $artifactEpoch
    artifactAgeSeconds = $artifactAgeSeconds
@@ -1605,6 +1674,8 @@ function Convert-AgentRecoveryReadback {
    hosts = $hosts
    checks = $checks
    failedChecks = $failedChecks
+    sloChecks = $sloChecks
+    sloFailedChecks = $sloFailedChecks
  }
 }

@@ -1641,7 +1712,7 @@ grep -E '^(POST_START_RESULT|POST_START_BLOCKED|SERVICE_GREEN|PRODUCT_DATA_GREEN
 grep -E '^HOST_BOOT ' "$latest/host-probe.txt" 2>/dev/null || true
 grep -E '^(EDGE_FALLBACK_READY|PRIVILEGED_EXECUTOR_READY|EDGE_EXECUTOR_PAYLOAD_PARITY)=' "$latest/public-maintenance-edge-fallback.txt" 2>/dev/null || true
 grep -E '^(VMWARE_AUTOSTART_CONFIG_READY|VMWARE_AUTOSTART_POWER_READY|WINDOWS_UPDATE_NO_AUTO_REBOOT_READY|VMWARE_AUTOSTART_VERIFY_READY)=' "$latest/windows99-vmware-verify.txt" 2>/dev/null || true
-printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX1NPVVJDRV9DT05UUk9MU19SRUFEWT0iK2Ioci5nZXQoInNvdXJjZV9jb250cm9sc19wcmVzZW50IikpKQpwcmludCgiU0NPUkVDQVJEX1BVQkxJQ19GQUxMQkFDS19SRUFEWT0iK2Ioci5nZXQoInB1YmxpY19tYWludGVuYW5jZV9mYWxsYmFja19ydW50aW1lX3JlYWR5IikpKQ== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true
+  printf %s aW1wb3J0IGpzb24sc3lzCmQ9anNvbi5sb2FkKG9wZW4oc3lzLmFyZ3ZbMV0pKQpyPWQuZ2V0KCJyZXF1aXJlZF9jaGVja3MiLHt9KQpvPWQuZ2V0KCJyb2xsdXBzIix7fSkKYj1sYW1iZGEgdjoiMSIgaWYgdiBlbHNlICIwIgpwcmludCgiU0NPUkVDQVJEX1NUQVRVUz0iK3N0cihkLmdldCgic3RhdHVzIiwiIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRVJfQ09VTlQ9IitzdHIoby5nZXQoImFjdGl2ZV9ibG9ja2VyX2NvdW50IiwtMSkpKQpwcmludCgiU0NPUkVDQVJEX1JFQURJTkVTU19QRVJDRU5UPSIrc3RyKG8uZ2V0KCJyZWFkaW5lc3NfcGVyY2VudCIsLTEpKSkKcHJpbnQoIlNDT1JFQ0FSRF9QUklNQVJZX0JMT0NLRVI9IitzdHIoZC5nZXQoInByaW1hcnlfYmxvY2tlciIsIiIpKSkKcHJpbnQoIlNDT1JFQ0FSRF9DQU5fQ0xBSU09IitiKGQuZ2V0KCJjYW5fY2xhaW1fYWxsX3NlcnZpY2VzX3JlY292ZXJlZF93aXRoaW5fdGFyZ2V0IikpKQpwcmludCgiU0NPUkVDQVJEX0ZSRVNIX1JFQk9PVF9XSU5ET1c9IitiKHIuZ2V0KCJmcmVzaF9yZWJvb3Rfd2luZG93X29ic2VydmVkIikpKQpwcmludCgiU0NPUkVDQVJEX0JMT0NLRURfQllfUkVCT09UX1dJTkRPV19PTkxZPSIrYihvLmdldCgiYmxvY2tlZF9ieV9mcmVzaF9yZWJvb3Rfd2luZG93X29ubHkiKSkpCnByaW50KCJTQ09SRUNBUkRfU09VUkNFX0NPTlRST0xTX1JFQURZPSIrYihyLmdldCgic291cmNlX2NvbnRyb2xzX3ByZXNlbnQiKSkpCnByaW50KCJTQ09SRUNBUkRfUFVCTElDX0ZBTExCQUNLX1JFQURZPSIrYihyLmdldCgicHVibGljX21haW50ZW5hbmNlX2ZhbGxiYWNrX3J1bnRpbWVfcmVhZHkiKSkpCg== | base64 -d | python3 - "$latest/scorecard.json" 2>/dev/null || true
 '@

  $deadline = (Get-Date).AddSeconds($(if ($RequireFreshRebootWindow) { $coordinator.freshWaitTimeoutSeconds } else { 1 }))
@@ -4319,8 +4390,13 @@ $recoveryConfig = Get-AgentRecoverySloConfig
 if ($Mode -in @("Status", "Recover")) {
  $bootState = Get-AgentBootState
 }
-if ($Mode -eq "Status" -and $bootState -and $bootState.detected) {
-  $recoveryTrigger = Start-AgentRecoveryFromObservation "host_reboot_detected" $bootState
+if (
+  $Mode -eq "Status" -and
+  $bootState -and
+  ($bootState.detected -or ($bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery))
+) {
+  $triggerReason = if ($bootState.detected) { "host_reboot_detected" } else { "host_reboot_recovery_pending" }
+  $recoveryTrigger = Start-AgentRecoveryFromObservation $triggerReason $bootState
 }

 if ($Mode -in @("Recover", "StartVMs")) {
@@ -4394,9 +4470,14 @@ if ($Mode -eq "Recover") {
  }
  $requireFreshRebootWindow = [bool](
    ($bootState -and $bootState.detected) -or
+    ($bootState -and $bootState.PSObject.Properties["pendingRecovery"] -and $bootState.pendingRecovery) -or
    ($windowsBootAgeSeconds -ge 0 -and $windowsBootAgeSeconds -le ($recoveryConfig.targetMinutes * 60))
  )
-  $recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $recoveryStartedAt $requireFreshRebootWindow
+  $coordinatorEvidenceNotBefore = $recoveryStartedAt
+  if ($requireFreshRebootWindow -and $bootState -and $bootState.bootTime) {
+    try { $coordinatorEvidenceNotBefore = [datetime]$bootState.bootTime } catch { $coordinatorEvidenceNotBefore = $recoveryStartedAt }
+  }
+  $recoveryCoordinator = Invoke-AgentRecoveryCoordinatorReadback $coordinatorEvidenceNotBefore $requireFreshRebootWindow
  $coordinatorStatus = if ($recoveryCoordinator.verified) { "ok" } else { "failed" }
  $coordinatorEvidence = if ($recoveryCoordinator.PSObject.Properties["artifactDir"]) { [string]$recoveryCoordinator.artifactDir } else { "" }
  $recoveryPhases += New-AgentRecoveryPhase "full_sop_scorecard" $coordinatorStatus (((Get-Date) - $recoveryStartedAt).TotalSeconds) $coordinatorEvidence
@@ -4479,6 +4560,8 @@ if ($Mode -eq "Recover") {
    rebootSloClaimed = $rebootSloClaimed
    phases = @($recoveryPhases)
  }
+  $updatedBootState = Update-AgentBootRecoveryState $recoverySlo
+  if ($updatedBootState) { $bootState = $updatedBootState }
  $sloSeverity = if ($withinSlo) { "info" } elseif ($recoveryCompleted) { "warning" } else { "critical" }
  $sloMessage = if ($withinSlo -and $requiresFreshWindow) {
    "全主機恢復完成，耗時 $elapsedSeconds 秒，fresh reboot scorecard 已證明符合 $($recoveryConfig.targetMinutes) 分鐘 SLA。"
--- a/agent99-register-tasks.ps1
+++ b/agent99-register-tasks.ps1
@@ -18,6 +18,12 @@ $settings = New-ScheduledTaskSettingsSet `
  -StartWhenAvailable `
  -MultipleInstances IgnoreNew `
  -ExecutionTimeLimit (New-TimeSpan -Minutes 10)
+$recoverySettings = New-ScheduledTaskSettingsSet `
+  -AllowStartIfOnBatteries `
+  -DontStopIfGoingOnBatteries `
+  -StartWhenAvailable `
+  -MultipleInstances IgnoreNew `
+  -ExecutionTimeLimit (New-TimeSpan -Minutes 20)
 $serviceSettings = New-ScheduledTaskSettingsSet `
  -AllowStartIfOnBatteries `
  -DontStopIfGoingOnBatteries `
@@ -34,7 +40,7 @@ Register-ScheduledTask `
  -Action $recoverAction `
  -Trigger @($startupTrigger, $logonTrigger) `
  -Principal $principal `
-  -Settings $settings `
+  -Settings $recoverySettings `
  -Description "Agent99 startup recovery for VMs, Harbor, K3s, and public routes." `
  -Force | Out-Null

--- a/agent99-synthetic-tests.ps1
+++ b/agent99-synthetic-tests.ps1
@@ -95,6 +95,7 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {
    "SCORECARD_PRIMARY_BLOCKER=",
    "SCORECARD_CAN_CLAIM=1",
    "SCORECARD_FRESH_REBOOT_WINDOW=1",
+    "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0",
    "SCORECARD_SOURCE_CONTROLS_READY=1",
    "SCORECARD_PUBLIC_FALLBACK_READY=1",
    "HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=windows uptime_seconds=30 systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown",
@@ -110,14 +111,19 @@ if ($formatFunctions.ContainsKey("Convert-AgentRecoveryReadback")) {

  $blockedFixture = $fixtureLines.Replace("SCORECARD_CAN_CLAIM=1", "SCORECARD_CAN_CLAIM=0").Replace("SCORECARD_BLOCKER_COUNT=0", "SCORECARD_BLOCKER_COUNT=3")
  $blockedReadback = Convert-AgentRecoveryReadback $blockedFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
-  Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 2) "missing SLO claim and active blockers fail closed"
+  Add-SyntheticCheck "recovery:coordinator_false_green_guard" (-not $blockedReadback.verified -and -not $blockedReadback.rebootSloClaimed -and @($blockedReadback.failedChecks).Count -ge 1) "non-SLO blockers fail closed even when service evidence is otherwise green"
+
+  $lateFixture = $blockedFixture.Replace("SCORECARD_FRESH_REBOOT_WINDOW=1", "SCORECARD_FRESH_REBOOT_WINDOW=0").Replace("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=0", "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY=1")
+  $lateReadback = Convert-AgentRecoveryReadback $lateFixture (Get-Date).AddSeconds(-30) $true $fixtureConfig
+  Add-SyntheticCheck "recovery:coordinator_late_recovery_terminal" ($lateReadback.verified -and -not $lateReadback.rebootSloClaimed -and @($lateReadback.failedChecks).Count -eq 0 -and @($lateReadback.sloFailedChecks).Count -ge 2) "service recovery can close late without rewriting the reboot SLO breach"
 } else {
  Add-SyntheticCheck "recovery:coordinator_parser" $false "Convert-AgentRecoveryReadback not found"
 }

 $control = $controlSource
 Add-SyntheticCheck "recovery:single_flight_queue" ($control.Contains("recovery_already_queued") -and $control.Contains("recovery_trigger_cooldown") -and $control.Contains('mode = "Recover"')) "automatic recovery is queued and deduplicated"
-Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
+Add-SyntheticCheck "recovery:full_sop_scorecard" ($control.Contains("Invoke-AgentRecoveryCoordinatorReadback") -and $control.Contains("full_sop_coordinator_verified") -and $control.Contains("SCORECARD_CAN_CLAIM") -and $control.Contains("SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY") -and $control.Contains("requiredHostAliases")) "full-host recovery is gated by fresh 110 scorecard evidence"
+Add-SyntheticCheck "recovery:durable_boot_terminal" ($control.Contains("pendingRecovery") -and $control.Contains("host_reboot_recovery_pending") -and $control.Contains("recovered_late")) "boot recovery remains durable through first-run termination and late closure"
 Add-SyntheticCheck "transport:identity_jump" ($control.Contains("IdentitiesOnly=yes") -and $control.Contains("preferJumpHost") -and $control.Contains("directHosts")) "dedicated identity and bounded routes are present"

 $failures = @($checks | Where-Object { -not $_.ok })
--- a/ops/reboot-recovery/external-l0-recovery-supervisor.example.json
+++ b/ops/reboot-recovery/external-l0-recovery-supervisor.example.json
@@ -0,0 +1,57 @@
+{
+  "schema_version": "external_l0_recovery_supervisor_config_v1",
+  "callback_allowlist_root": "/usr/local/libexec/awoooi-l0",
+  "thresholds": {
+    "outage_consecutive_samples": 2,
+    "recovery_consecutive_samples": 2,
+    "minimum_failed_targets": 3,
+    "minimum_healthy_targets": 4
+  },
+  "targets": [
+    {
+      "id": "awoooi-api",
+      "url": "https://awoooi.wooo.work/api/v1/health",
+      "expected_statuses": [200],
+      "timeout_seconds": 5
+    },
+    {
+      "id": "awoooi-web",
+      "url": "https://awoooi.wooo.work/",
+      "expected_statuses": [200, 307, 308],
+      "timeout_seconds": 5
+    },
+    {
+      "id": "gitea",
+      "url": "https://gitea.wooo.work/",
+      "expected_statuses": [200],
+      "timeout_seconds": 5
+    },
+    {
+      "id": "stock-freshness",
+      "url": "https://stock.wooo.work/api/v1/system/freshness",
+      "expected_statuses": [200],
+      "timeout_seconds": 5
+    }
+  ],
+  "required_callback_ids": {
+    "outage_confirmed": ["telegram_down", "maintenance_activate", "power_restore_99"],
+    "recovering": ["agent99_recover_enqueue", "telegram_recovering"],
+    "recovered": ["maintenance_clear", "telegram_recovered", "closure_writeback"]
+  },
+  "callbacks": {
+    "outage_confirmed": [
+      {"id": "telegram_down", "path": "/usr/local/libexec/awoooi-l0/telegram-down", "timeout_seconds": 15},
+      {"id": "maintenance_activate", "path": "/usr/local/libexec/awoooi-l0/maintenance-activate", "timeout_seconds": 30},
+      {"id": "power_restore_99", "path": "/usr/local/libexec/awoooi-l0/power-restore-99", "timeout_seconds": 30}
+    ],
+    "recovering": [
+      {"id": "agent99_recover_enqueue", "path": "/usr/local/libexec/awoooi-l0/agent99-recover-enqueue", "timeout_seconds": 30},
+      {"id": "telegram_recovering", "path": "/usr/local/libexec/awoooi-l0/telegram-recovering", "timeout_seconds": 15}
+    ],
+    "recovered": [
+      {"id": "maintenance_clear", "path": "/usr/local/libexec/awoooi-l0/maintenance-clear", "timeout_seconds": 30},
+      {"id": "telegram_recovered", "path": "/usr/local/libexec/awoooi-l0/telegram-recovered", "timeout_seconds": 15},
+      {"id": "closure_writeback", "path": "/usr/local/libexec/awoooi-l0/closure-writeback", "timeout_seconds": 30}
+    ]
+  }
+}
--- a/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service
+++ b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.service
@@ -0,0 +1,21 @@
+[Unit]
+Description=AWOOOI external L0 outage and recovery supervisor
+Documentation=file:/opt/awoooi/reboot-recovery/docs/runbooks/REBOOT-RECOVERY-SOP.md
+Wants=network-online.target
+After=network-online.target
+
+[Service]
+Type=oneshot
+User=awoooi-l0
+Group=awoooi-l0
+WorkingDirectory=/opt/awoooi/reboot-recovery
+ExecStart=/usr/bin/python3 /opt/awoooi/reboot-recovery/scripts/reboot-recovery/external-l0-recovery-supervisor.py --apply --config /etc/awoooi/external-l0-recovery-supervisor.json --state-file /var/lib/awoooi/external-l0/state.json --artifact-dir /var/lib/awoooi/external-l0/evidence
+NoNewPrivileges=true
+PrivateTmp=true
+ProtectHome=true
+ProtectSystem=strict
+ReadWritePaths=/var/lib/awoooi/external-l0
+TimeoutStartSec=55
+
+[Install]
+WantedBy=multi-user.target
--- a/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer
+++ b/scripts/reboot-recovery/awoooi-external-l0-recovery-supervisor.timer
@@ -0,0 +1,12 @@
+[Unit]
+Description=Probe AWOOOI external L0 recovery state every 15 seconds
+
+[Timer]
+OnBootSec=15s
+OnUnitInactiveSec=15s
+AccuracySec=2s
+Persistent=true
+Unit=awoooi-external-l0-recovery-supervisor.service
+
+[Install]
+WantedBy=timers.target
--- a/scripts/reboot-recovery/external-l0-recovery-supervisor.py
+++ b/scripts/reboot-recovery/external-l0-recovery-supervisor.py
@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+"""Off-LAN outage state machine for the Windows 99 recovery dependency chain."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import sys
+import urllib.error
+import urllib.request
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+SCHEMA_VERSION = "external_l0_recovery_supervisor_v1"
+STATE_SCHEMA_VERSION = "external_l0_recovery_state_v1"
+VALID_LIFECYCLE_STATES = {
+    "healthy",
+    "suspected_outage",
+    "outage_confirmed",
+    "recovering",
+    "recovered",
+}
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--config", required=True, type=Path)
+    parser.add_argument("--state-file", required=True, type=Path)
+    parser.add_argument("--artifact-dir", required=True, type=Path)
+    parser.add_argument("--samples-json", type=Path)
+    parser.add_argument("--now")
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--check", action="store_true")
+    mode.add_argument("--apply", action="store_true")
+    return parser.parse_args()
+
+
+def now_utc(value: str | None) -> datetime:
+    if not value:
+        return datetime.now(timezone.utc)
+    parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    return parsed.astimezone(timezone.utc)
+
+
+def isoformat(value: datetime) -> str:
+    return value.replace(microsecond=0).isoformat().replace("+00:00", "Z")
+
+
+def read_json(path: Path) -> dict[str, Any]:
+    payload = json.loads(path.read_text(encoding="utf-8"))
+    if not isinstance(payload, dict):
+        raise ValueError(f"JSON root must be an object: {path}")
+    return payload
+
+
+def write_json(path: Path, payload: dict[str, Any]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = path.with_suffix(path.suffix + ".tmp")
+    temporary.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
+    os.replace(temporary, path)
+
+
+def validate_config(config: dict[str, Any]) -> None:
+    if config.get("schema_version") != "external_l0_recovery_supervisor_config_v1":
+        raise ValueError("unsupported config schema_version")
+    targets = config.get("targets")
+    if not isinstance(targets, list) or len(targets) < 2:
+        raise ValueError("at least two external targets are required")
+    target_ids: set[str] = set()
+    for target in targets:
+        if not isinstance(target, dict):
+            raise ValueError("each target must be an object")
+        target_id = str(target.get("id") or "")
+        url = str(target.get("url") or "")
+        if not target_id or target_id in target_ids:
+            raise ValueError("target ids must be present and unique")
+        if not url.startswith("https://"):
+            raise ValueError("external targets must use https")
+        target_ids.add(target_id)
+
+    thresholds = config.get("thresholds") or {}
+    for key in (
+        "outage_consecutive_samples",
+        "recovery_consecutive_samples",
+        "minimum_failed_targets",
+        "minimum_healthy_targets",
+    ):
+        value = thresholds.get(key)
+        if not isinstance(value, int) or value < 1:
+            raise ValueError(f"threshold must be a positive integer: {key}")
+    if thresholds["minimum_failed_targets"] > len(targets):
+        raise ValueError("minimum_failed_targets exceeds target count")
+    if thresholds["minimum_healthy_targets"] > len(targets):
+        raise ValueError("minimum_healthy_targets exceeds target count")
+
+    allow_root = Path(str(config.get("callback_allowlist_root") or ""))
+    if not allow_root.is_absolute():
+        raise ValueError("callback_allowlist_root must be absolute")
+    callbacks = config.get("callbacks") or {}
+    if not isinstance(callbacks, dict):
+        raise ValueError("callbacks must be an object")
+    for lifecycle, actions in callbacks.items():
+        if lifecycle not in VALID_LIFECYCLE_STATES:
+            raise ValueError(f"unsupported callback lifecycle: {lifecycle}")
+        if not isinstance(actions, list):
+            raise ValueError("callback lifecycle entries must be arrays")
+        for action in actions:
+            if not isinstance(action, dict) or not action.get("id") or not action.get("path"):
+                raise ValueError("callbacks require id and path")
+            action_path = Path(str(action["path"]))
+            if not action_path.is_absolute() or not action_path.is_relative_to(allow_root):
+                raise ValueError(f"callback path outside allowlist root: {action_path}")
+
+
+def default_state(observed_at: str) -> dict[str, Any]:
+    return {
+        "schema_version": STATE_SCHEMA_VERSION,
+        "lifecycle_state": "healthy",
+        "incident_id": None,
+        "failure_streak": 0,
+        "success_streak": 0,
+        "first_failure_at": None,
+        "last_transition_at": observed_at,
+        "last_observed_at": observed_at,
+    }
+
+
+def load_state(path: Path, observed_at: str) -> dict[str, Any]:
+    if not path.exists():
+        return default_state(observed_at)
+    state = read_json(path)
+    if state.get("schema_version") != STATE_SCHEMA_VERSION:
+        raise ValueError("unsupported state schema_version")
+    if state.get("lifecycle_state") not in VALID_LIFECYCLE_STATES:
+        raise ValueError("invalid lifecycle state")
+    return state
+
+
+def probe_target(target: dict[str, Any]) -> dict[str, Any]:
+    expected = {int(value) for value in target.get("expected_statuses", [200])}
+    timeout_seconds = float(target.get("timeout_seconds", 5))
+    request = urllib.request.Request(
+        str(target["url"]),
+        headers={"User-Agent": "AWOOOI-External-L0-Recovery-Supervisor/1.0"},
+    )
+    status = 0
+    error_class = ""
+    try:
+        with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
+            status = int(response.status)
+    except urllib.error.HTTPError as exc:
+        status = int(exc.code)
+        error_class = "http_error"
+    except urllib.error.URLError:
+        error_class = "url_error"
+    except TimeoutError:
+        error_class = "timeout"
+    return {
+        "id": str(target["id"]),
+        "url": str(target["url"]),
+        "status": status,
+        "healthy": status in expected,
+        "error_class": error_class,
+        "response_body_read": False,
+    }
+
+
+def collect_results(config: dict[str, Any], samples_json: Path | None) -> list[dict[str, Any]]:
+    if samples_json:
+        payload = read_json(samples_json)
+        results = payload.get("target_results")
+        if not isinstance(results, list):
+            raise ValueError("samples-json requires target_results array")
+        configured_ids = {str(target["id"]) for target in config["targets"]}
+        result_ids = {str(result.get("id") or "") for result in results if isinstance(result, dict)}
+        if configured_ids != result_ids:
+            raise ValueError("samples-json target ids do not match config")
+        return [dict(result) for result in results]
+    return [probe_target(target) for target in config["targets"]]
+
+
+def advance_state(
+    state: dict[str, Any],
+    results: list[dict[str, Any]],
+    thresholds: dict[str, int],
+    observed_at: str,
+) -> tuple[dict[str, Any], str | None, dict[str, Any]]:
+    previous = str(state["lifecycle_state"])
+    failed = [str(result["id"]) for result in results if not bool(result.get("healthy"))]
+    healthy = [str(result["id"]) for result in results if bool(result.get("healthy"))]
+    outage_sample = len(failed) >= thresholds["minimum_failed_targets"]
+    recovery_sample = len(healthy) >= thresholds["minimum_healthy_targets"]
+    transition: str | None = None
+
+    if previous in {"healthy", "recovered", "suspected_outage"}:
+        if outage_sample:
+            state["failure_streak"] = int(state.get("failure_streak", 0)) + 1
+            state["success_streak"] = 0
+            state["first_failure_at"] = state.get("first_failure_at") or observed_at
+            desired = "outage_confirmed" if state["failure_streak"] >= thresholds["outage_consecutive_samples"] else "suspected_outage"
+        else:
+            state["failure_streak"] = 0
+            state["success_streak"] = 0
+            state["first_failure_at"] = None
+            desired = "healthy" if previous != "recovered" else "recovered"
+    else:
+        if recovery_sample:
+            state["success_streak"] = int(state.get("success_streak", 0)) + 1
+            state["failure_streak"] = 0
+            desired = "recovered" if state["success_streak"] >= thresholds["recovery_consecutive_samples"] else "recovering"
+        else:
+            state["success_streak"] = 0
+            desired = "outage_confirmed"
+
+    if desired != previous:
+        transition = desired
+        state["lifecycle_state"] = desired
+        state["last_transition_at"] = observed_at
+        if desired == "outage_confirmed" and not state.get("incident_id"):
+            compact = observed_at.replace("-", "").replace(":", "").replace("T", "-").replace("Z", "")
+            state["incident_id"] = f"INC-L0-{compact}"
+        if desired == "healthy":
+            state["incident_id"] = None
+    state["last_observed_at"] = observed_at
+    sample = {
+        "outage_candidate": outage_sample,
+        "recovery_candidate": recovery_sample,
+        "failed_target_ids": failed,
+        "healthy_target_ids": healthy,
+    }
+    return state, transition, sample
+
+
+def validate_required_callbacks(config: dict[str, Any], lifecycle: str) -> list[str]:
+    configured = {str(item["id"]) for item in (config.get("callbacks") or {}).get(lifecycle, [])}
+    required = {str(item) for item in (config.get("required_callback_ids") or {}).get(lifecycle, [])}
+    return sorted(required - configured)
+
+
+def run_callbacks(
+    config: dict[str, Any],
+    lifecycle: str,
+    event_file: Path,
+    apply: bool,
+) -> tuple[list[dict[str, Any]], list[str]]:
+    receipts: list[dict[str, Any]] = []
+    blockers: list[str] = []
+    missing = validate_required_callbacks(config, lifecycle)
+    blockers.extend(f"required_callback_missing:{callback_id}" for callback_id in missing)
+    for action in (config.get("callbacks") or {}).get(lifecycle, []):
+        action_id = str(action["id"])
+        action_path = Path(str(action["path"]))
+        if not action_path.is_file() or not os.access(action_path, os.X_OK):
+            blockers.append(f"callback_not_executable:{action_id}")
+            receipts.append({"id": action_id, "status": "blocked_not_executable", "path": str(action_path)})
+            continue
+        if not apply:
+            receipts.append({"id": action_id, "status": "check_only", "path": str(action_path)})
+            continue
+        try:
+            completed = subprocess.run(
+                [str(action_path), "--event-file", str(event_file)],
+                stdin=subprocess.DEVNULL,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+                timeout=int(action.get("timeout_seconds", 30)),
+                check=False,
+            )
+            status = "executed" if completed.returncode == 0 else "failed"
+            receipts.append({"id": action_id, "status": status, "exit_code": completed.returncode})
+            if completed.returncode != 0:
+                blockers.append(f"callback_failed:{action_id}")
+        except subprocess.TimeoutExpired:
+            receipts.append({"id": action_id, "status": "timeout"})
+            blockers.append(f"callback_timeout:{action_id}")
+    return receipts, blockers
+
+
+def main() -> int:
+    args = parse_args()
+    observed = now_utc(args.now)
+    observed_at = isoformat(observed)
+    config = read_json(args.config)
+    validate_config(config)
+    state = load_state(args.state_file, observed_at)
+    previous_state = str(state["lifecycle_state"])
+    results = collect_results(config, args.samples_json)
+    thresholds = {key: int(value) for key, value in config["thresholds"].items()}
+    state, transition, sample = advance_state(state, results, thresholds, observed_at)
+    run_id = f"external-l0-{observed.strftime('%Y%m%dT%H%M%SZ')}"
+    artifact_file = args.artifact_dir / f"{run_id}.json"
+    event: dict[str, Any] = {
+        "schema_version": SCHEMA_VERSION,
+        "run_id": run_id,
+        "observed_at": observed_at,
+        "mode": "apply" if args.apply else "check",
+        "previous_state": previous_state,
+        "lifecycle_state": state["lifecycle_state"],
+        "transition": transition,
+        "incident_id": state.get("incident_id"),
+        "sample": sample,
+        "target_results": results,
+        "callback_receipts": [],
+        "blockers": [],
+        "secret_value_read": False,
+        "arbitrary_shell_allowed": False,
+        "remote_write_performed": False,
+    }
+    write_json(artifact_file, event)
+    callback_receipts: list[dict[str, Any]] = []
+    blockers: list[str] = []
+    if transition:
+        callback_receipts, blockers = run_callbacks(config, transition, artifact_file, args.apply)
+    event["callback_receipts"] = callback_receipts
+    event["blockers"] = blockers
+    event["runtime_ready"] = not blockers
+    event["remote_write_performed"] = args.apply and any(
+        receipt.get("status") == "executed" for receipt in callback_receipts
+    )
+    write_json(artifact_file, event)
+    write_json(args.state_file, state)
+    print(json.dumps({
+        "run_id": run_id,
+        "state": state["lifecycle_state"],
+        "transition": transition,
+        "incident_id": state.get("incident_id"),
+        "runtime_ready": event["runtime_ready"],
+        "artifact": str(artifact_file),
+    }, ensure_ascii=True))
+    return 2 if args.apply and blockers else 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except (OSError, ValueError, json.JSONDecodeError) as exc:
+        print(f"external_l0_supervisor_error={exc}", file=sys.stderr)
+        raise SystemExit(2)
--- a/scripts/reboot-recovery/full-host-reboot-drill-observer.sh
+++ b/scripts/reboot-recovery/full-host-reboot-drill-observer.sh
@@ -4,6 +4,7 @@ set -u
 DURATION_SECONDS="${DURATION_SECONDS:-900}"
 INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}"
 CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-3}"
+TCP_PROBE_PYTHON="${TCP_PROBE_PYTHON:-python3}"
 RUN_ID="${RUN_ID:-$(date -u '+%Y%m%dT%H%M%SZ')}"
 ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-full-host-reboot-drill/${RUN_ID}}"

@@ -65,6 +66,11 @@ if ! is_positive_integer "$DURATION_SECONDS" || ! is_positive_integer "$INTERVAL
  exit 64
 fi

+if ! command -v "$TCP_PROBE_PYTHON" >/dev/null 2>&1; then
+  printf 'error=tcp_probe_python_not_found:%s\n' "$TCP_PROBE_PYTHON" >&2
+  exit 69
+fi
+
 mkdir -p "$ARTIFACT_DIR/state"
 SAMPLES_TSV="$ARTIFACT_DIR/samples.tsv"
 EVENTS_TSV="$ARTIFACT_DIR/events.tsv"
@@ -78,7 +84,16 @@ sanitize_name() {

 probe_host() {
  local observed_at="$1" alias="$2" address="$3" port="$4" output="$5"
-  if nc -z -w "$CONNECT_TIMEOUT_SECONDS" "$address" "$port" >/dev/null 2>&1; then
+  if "$TCP_PROBE_PYTHON" - "$address" "$port" "$CONNECT_TIMEOUT_SECONDS" >/dev/null 2>&1 <<'PY'; then
+import socket
+import sys
+
+address = sys.argv[1]
+port = int(sys.argv[2])
+timeout_seconds = float(sys.argv[3])
+with socket.create_connection((address, port), timeout=timeout_seconds):
+    pass
+PY
    printf '%s\thost\t%s\tup\ttcp_%s_open\n' "$observed_at" "$alias" "$port" >"$output"
  else
    printf '%s\thost\t%s\tdown\ttcp_%s_closed_or_unreachable\n' "$observed_at" "$alias" "$port" >"$output"
--- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
+++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
@@ -50,10 +50,28 @@ def test_fresh_reboot_claim_requires_fresh_artifact_and_zero_blockers() -> None:
    assert 'name = "fresh_reboot_window_observed"' in control
    assert 'name = "scorecard_can_claim_slo"' in control
    assert 'name = "scorecard_has_no_blockers"' in control
-    assert "$artifactEpoch -ge ($startedEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control
+    assert 'name = "scorecard_non_slo_blockers_clear"' in control
+    assert "SCORECARD_BLOCKED_BY_REBOOT_WINDOW_ONLY" in control
+    assert "$artifactEpoch -ge ($evidenceNotBeforeEpoch - $CoordinatorConfig.artifactClockSkewSeconds)" in control
    assert "-not $requiresFreshWindow -or $rebootSloClaimed" in control


+def test_reboot_event_survives_a_killed_first_recovery_and_can_close_late() -> None:
+    control = read("agent99-control-plane.ps1")
+    tasks = read("agent99-register-tasks.ps1")
+
+    assert "pendingRecovery = $pendingRecovery" in control
+    assert 'host_reboot_recovery_pending' in control
+    assert "Update-AgentBootRecoveryState" in control
+    assert '"slo_breached_recovery_pending"' in control
+    assert '"recovered_late"' in control
+    assert "$bootState.pendingRecovery" in control
+    assert "$coordinatorEvidenceNotBefore = [datetime]$bootState.bootTime" in control
+    assert "$recoverySettings" in tasks
+    assert "New-TimeSpan -Minutes 20" in tasks
+    assert '-Settings $recoverySettings' in tasks
+
+
 def test_runtime_deployer_migrates_recovery_coordinator_defaults() -> None:
    deployer = read("agent99-deploy.ps1")
    config = read("agent99.config.99.example.json")
--- a/scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py
+++ b/scripts/reboot-recovery/tests/test_external_l0_recovery_supervisor.py
@@ -0,0 +1,105 @@
+import json
+import subprocess
+import sys
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "reboot-recovery" / "external-l0-recovery-supervisor.py"
+EXAMPLE_CONFIG = ROOT / "ops" / "reboot-recovery" / "external-l0-recovery-supervisor.example.json"
+
+
+def write_samples(path: Path, healthy: bool) -> None:
+    ids = ("awoooi-api", "awoooi-web", "gitea", "stock-freshness")
+    path.write_text(
+        json.dumps(
+            {
+                "target_results": [
+                    {
+                        "id": target_id,
+                        "url": f"https://{target_id}.example.invalid/",
+                        "status": 200 if healthy else 0,
+                        "healthy": healthy,
+                        "error_class": "" if healthy else "timeout",
+                        "response_body_read": False,
+                    }
+                    for target_id in ids
+                ]
+            }
+        ),
+        encoding="utf-8",
+    )
+
+
+def run_once(tmp_path: Path, samples: Path, observed_at: str) -> dict[str, object]:
+    completed = subprocess.run(
+        [
+            sys.executable,
+            str(SCRIPT),
+            "--check",
+            "--config",
+            str(EXAMPLE_CONFIG),
+            "--state-file",
+            str(tmp_path / "state.json"),
+            "--artifact-dir",
+            str(tmp_path / "evidence"),
+            "--samples-json",
+            str(samples),
+            "--now",
+            observed_at,
+        ],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return json.loads(completed.stdout)
+
+
+def test_external_l0_state_machine_is_deduplicated_and_recovers(tmp_path: Path) -> None:
+    failed = tmp_path / "failed.json"
+    healthy = tmp_path / "healthy.json"
+    write_samples(failed, healthy=False)
+    write_samples(healthy, healthy=True)
+
+    first = run_once(tmp_path, failed, "2026-07-11T08:00:00Z")
+    assert first["state"] == "suspected_outage"
+    assert first["transition"] == "suspected_outage"
+
+    second = run_once(tmp_path, failed, "2026-07-11T08:00:15Z")
+    assert second["state"] == "outage_confirmed"
+    assert second["transition"] == "outage_confirmed"
+    assert str(second["incident_id"]).startswith("INC-L0-")
+    assert second["runtime_ready"] is False
+
+    evidence = json.loads(Path(str(second["artifact"])).read_text(encoding="utf-8"))
+    assert sorted(evidence["blockers"]) == [
+        "callback_not_executable:maintenance_activate",
+        "callback_not_executable:power_restore_99",
+        "callback_not_executable:telegram_down",
+    ]
+
+    third = run_once(tmp_path, failed, "2026-07-11T08:00:30Z")
+    assert third["state"] == "outage_confirmed"
+    assert third["transition"] is None
+
+    fourth = run_once(tmp_path, healthy, "2026-07-11T08:00:45Z")
+    assert fourth["state"] == "recovering"
+    assert fourth["transition"] == "recovering"
+
+    fifth = run_once(tmp_path, healthy, "2026-07-11T08:01:00Z")
+    assert fifth["state"] == "recovered"
+    assert fifth["transition"] == "recovered"
+
+
+def test_external_l0_contract_is_off_lan_and_never_reads_response_bodies() -> None:
+    config = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8"))
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert len(config["targets"]) == 4
+    assert all(target["url"].startswith("https://") for target in config["targets"])
+    assert "192.168.0." not in EXAMPLE_CONFIG.read_text(encoding="utf-8")
+    assert "response_body_read" in text
+    assert "TELEGRAM_BOT_TOKEN" not in text
+    assert "shell=True" not in text
+    assert "callback_allowlist_root" in text
+    assert '"power_restore_99"' in EXAMPLE_CONFIG.read_text(encoding="utf-8")
--- a/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py
+++ b/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py
@@ -26,6 +26,10 @@ def test_observer_is_read_only_and_records_transitions() -> None:
    assert "probe_host" in text
    assert "probe_route" in text
    assert "record_transition" in text
+    assert "socket.create_connection" in text
+    assert 'TCP_PROBE_PYTHON="${TCP_PROBE_PYTHON:-python3}"' in text
+    assert "nc -z" not in text
+    assert "telnet://" not in text
    assert "samples.tsv" in text
    assert "events.tsv" in text
    assert "secret_value_read=false" in text