fix(agent99): unblock bounded maintenance deployment

scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py

@@ -5,6 +5,7 @@ import os
 import shutil
 import subprocess
 from copy import deepcopy
+from datetime import datetime, timedelta, timezone
 from pathlib import Path
 from typing import Any
 
@@ -24,7 +25,7 @@ def _function_source() -> str:
 
 def _evidence_function_source() -> str:
     source = PREFLIGHT.read_text(encoding="utf-8")
-    start = source.index("function Get-EvidenceReadback")
+    start = source.index("function Get-AgentBoundedNewestFiles")
     end = source.index("$requiredTasks = @(", start)
     return source[start:end]
 
@@ -52,7 +53,8 @@ def _base_case() -> dict[str, Any]:
         "staleAlertExecutionCount": 0,
         "alertIncomingCount": 0,
         "staleAlertIncomingCount": 0,
-        "pendingCommandCount": 0,
+        "pendingCommandBlockingCount": 0,
+        "promotionDeferredCommandCount": 0,
     }
 
 
@@ -96,7 +98,8 @@ $parameters = @{{
   StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
   AlertIncomingCount = [int]$case.alertIncomingCount
   StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
-  PendingCommandCount = [int]$case.pendingCommandCount
+  PendingCommandBlockingCount = [int]$case.pendingCommandBlockingCount
+  PromotionDeferredCommandCount = [int]$case.promotionDeferredCommandCount
 }}
 Get-AgentLivePreflightDecision @parameters |
   ConvertTo-Json -Depth 6 -Compress
@@ -121,6 +124,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
     assert '$warningReasons += "self_health_warning"' in decision
     assert '$warningReasons += "performance_warning"' in decision
     assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
+    assert (
+        '$warningReasons += "maintenance_pending_commands_deferred_until_post_promotion"'
+        in decision
+    )
     assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision
     assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
     assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
@@ -147,9 +154,45 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
     assert '"agent99_background_mode_deferred_v1"' in source
     assert '"agent99_recovery_transport_deferred_v1"' in source
     assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
+    assert "[IO.Directory]::EnumerateFiles(" in source
+    assert "[Collections.Generic.SortedSet[string]]::new(" in source
+    assert "$null = $selected.Remove($selected.Min)" in source
+    assert "candidateWindowExhausted = $candidateWindowExhausted" in source
+    assert "function Get-AgentControlLoopMaintenanceReadback" in source
+    assert '"agent99-control-loop-maintenance-2*.json"' in source
+    assert '$ageMinutes -le $(if ($legacyReceipt) { 360 } else { 720 })' in source
+    assert '$legacyPropertySetMatches' in source
+    assert '$legacyIdentityFieldsValid' in source
+    assert '$receipt.taskEnabled -is [bool] -and $receipt.taskEnabled' in source
+    assert '"legacy_v2_exact_upgrade_bridge"' in source
+    assert '$receipt.precheckPassed -is [bool]' in source
+    assert '$receipt.taskEnabledAfter -is [bool]' in source
+    assert '$receipt.runtimeWritePerformed -is [bool]' in source
+    assert '$intent.secretValueRead -is [bool]' in source
+    assert '$activeControlProcesses.Count -eq 0' in source
+    assert '-not $leasePresent' in source
+    assert '-not $recoveryMutexActive' in source
+    assert '$promotionEvidenceRequired = [bool]($ReadinessScope -eq "FullRuntime")' in source
+    assert '$task | Add-Member -NotePropertyName runtimeHealthy' in source
+    assert '$task | Add-Member -NotePropertyName promotionReady' in source
+    assert '$controlLoopMaintenance.valid' in source
+    assert "function Get-AgentPendingCommandPromotionReadback" in source
+    assert '"bounded_maintenance_auto_recover_deferred_until_post_promotion"' in source
+    assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
+    assert '$payload.controlledApply -is [bool] -and $payload.controlledApply' in source
+    assert '$payload.observation -is [pscustomobject]' in source
+    assert 'instructionContentReadback = $false' in source
     assert '$rootTaskPath = "\\"' in source
     assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
     assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
+    assert "[string[]]$RequiredArgumentMarkers" in source
+    assert "$actions.Count -eq 1" in source
+    assert "actionDefinitionReady = $actionDefinitionReady" in source
+    assert "principalReady = $principalReady" in source
+    assert "definitionReady = $definitionReady" in source
+    assert '$task.Principal.LogonType -eq "S4U"' in source
+    assert '$task.Principal.RunLevel -eq "Highest"' in source
+    assert "$task.definitionReady" in source
 
 
 def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
@@ -274,6 +317,165 @@ Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
     assert payload["healthy"] is False
 
 
+def test_evidence_candidate_window_fails_closed_when_recent_files_are_all_deferred(
+    tmp_path: Path,
+) -> None:
+    powershell = shutil.which("pwsh") or shutil.which("powershell")
+    if powershell is None:
+        pytest.skip("PowerShell runtime is not installed on this macOS verifier")
+
+    evidence = tmp_path / "evidence"
+    evidence.mkdir()
+    canonical = evidence / "agent99-SelfCheck-20260715-190000.json"
+    canonical.write_text(
+        json.dumps(
+            {
+                "mode": "SelfCheck",
+                "selfHealth": {"severity": "ok"},
+            }
+        ),
+        encoding="utf-8",
+    )
+    base_time = canonical.stat().st_mtime
+    os.utime(canonical, (base_time - 1000, base_time - 1000))
+    for index in range(40):
+        deferred = evidence / f"agent99-SelfCheck-20260715-20{index:04d}.json"
+        deferred.write_text(
+            json.dumps(
+                {
+                    "schemaVersion": "agent99_background_mode_deferred_v1",
+                    "deferred": True,
+                }
+            ),
+            encoding="utf-8",
+        )
+        candidate_time = base_time + index
+        os.utime(deferred, (candidate_time, candidate_time))
+
+    agent_root = str(tmp_path).replace("'", "''")
+    command = f"""
+{_evidence_function_source()}
+$AgentRoot = '{agent_root}'
+Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
+  ConvertTo-Json -Depth 8 -Compress
+"""
+    result = subprocess.run(
+        [powershell, "-NoProfile", "-NonInteractive", "-Command", command],
+        check=False,
+        capture_output=True,
+        text=True,
+    )
+    assert result.returncode == 0, result.stdout + result.stderr
+    payload = json.loads(
+        next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
+    )
+    assert payload["enumeratedCandidateCount"] == 41
+    assert payload["candidateWindowLimit"] == 32
+    assert payload["candidateWindowTruncated"] is True
+    assert payload["candidateWindowExhausted"] is True
+    assert payload["deferredCandidateCount"] == 32
+    assert payload["parseError"] == "EvidenceCandidateWindowExhausted"
+    assert payload["healthy"] is False
+
+
+def test_promotion_reserve_defers_only_exact_maintenance_auto_recover(
+    tmp_path: Path,
+) -> None:
+    powershell = shutil.which("pwsh") or shutil.which("powershell")
+    if powershell is None:
+        pytest.skip("PowerShell runtime is not installed on this macOS verifier")
+
+    queue = tmp_path / "queue"
+    queue.mkdir()
+    now = datetime.now(timezone.utc)
+    incident_id = "agent99-auto-" + ("a" * 20)
+    command_id = "auto-recover-20260719-152746-39ca4a72"
+    command_path = queue / f"{command_id}.json"
+    payload = {
+        "approvalId": "",
+        "automationRunId": "096c8173-96b9-5659-94d3-5679c3fb770f",
+        "canonicalDigest": "b" * 64,
+        "controlledApply": True,
+        "correlationKey": "agent99-controlled:" + ("c" * 64),
+        "createdAt": now.isoformat(),
+        "dispatchIdentitySchemaVersion": "agent99_controlled_dispatch_identity_v1",
+        "dispatchRouteId": "agent99_local_observer",
+        "executionGeneration": "1",
+        "externalId": "host_unreachable_detected",
+        "id": command_id,
+        "idempotencyKey": "agent99-controlled:" + ("c" * 64),
+        "incidentId": incident_id,
+        "instruction": "recover exact observed unreachable host through controlled playbook",
+        "lockOwner": "096c8173-96b9-5659-94d3-5679c3fb770f",
+        "mode": "Recover",
+        "observation": {},
+        "projectId": "awoooi",
+        "reason": "host_unreachable_detected",
+        "requestedBy": "Agent99",
+        "singleFlightKey": "agent99-controlled-flight:" + ("d" * 64),
+        "source": "agent99-recovery-observer",
+        "sourceFingerprint": "e" * 64,
+        "traceId": "00-" + ("f" * 32) + "-" + ("1" * 16) + "-01",
+        "workItemId": f"agent99-auto:{incident_id}:host_unreachable_detected",
+    }
+
+    function_source = _evidence_function_source()
+    agent_root = str(tmp_path).replace("'", "''")
+    maintenance_timestamp = (now - timedelta(minutes=10)).isoformat()
+
+    def run(scope: str) -> dict[str, Any]:
+        command = f"""
+{function_source}
+$AgentRoot = '{agent_root}'
+$maintenance = [pscustomobject]@{{
+  valid = $true
+  receiptFieldsValid = $true
+  intentFieldsValid = $true
+  receiptTimestamp = '{maintenance_timestamp}'
+}}
+Get-AgentPendingCommandPromotionReadback -MaintenanceReadback $maintenance -Scope {scope} |
+  ConvertTo-Json -Depth 8 -Compress
+"""
+        result = subprocess.run(
+            [powershell, "-NoProfile", "-NonInteractive", "-Command", command],
+            check=False,
+            capture_output=True,
+            text=True,
+        )
+        assert result.returncode == 0, result.stdout + result.stderr
+        return json.loads(
+            next(
+                line
+                for line in reversed(result.stdout.splitlines())
+                if line.startswith("{")
+            )
+        )
+
+    command_path.write_bytes(b"\xef\xbb\xbf" + json.dumps(payload).encode("utf-8"))
+    reserve = run("PromotionReserve")
+    assert reserve["observedCount"] == 1
+    assert reserve["deferredCount"] == 1
+    assert reserve["blockingCount"] == 0
+    assert reserve["contractValid"] is True
+    assert reserve["items"][0]["instructionContentReadback"] is False
+    assert reserve["items"][0]["identity"].startswith(f"{command_id}|")
+
+    full_runtime = run("FullRuntime")
+    assert full_runtime["deferredCount"] == 0
+    assert full_runtime["blockingCount"] == 1
+    assert full_runtime["eligibleMaintenanceAutoRecoverCount"] == 1
+    assert full_runtime["reason"] == (
+        "full_runtime_requires_maintenance_auto_recover_to_drain"
+    )
+
+    payload["mode"] = "Status"
+    command_path.write_text(json.dumps(payload), encoding="utf-8")
+    invalid = run("PromotionReserve")
+    assert invalid["deferredCount"] == 0
+    assert invalid["blockingCount"] == 1
+    assert invalid["contractValid"] is False
+
+
 def test_decision_behavior_when_powershell_is_available() -> None:
     powershell = shutil.which("pwsh") or shutil.which("powershell")
     if powershell is None:
@@ -348,11 +550,17 @@ def test_decision_behavior_when_powershell_is_available() -> None:
             (),
         ),
         (
-            _with_overrides(pendingCommandCount=1),
+            _with_overrides(pendingCommandBlockingCount=1),
             False,
             ("pending_work_before_reboot",),
             (),
         ),
+        (
+            _with_overrides(promotionDeferredCommandCount=1),
+            True,
+            (),
+            ("maintenance_pending_commands_deferred_until_post_promotion",),
+        ),
     )
 
     for case, expected_eligible, expected_blockers, expected_warnings in cases:

scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py

@@ -391,11 +391,39 @@ def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() ->
     assert "failedTasks" in source
     assert "failedEvidenceNames" in source
     assert "failedEvidence" in source
+    assert "actionDefinitionReady = [bool]$_.actionDefinitionReady" in source
+    assert "principalReady = [bool]$_.principalReady" in source
+    assert "definitionReady = [bool]$_.definitionReady" in source
     assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
     assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
     assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
 
 
+def test_post_promotion_verifier_waits_only_for_bounded_scheduler_freshness() -> None:
+    source = RECEIVER.read_text(encoding="utf-8")
+
+    assert "$LivePreflightTimeoutSeconds = 60" in source
+    assert "$PostVerifyMaxAttempts = 4" in source
+    assert "$PostVerifyRetryWaitSeconds = 45" in source
+    assert "function Test-AgentPostPromotionReadinessRetryable" in source
+    assert '[string]$Readiness.readinessScope -ne "FullRuntime"' in source
+    assert '"required_evidence_stale"' in source
+    assert '"required_evidence_content_unhealthy"' in source
+    assert '$_.name -notin @("telegram_inbox", "sre_alert_inbox")' in source
+    assert "-not $_.definitionReady" in source
+    assert "[int]$Readiness.alertRunningCount -ne 0" in source
+    assert "$pendingCommandsRetryable" in source
+    assert "$ExpectedMaintenancePendingCommandIdentities" in source
+    assert '"pending_work_before_reboot"' in source
+    assert '"full_runtime_requires_maintenance_auto_recover_to_drain"' in source
+    assert "maintenancePendingCommandIdentities" in source
+    assert "$postVerifyAttempt -le $PostVerifyMaxAttempts" in source
+    assert "Start-Sleep -Seconds $PostVerifyRetryWaitSeconds" in source
+    assert "automaticTaskTriggerAllowed = $false" in source
+    assert "postVerifyAttempts = $postVerifyAttempts" in source
+    assert "postVerifyRetryCount = $postVerifyRetryCount" in source
+
+
 def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
     sender = SENDER.read_text(encoding="utf-8")
     projector = sender[
@@ -607,7 +635,7 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
     assert "$RelayPollIntervalSeconds = 1" in source
     assert "$ValidateOnlyTimeoutSeconds = 300" in source
     assert "$LiveDeployTimeoutSeconds = 300" in source
-    assert "$LivePreflightTimeoutSeconds = 120" in source
+    assert "$LivePreflightTimeoutSeconds = 60" in source
     assert "$PromotionReadinessMaxAttempts = 2" in source
     assert "$PromotionReadinessRetryWaitSeconds = 30" in source
     assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
@@ -648,18 +676,23 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
     promotion_readiness_attempts = 2
     promotion_readiness_retry_wait_seconds = 30
     deploy_seconds = 300
-    live_preflight_seconds = 120
+    live_preflight_seconds = 60
+    post_verify_attempts = 4
+    post_verify_retry_wait_seconds = 45
+    windows_control_baseline_seconds = 300
     worst_case_seconds = (
         validate_seconds
         + (promotion_readiness_attempts * live_preflight_seconds)
         + promotion_readiness_retry_wait_seconds
         + deploy_seconds
-        + live_preflight_seconds
+        + (post_verify_attempts * live_preflight_seconds)
+        + ((post_verify_attempts - 1) * post_verify_retry_wait_seconds)
         + (2 * (60 + 360))
+        + windows_control_baseline_seconds
     )
     assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
-    assert worst_case_seconds == 1830
-    assert wrapper_budget_seconds - worst_case_seconds == 570
+    assert worst_case_seconds == 2265
+    assert wrapper_budget_seconds - worst_case_seconds == 135
 
     deploy = source.index(
         "$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
@@ -781,6 +814,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
     assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
     assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
     assert '[int]$summary.alertRunningCount -eq 0' in preflight
+    assert '[int]$summary.pendingCommandBlockingCount -eq 0' in preflight
+    assert '$ReadinessScope -eq "PromotionReserve"' in preflight
     assert '[int]$summary.pendingCommandCount -eq 0' in preflight
     assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
     assert '[string[]]$blockingReasons = @()' in preflight

scripts/reboot-recovery/tests/test_agent99_stale_queue_quarantine_contract.py

@@ -0,0 +1,44 @@
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts/reboot-recovery/agent99-stale-queue-quarantine.ps1"
+
+
+def test_stale_queue_quarantine_is_exact_recoverable_and_receipted() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+
+    assert '[ValidateSet("Check", "Apply")]' in source
+    assert "ExpectedQueueSha256" in source
+    assert "ExpectedMaintenanceSha256" in source
+    assert '"Global\\WoooAgent99StaleQueueMaintenanceV1"' in source
+    assert '"Global\\WoooAgent99RecoveryTransportV1"' in source
+    assert 'Get-ScheduledTask `' in source
+    assert '"Wooo-Agent99-Control-Loop"' in source
+    assert 'throw "control_loop_not_frozen"' in source
+    assert 'throw "active_control_or_recover_process"' in source
+    assert 'throw "recovery_lease_present"' in source
+    assert 'throw "recovery_mutex_active"' in source
+    assert '[string]$command.mode -ne "Recover"' in source
+    assert '[string]$command.source -ne "agent99-recovery-observer"' in source
+    assert "$command.controlledApply -isnot [bool]" in source
+    assert '"atomic_move_exact_queue_item_to_failed"' in source
+    assert "[IO.File]::Move($sourcePath, $destinationPath)" in source
+    assert "[IO.File]::Move($destinationPath, $sourcePath)" in source
+    assert '"verified_quarantined"' in source
+    assert '"failed_rolled_back"' in source
+    assert '"rollback_unverified"' in source
+
+
+def test_stale_queue_quarantine_does_not_execute_or_delete_command() -> None:
+    source = SCRIPT.read_text(encoding="utf-8")
+
+    assert "commandExecuted = $false" in source
+    assert "deletePerformed = $false" in source
+    assert "otherQueueMutationPerformed = $false" in source
+    assert "rawInstructionStored = $false" in source
+    assert "secretValueRead = $false" in source
+    assert "Remove-Item -LiteralPath $sourcePath" not in source
+    assert "Start-ScheduledTask" not in source
+    assert "Invoke-Expression" not in source
+    assert "& $command" not in source