wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): reserve pre-promotion readiness

Browse Source
This commit is contained in:
ogt
2026-07-16 23:47:05 +08:00
parent 59d03fe277
commit d3d53a4f69
5 changed files with 445 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py
View File

@@ -40,9 +40,11 @@ def _base_case() -> dict[str, Any]:
"mismatchCount": 0,
"parseError": "",
},
"manifestCheckRequired": True,
"taskFailureCount": 0,
"relayListenerCount": 1,
"requiredEvidenceStaleCount": 0,
"requiredEvidenceReserveFailureCount": 0,
"requiredEvidenceParseFailureCount": 0,
"requiredEvidenceContentFailureCount": 0,
"selfHealthSeverity": "ok",
@@ -81,10 +83,12 @@ $case = @'
$parameters = @{{
AgentRootPresent = [bool]$case.agentRootPresent
Manifest = $case.manifest
ManifestCheckRequired = [bool]$case.manifestCheckRequired
ExpectedRuntimeFileCount = 16
TaskFailureCount = [int]$case.taskFailureCount
RelayListenerCount = [int]$case.relayListenerCount
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
RequiredEvidenceReserveFailureCount = [int]$case.requiredEvidenceReserveFailureCount
RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount
RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount
SelfHealthSeverity = [string]$case.selfHealthSeverity
@@ -123,6 +127,10 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '$blockingReasons += "runtime_manifest_mismatch"' in decision
assert '$blockingReasons += "scheduled_task_unhealthy"' in decision
assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision
assert (
'$blockingReasons += "required_evidence_freshness_reserve_insufficient"'
in decision
)
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
assert "-ExpectedRuntimeFileCount 16" in source
@@ -139,6 +147,9 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '"agent99_background_mode_deferred_v1"' in source
assert '"agent99_recovery_transport_deferred_v1"' in source
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
assert '$rootTaskPath = "\\"' in source
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
@@ -224,6 +235,45 @@ Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
assert malformed_payload["healthy"] is False
def test_evidence_reserve_rejects_fresh_but_expiring_evidence_without_extending_slo(
tmp_path: Path,
) -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
evidence = tmp_path / "evidence"
evidence.mkdir()
candidate = evidence / "agent99-Status-20260716-230000.json"
candidate.write_text(json.dumps({"mode": "Status"}), encoding="utf-8")
six_minutes_ago = candidate.stat().st_mtime - (6 * 60)
os.utime(candidate, (six_minutes_ago, six_minutes_ago))
agent_root = str(tmp_path).replace("'", "''")
command = f"""
{_evidence_function_source()}
$AgentRoot = '{agent_root}'
Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
ConvertTo-Json -Depth 8 -Compress
"""
result = subprocess.run(
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stdout + result.stderr
payload = json.loads(
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
)
assert payload["fresh"] is True
assert payload["maxAgeMinutes"] == 20
assert payload["minimumFreshnessReserveMinutes"] == 15
assert payload["freshnessReserveReady"] is False
assert payload["usable"] is False
assert payload["healthy"] is False
def test_decision_behavior_when_powershell_is_available() -> None:
powershell = shutil.which("pwsh") or shutil.which("powershell")
if powershell is None:
@@ -252,6 +302,12 @@ def test_decision_behavior_when_powershell_is_available() -> None:
("scheduled_task_unhealthy",),
(),
),
(
_with_overrides(requiredEvidenceReserveFailureCount=1),
False,
("required_evidence_freshness_reserve_insufficient",),
(),
),
(
_with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}),
False,
@@ -305,6 +361,29 @@ def test_decision_behavior_when_powershell_is_available() -> None:
assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers)
assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings)
promotion_reserve = _run_decision(
powershell,
_with_overrides(
manifestCheckRequired=False,
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
),
)
assert promotion_reserve["deploymentEligible"] is True
assert promotion_reserve["manifestCheckRequired"] is False
promotion_reserve_task_failure = _run_decision(
powershell,
_with_overrides(
manifestCheckRequired=False,
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
taskFailureCount=1,
),
)
assert promotion_reserve_task_failure["deploymentEligible"] is False
assert promotion_reserve_task_failure["blockingReasons"] == [
"scheduled_task_unhealthy"
]
preflight_path = str(PREFLIGHT).replace("'", "''")
parser_command = (
"$tokens=$null; $errors=$null; "

5
scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py
View File

@@ -179,7 +179,10 @@ def test_live_preflight_rejects_fresh_but_unparseable_required_evidence() -> Non
preflight = read("scripts/reboot-recovery/agent99-live-preflight.ps1")
assert '$parsed = [bool]($file -and -not $parseError)' in preflight
assert '$usable = [bool]($fresh -and $parsed -and $contentHealthy)' in preflight
assert (
"$usable = [bool]($fresh -and $freshnessReserveReady -and $parsed -and "
"$contentHealthy)" in preflight
)
assert 'healthy = [bool](-not $Required -or $usable)' in preflight
assert '$_.required -and -not $_.healthy' in preflight
assert 'required_evidence_parse_failed' in preflight

83
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
View File

@@ -209,8 +209,10 @@ def test_receiver_validates_manifest_before_staging_and_validate_only_before_pro
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision"
)
assert manifest_check < stage_write < validate < promote
readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"')
assert manifest_check < stage_write < validate < readiness < promote
assert 'status = "validate_only_failed_no_promotion"' in source
assert 'status = "blocked_pre_promotion_readiness_no_live_write"' in source
assert "liveRuntimeWritePerformed = $false" in source
assert "livePromotionPerformed = $false" in source
@@ -236,7 +238,12 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
assert "function Invoke-AgentBoundedPowerShell" in source
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
assert "$process.Kill()" in source
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
assert "$ValidateOnlyTimeoutSeconds = 300" in source
assert "$LiveDeployTimeoutSeconds = 300" in source
assert (
"$timeoutSeconds = if ($ValidateOnly) { $ValidateOnlyTimeoutSeconds } "
"else { $LiveDeployTimeoutSeconds }" in source
)
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
assert '$runtimeManifest.fileCount -eq 16' in source
@@ -301,6 +308,60 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
assert "[bool]$prior.relayReload.newGenerationVerified" in source
def test_receiver_blocks_before_live_write_with_sanitized_readiness_receipt() -> None:
source = RECEIVER.read_text(encoding="utf-8")
validate = source.index(
"$validate = Invoke-AgentDeployChild -SourceRoot $stagePath "
"-SourceRevision $sourceRevision -ValidateOnly"
)
readiness = source.index('$script:ReceiverOperationPhase = "pre_promotion_readiness"')
live_write = source.index('$script:ReceiverOperationPhase = "live_deploy"')
assert validate < readiness < live_write
assert '-ReadinessScope "PromotionReserve"' in source
assert (
"-MinimumEvidenceFreshnessReserveMinutes "
"$PromotionReadinessReserveMinutes" in source
)
assert "$PromotionReadinessReserveMinutes = 15" in source
assert "$PromotionReadinessMaxAttempts = 2" in source
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
assert "function Test-AgentPromotionReadinessRetryable" in source
assert '"scheduled_task_unhealthy"' in source
assert '"required_evidence_stale"' in source
assert '"required_evidence_freshness_reserve_insufficient"' in source
assert '[string]$Readiness.readinessScope -ne "PromotionReserve"' in source
assert (
"[int]$Readiness.minimumEvidenceFreshnessReserveMinutes -ne "
"$PromotionReadinessReserveMinutes" in source
)
assert '"scheduled_task_unhealthy" -in $reasons' in source
assert "@($Readiness.failedTasks).Count -eq 0" in source
assert "@($Readiness.failedEvidence).Count -eq 0" in source
assert '$_.state -notin @("Ready", "Queued")' in source
assert "automaticTaskTriggerAllowed = $false" in source
assert "unboundedRetryAllowed = $false" in source
assert "Start-ScheduledTask" not in source[readiness:live_write]
blocked = source[
source.index('$status = "blocked_pre_promotion_readiness_no_live_write"')
if '$status = "blocked_pre_promotion_readiness_no_live_write"' in source
else source.index('status = "blocked_pre_promotion_readiness_no_live_write"') : live_write
]
assert "liveRuntimeWritePerformed = $false" in blocked
assert "livePromotionAttempted = $false" in blocked
assert "livePromotionPerformed = $false" in blocked
assert "readinessCheckWritePerformed = $false" in blocked
assert "failedTaskNames" in source
assert "failedTasks" in source
assert "failedEvidenceNames" in source
assert "failedEvidence" in source
assert 'taskPath = if ([string]$_.taskPath -eq "\\") { "\\" }' in source
assert "ageMinutes = if ($null -ne $_.ageMinutes)" in source
assert "maxAgeMinutes = if ($null -ne $_.maxAgeMinutes)" in source
def test_check_and_apply_receipts_project_the_same_run_identity() -> None:
sender = SENDER.read_text(encoding="utf-8")
projector = sender[
@@ -510,6 +571,11 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert "$RelayStopTimeoutSeconds = 60" in source
assert "$RelayStartTimeoutSeconds = 360" in source
assert "$RelayPollIntervalSeconds = 1" in source
assert "$ValidateOnlyTimeoutSeconds = 300" in source
assert "$LiveDeployTimeoutSeconds = 300" in source
assert "$LivePreflightTimeoutSeconds = 120" in source
assert "$PromotionReadinessMaxAttempts = 2" in source
assert "$PromotionReadinessRetryWaitSeconds = 30" in source
assert "Get-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Get-ScheduledTaskInfo -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
assert "Stop-ScheduledTask -TaskPath $RelayTaskPath -TaskName $RelayTaskName" in source
@@ -545,18 +611,21 @@ def test_receiver_reloads_only_the_exact_existing_relay_task_and_proves_generati
assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in wrapper
wrapper_budget_seconds = 2400
validate_seconds = 300
deploy_seconds = 900
promotion_readiness_attempts = 2
promotion_readiness_retry_wait_seconds = 30
deploy_seconds = 300
live_preflight_seconds = 120
worst_case_seconds = (
validate_seconds
+ (promotion_readiness_attempts * live_preflight_seconds)
+ promotion_readiness_retry_wait_seconds
+ deploy_seconds
+ live_preflight_seconds
+ (2 * (60 + 360))
)
assert "$timeoutSeconds = if ($ValidateOnly) { 300 } else { 900 }" in source
assert "-TimeoutSeconds 120" in source
assert worst_case_seconds == 2160
assert wrapper_budget_seconds - worst_case_seconds == 240
assert "-TimeoutSeconds $LivePreflightTimeoutSeconds" in source
assert worst_case_seconds == 1830
assert wrapper_budget_seconds - worst_case_seconds == 570
deploy = source.index(
"$deploy = Invoke-AgentDeployChild -SourceRoot $stagePath "