408 lines
14 KiB
Python
408 lines
14 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
from copy import deepcopy
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
PREFLIGHT = ROOT / "scripts/reboot-recovery/agent99-live-preflight.ps1"
|
|
|
|
|
|
def _function_source() -> str:
|
|
source = PREFLIGHT.read_text(encoding="utf-8")
|
|
start = source.index("function Get-AgentLivePreflightDecision")
|
|
end = source.index("function Get-TaskReadback", start)
|
|
return source[start:end]
|
|
|
|
|
|
def _evidence_function_source() -> str:
|
|
source = PREFLIGHT.read_text(encoding="utf-8")
|
|
start = source.index("function Get-EvidenceReadback")
|
|
end = source.index("$requiredTasks = @(", start)
|
|
return source[start:end]
|
|
|
|
|
|
def _base_case() -> dict[str, Any]:
|
|
return {
|
|
"agentRootPresent": True,
|
|
"manifest": {
|
|
"exists": True,
|
|
"sourceRevision": "a" * 40,
|
|
"runtimeMatched": True,
|
|
"fileCount": 18,
|
|
"mismatchCount": 0,
|
|
"parseError": "",
|
|
},
|
|
"manifestCheckRequired": True,
|
|
"taskFailureCount": 0,
|
|
"relayListenerCount": 1,
|
|
"requiredEvidenceStaleCount": 0,
|
|
"requiredEvidenceReserveFailureCount": 0,
|
|
"requiredEvidenceParseFailureCount": 0,
|
|
"requiredEvidenceContentFailureCount": 0,
|
|
"selfHealthSeverity": "ok",
|
|
"selfHealthPerformanceIssueCount": 0,
|
|
"staleAlertExecutionCount": 0,
|
|
"alertIncomingCount": 0,
|
|
"staleAlertIncomingCount": 0,
|
|
"pendingCommandCount": 0,
|
|
}
|
|
|
|
|
|
def _with_overrides(**overrides: Any) -> dict[str, Any]:
|
|
case = deepcopy(_base_case())
|
|
manifest_overrides = overrides.pop("manifest", None)
|
|
case.update(overrides)
|
|
if manifest_overrides:
|
|
case["manifest"].update(manifest_overrides)
|
|
return case
|
|
|
|
|
|
def _as_list(value: Any) -> list[str]:
|
|
if value is None:
|
|
return []
|
|
if isinstance(value, list):
|
|
return [str(item) for item in value]
|
|
return [str(value)]
|
|
|
|
|
|
def _run_decision(powershell: str, case: dict[str, Any]) -> dict[str, Any]:
|
|
fixture = json.dumps(case, separators=(",", ":"))
|
|
command = f"""
|
|
{_function_source()}
|
|
$case = @'
|
|
{fixture}
|
|
'@ | ConvertFrom-Json
|
|
$parameters = @{{
|
|
AgentRootPresent = [bool]$case.agentRootPresent
|
|
Manifest = $case.manifest
|
|
ManifestCheckRequired = [bool]$case.manifestCheckRequired
|
|
ExpectedRuntimeFileCount = 18
|
|
TaskFailureCount = [int]$case.taskFailureCount
|
|
RelayListenerCount = [int]$case.relayListenerCount
|
|
RequiredEvidenceStaleCount = [int]$case.requiredEvidenceStaleCount
|
|
RequiredEvidenceReserveFailureCount = [int]$case.requiredEvidenceReserveFailureCount
|
|
RequiredEvidenceParseFailureCount = [int]$case.requiredEvidenceParseFailureCount
|
|
RequiredEvidenceContentFailureCount = [int]$case.requiredEvidenceContentFailureCount
|
|
SelfHealthSeverity = [string]$case.selfHealthSeverity
|
|
SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount
|
|
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
|
|
AlertIncomingCount = [int]$case.alertIncomingCount
|
|
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
|
|
PendingCommandCount = [int]$case.pendingCommandCount
|
|
}}
|
|
Get-AgentLivePreflightDecision @parameters |
|
|
ConvertTo-Json -Depth 6 -Compress
|
|
"""
|
|
result = subprocess.run(
|
|
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = next(
|
|
line for line in reversed(result.stdout.splitlines()) if line.startswith("{")
|
|
)
|
|
return json.loads(payload)
|
|
|
|
|
|
def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> None:
|
|
source = PREFLIGHT.read_text(encoding="utf-8")
|
|
decision = _function_source()
|
|
|
|
assert '$warningReasons += "self_health_warning"' in decision
|
|
assert '$warningReasons += "performance_warning"' in decision
|
|
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
|
|
assert '"critical" { $blockingReasons += "self_health_not_ok" }' in decision
|
|
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
|
|
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
|
|
assert '$blockingReasons += "runtime_manifest_mismatch"' in decision
|
|
assert '$blockingReasons += "scheduled_task_unhealthy"' in decision
|
|
assert '$blockingReasons += "sre_alert_relay_not_listening"' in decision
|
|
assert (
|
|
'$blockingReasons += "required_evidence_freshness_reserve_insufficient"'
|
|
in decision
|
|
)
|
|
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
|
|
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
|
|
assert "-ExpectedRuntimeFileCount 18" in source
|
|
assert "warningReasons = $warningReasons" in source
|
|
assert "selfHealthPerformanceIssues = @(" in source
|
|
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source
|
|
assert 'if ($reason -match "^[a-z0-9_]{1,80}$")' in source
|
|
assert (
|
|
"[int]$selfCheckEvidence.safeSummary.selfHealthPerformanceIssueCount"
|
|
in source
|
|
)
|
|
assert 'if ($result.summary.deploymentEligible)' in source
|
|
assert 'if ($selfHealthSeverity -ne "ok")' not in source
|
|
assert '"agent99_background_mode_deferred_v1"' in source
|
|
assert '"agent99_recovery_transport_deferred_v1"' in source
|
|
assert "deferredCandidateCount = [int]$deferredCandidates.Count" in source
|
|
assert '$rootTaskPath = "\\"' in source
|
|
assert "Get-ScheduledTask -TaskPath $TaskPath -TaskName $TaskName" in source
|
|
assert "Get-ScheduledTaskInfo -TaskPath $TaskPath -TaskName $TaskName" in source
|
|
|
|
|
|
def test_evidence_selection_skips_deferred_but_not_malformed_candidates(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
|
if powershell is None:
|
|
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
|
|
|
evidence = tmp_path / "evidence"
|
|
evidence.mkdir()
|
|
canonical = evidence / "agent99-SelfCheck-20260715-200000.json"
|
|
deferred = evidence / "agent99-SelfCheck-20260715-200500.json"
|
|
canonical.write_text(
|
|
json.dumps(
|
|
{
|
|
"mode": "SelfCheck",
|
|
"selfHealth": {
|
|
"severity": "warning",
|
|
"latestPerformance": {"severity": "ok", "hosts": []},
|
|
},
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
deferred.write_text(
|
|
json.dumps(
|
|
{
|
|
"schemaVersion": "agent99_background_mode_deferred_v1",
|
|
"mode": "SelfCheck",
|
|
"deferred": True,
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
now = canonical.stat().st_mtime
|
|
os.utime(canonical, (now - 10, now - 10))
|
|
os.utime(deferred, (now, now))
|
|
|
|
agent_root = str(tmp_path).replace("'", "''")
|
|
command = f"""
|
|
{_evidence_function_source()}
|
|
$AgentRoot = '{agent_root}'
|
|
Get-EvidenceReadback 'self_check' 'agent99-SelfCheck-*.json' 20 $true |
|
|
ConvertTo-Json -Depth 8 -Compress
|
|
"""
|
|
result = subprocess.run(
|
|
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(
|
|
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
|
|
)
|
|
assert payload["file"] == canonical.name
|
|
assert payload["deferredCandidateCount"] == 1
|
|
assert payload["parsed"] is True
|
|
assert payload["safeSummary"]["selfHealthSeverity"] == "warning"
|
|
|
|
malformed = evidence / "agent99-SelfCheck-20260715-201000.json"
|
|
malformed.write_text("{not-json", encoding="utf-8")
|
|
os.utime(malformed, (now + 10, now + 10))
|
|
malformed_result = subprocess.run(
|
|
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert malformed_result.returncode == 0, (
|
|
malformed_result.stdout + malformed_result.stderr
|
|
)
|
|
malformed_payload = json.loads(
|
|
next(
|
|
line
|
|
for line in reversed(malformed_result.stdout.splitlines())
|
|
if line.startswith("{")
|
|
)
|
|
)
|
|
assert malformed_payload["file"] == malformed.name
|
|
assert malformed_payload["parsed"] is False
|
|
assert malformed_payload["healthy"] is False
|
|
|
|
|
|
def test_evidence_reserve_rejects_fresh_but_expiring_evidence_without_extending_slo(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
|
if powershell is None:
|
|
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
|
|
|
evidence = tmp_path / "evidence"
|
|
evidence.mkdir()
|
|
candidate = evidence / "agent99-Status-20260716-230000.json"
|
|
candidate.write_text(json.dumps({"mode": "Status"}), encoding="utf-8")
|
|
six_minutes_ago = candidate.stat().st_mtime - (6 * 60)
|
|
os.utime(candidate, (six_minutes_ago, six_minutes_ago))
|
|
|
|
agent_root = str(tmp_path).replace("'", "''")
|
|
command = f"""
|
|
{_evidence_function_source()}
|
|
$AgentRoot = '{agent_root}'
|
|
Get-EvidenceReadback 'status' 'agent99-Status-*.json' 20 $true 15 |
|
|
ConvertTo-Json -Depth 8 -Compress
|
|
"""
|
|
result = subprocess.run(
|
|
[powershell, "-NoProfile", "-NonInteractive", "-Command", command],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
payload = json.loads(
|
|
next(line for line in reversed(result.stdout.splitlines()) if line.startswith("{"))
|
|
)
|
|
assert payload["fresh"] is True
|
|
assert payload["maxAgeMinutes"] == 20
|
|
assert payload["minimumFreshnessReserveMinutes"] == 15
|
|
assert payload["freshnessReserveReady"] is False
|
|
assert payload["usable"] is False
|
|
assert payload["healthy"] is False
|
|
|
|
|
|
def test_decision_behavior_when_powershell_is_available() -> None:
|
|
powershell = shutil.which("pwsh") or shutil.which("powershell")
|
|
if powershell is None:
|
|
pytest.skip("PowerShell runtime is not installed on this macOS verifier")
|
|
|
|
cases = (
|
|
(_base_case(), True, (), ()),
|
|
(
|
|
_with_overrides(
|
|
selfHealthSeverity="warning",
|
|
selfHealthPerformanceIssueCount=1,
|
|
),
|
|
True,
|
|
(),
|
|
("self_health_warning", "performance_warning"),
|
|
),
|
|
(
|
|
_with_overrides(selfHealthSeverity="critical"),
|
|
False,
|
|
("self_health_not_ok",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(taskFailureCount=1),
|
|
False,
|
|
("scheduled_task_unhealthy",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(requiredEvidenceReserveFailureCount=1),
|
|
False,
|
|
("required_evidence_freshness_reserve_insufficient",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(manifest={"runtimeMatched": False, "mismatchCount": 1}),
|
|
False,
|
|
("runtime_manifest_mismatch",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(manifest={"sourceRevision": "not-a-full-sha"}),
|
|
False,
|
|
("runtime_manifest_identity_invalid",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(manifest={"fileCount": 15}),
|
|
False,
|
|
("runtime_manifest_identity_invalid",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(relayListenerCount=0),
|
|
False,
|
|
("sre_alert_relay_not_listening",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(alertIncomingCount=1),
|
|
True,
|
|
(),
|
|
("fresh_alert_pending_next_inbox_tick",),
|
|
),
|
|
(
|
|
_with_overrides(
|
|
alertIncomingCount=1,
|
|
staleAlertIncomingCount=1,
|
|
),
|
|
False,
|
|
("pending_work_before_reboot",),
|
|
(),
|
|
),
|
|
(
|
|
_with_overrides(pendingCommandCount=1),
|
|
False,
|
|
("pending_work_before_reboot",),
|
|
(),
|
|
),
|
|
)
|
|
|
|
for case, expected_eligible, expected_blockers, expected_warnings in cases:
|
|
result = _run_decision(powershell, case)
|
|
assert result["deploymentEligible"] is expected_eligible
|
|
assert set(_as_list(result.get("blockingReasons"))) == set(expected_blockers)
|
|
assert set(_as_list(result.get("warningReasons"))) == set(expected_warnings)
|
|
|
|
promotion_reserve = _run_decision(
|
|
powershell,
|
|
_with_overrides(
|
|
manifestCheckRequired=False,
|
|
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
|
|
),
|
|
)
|
|
assert promotion_reserve["deploymentEligible"] is True
|
|
assert promotion_reserve["manifestCheckRequired"] is False
|
|
|
|
promotion_reserve_task_failure = _run_decision(
|
|
powershell,
|
|
_with_overrides(
|
|
manifestCheckRequired=False,
|
|
manifest={"sourceRevision": "old", "fileCount": 15, "runtimeMatched": False},
|
|
taskFailureCount=1,
|
|
),
|
|
)
|
|
assert promotion_reserve_task_failure["deploymentEligible"] is False
|
|
assert promotion_reserve_task_failure["blockingReasons"] == [
|
|
"scheduled_task_unhealthy"
|
|
]
|
|
|
|
preflight_path = str(PREFLIGHT).replace("'", "''")
|
|
parser_command = (
|
|
"$tokens=$null; $errors=$null; "
|
|
"[System.Management.Automation.Language.Parser]::ParseFile("
|
|
f"'{preflight_path}', [ref]$tokens, [ref]$errors) | Out-Null; "
|
|
"if ($errors.Count -gt 0) { "
|
|
"$errors | ForEach-Object { $_.Message }; exit 1 }"
|
|
)
|
|
parsed = subprocess.run(
|
|
[
|
|
powershell,
|
|
"-NoProfile",
|
|
"-NonInteractive",
|
|
"-Command",
|
|
parser_command,
|
|
],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert parsed.returncode == 0, parsed.stdout + parsed.stderr
|