1519 lines
59 KiB
Python
1519 lines
59 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import fcntl
|
|
import grp
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import os
|
|
import pwd
|
|
import re
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
|
|
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
|
|
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
|
|
BROKER_CONTRACT_REPLAY = (
|
|
ROOT / "scripts" / "reboot-recovery" / "tests" / "agent99-host110-broker-contract-replay.ps1"
|
|
)
|
|
|
|
|
|
EXPECTED_RUNTIME_FILES = {
|
|
"common.sh",
|
|
"backup-all.sh",
|
|
"backup-host188-products.sh",
|
|
"verify-host188-products-backup.sh",
|
|
"verify-host188-products-archive.py",
|
|
"backup-gitea.sh",
|
|
"gitea-full-backup-restore-drill.sh",
|
|
"backup-configs.sh",
|
|
"backup-awoooi.sh",
|
|
"backup-awoooi-frequent.sh",
|
|
"backup-clawbot.sh",
|
|
"backup-sentry.sh",
|
|
"check-backup-integrity.sh",
|
|
"sync-offsite-backups.sh",
|
|
"backup-offsite-readiness-gate.sh",
|
|
"verify-offsite-full-sync.sh",
|
|
"enforce-latest-only-retention.sh",
|
|
}
|
|
|
|
|
|
def _load_verifier():
|
|
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def _powershell_string(source: str, variable: str) -> str:
|
|
match = re.search(rf'\${re.escape(variable)}\s*=\s*"([^"]+)"', source)
|
|
assert match is not None, variable
|
|
return match.group(1)
|
|
|
|
|
|
def _powershell_array(source: str, variable: str) -> tuple[str, ...]:
|
|
match = re.search(
|
|
rf'\${re.escape(variable)}\s*=\s*@\((.*?)\n\)',
|
|
source,
|
|
re.DOTALL,
|
|
)
|
|
assert match is not None, variable
|
|
return tuple(re.findall(r'"([^"]+)"', match.group(1)))
|
|
|
|
|
|
def _powershell_function(source: str, name: str) -> str:
|
|
start = source.index(f"function {name} {{")
|
|
next_function = source.find("\nfunction ", start + 1)
|
|
return source[start:] if next_function < 0 else source[start:next_function]
|
|
|
|
|
|
def _executor_payload_order(source: str) -> tuple[str, ...]:
|
|
match = re.search(r"readonly -a RUNTIME_FILES=\((.*?)\n\)", source, re.DOTALL)
|
|
assert match is not None
|
|
runtime_files = tuple(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", match.group(1), re.MULTILINE))
|
|
exporter = re.search(r'^readonly EXPORTER_FILE="([^"]+)"$', source, re.MULTILINE)
|
|
assert exporter is not None
|
|
return (*runtime_files, exporter.group(1))
|
|
|
|
|
|
def _write_executable(path: Path, source: str) -> None:
|
|
path.write_text(source, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _write_fcntl_flock(path: Path) -> None:
|
|
_write_executable(
|
|
path,
|
|
"""#!/usr/bin/env python3
|
|
import fcntl
|
|
import sys
|
|
import time
|
|
|
|
args = sys.argv[1:]
|
|
operation = fcntl.LOCK_SH if "-s" in args else fcntl.LOCK_EX
|
|
wait_seconds = float(args[args.index("-w") + 1]) if "-w" in args else 0.0
|
|
descriptor = int(args[-1])
|
|
deadline = time.monotonic() + wait_seconds
|
|
while True:
|
|
try:
|
|
fcntl.flock(descriptor, operation | fcntl.LOCK_NB)
|
|
raise SystemExit(0)
|
|
except BlockingIOError:
|
|
if time.monotonic() >= deadline:
|
|
raise SystemExit(1)
|
|
time.sleep(0.02)
|
|
""",
|
|
)
|
|
|
|
|
|
def _build_shared_lock_gate_fixture(
|
|
tmp_path: Path,
|
|
entrypoint: str,
|
|
*,
|
|
reject_fd_reopen: bool = False,
|
|
body_sleep_seconds: float = 0.0,
|
|
) -> tuple[Path, Path, Path, dict[str, str]]:
|
|
runtime_root = tmp_path / "runtime-scripts"
|
|
runtime_root.mkdir()
|
|
lock_path = tmp_path / "runtime.lock"
|
|
fake_bin = tmp_path / "fake-bin"
|
|
fake_bin.mkdir()
|
|
_write_fcntl_flock(fake_bin / "flock")
|
|
|
|
source_map = {
|
|
"common": ROOT / "scripts/backup/common.sh",
|
|
"restore": ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh",
|
|
"offsite": ROOT / "scripts/backup/backup-offsite-readiness-gate.sh",
|
|
}
|
|
source = source_map[entrypoint].read_text(encoding="utf-8")
|
|
source = source.replace("/backup/scripts", str(runtime_root))
|
|
source = source.replace("/tmp/agent99-host110-backup-runtime.lock", str(lock_path))
|
|
source = source.replace("flock -s -w 60", "flock -s -w 1")
|
|
if reject_fd_reopen:
|
|
source, replacement_count = re.subn(
|
|
r"(?m)^(\s*)exec 197>>.*$",
|
|
r"\1printf 'FD_REOPENED=1\\n' >&2; exit 98",
|
|
source,
|
|
)
|
|
assert replacement_count == 1
|
|
|
|
body = "printf 'BODY_REACHED=1\\n'\n"
|
|
if body_sleep_seconds:
|
|
body += f"sleep {body_sleep_seconds}\n"
|
|
if entrypoint == "common":
|
|
common_path = runtime_root / "common.sh"
|
|
gate_source = source.split("# WOOO AIOps - 備份共用函式庫", 1)[0]
|
|
_write_executable(common_path, gate_source)
|
|
script_path = runtime_root / "backup-gitea.sh"
|
|
_write_executable(
|
|
script_path,
|
|
"#!/usr/bin/env bash\n"
|
|
"set -euo pipefail\n"
|
|
f"source {str(common_path)!r}\n"
|
|
+ body,
|
|
)
|
|
else:
|
|
product_start = "DUMP_ZIP=" if entrypoint == "restore" else "BACKUP_BASE="
|
|
gate_source = source.split(product_start, 1)[0]
|
|
script_path = runtime_root / source_map[entrypoint].name
|
|
_write_executable(script_path, gate_source + body)
|
|
|
|
environment = os.environ.copy()
|
|
environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
|
|
return script_path, runtime_root, lock_path, environment
|
|
|
|
|
|
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
|
|
destination = tmp_path / "backup-scripts"
|
|
exporter_root = tmp_path / "exporter"
|
|
status_root = tmp_path / "status"
|
|
stage_root = tmp_path / "stage"
|
|
rollback_root = tmp_path / "rollback"
|
|
source_stage = tmp_path / f"source-{run_id}"
|
|
lock_path = tmp_path / "runtime.lock"
|
|
fake_bin = tmp_path / "fake-bin"
|
|
for path in (destination, exporter_root, source_stage, fake_bin):
|
|
path.mkdir(parents=True)
|
|
|
|
user_name = pwd.getpwuid(os.getuid()).pw_name
|
|
group_name = grp.getgrgid(os.getgid()).gr_name
|
|
shell_path = shutil.which("bash") or "/bin/bash"
|
|
shell_version = subprocess.run(
|
|
[shell_path, "-c", 'printf "%s" "${BASH_VERSINFO[0]:-0}"'],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
).stdout
|
|
use_zsh_adapter = not shell_version.isdigit() or int(shell_version) < 4
|
|
if use_zsh_adapter:
|
|
shell_path = shutil.which("zsh") or "/bin/zsh"
|
|
executor_source = EXECUTOR.read_text(encoding="utf-8")
|
|
replacements = {
|
|
'readonly EXPECTED_USER="wooo"': f'readonly EXPECTED_USER="{user_name}"',
|
|
'readonly DEST_ROOT="/backup/scripts"': f'readonly DEST_ROOT="{destination}"',
|
|
'readonly EXPORTER_ROOT="/home/wooo/scripts"': f'readonly EXPORTER_ROOT="{exporter_root}"',
|
|
'readonly STATUS_ROOT="/backup/status"': f'readonly STATUS_ROOT="{status_root}"',
|
|
'readonly STAGE_ROOT="/backup/.agent99-backup-runtime-stage"': f'readonly STAGE_ROOT="{stage_root}"',
|
|
'readonly ROLLBACK_ROOT="/backup/.agent99-backup-runtime-rollback"': f'readonly ROLLBACK_ROOT="{rollback_root}"',
|
|
'readonly LOCK_PATH="/tmp/agent99-host110-backup-runtime.lock"': f'readonly LOCK_PATH="{lock_path}"',
|
|
'expected_source_stage="/tmp/agent99-host110-backup-runtime-${RUN_ID}"': f'expected_source_stage="{source_stage}"',
|
|
'"wooo:wooo:755"': f'"{user_name}:{group_name}:755"',
|
|
'"wooo:wooo:700"': f'"{user_name}:{group_name}:700"',
|
|
'"wooo:wooo"': f'"{user_name}:{group_name}"',
|
|
}
|
|
for before, after in replacements.items():
|
|
assert before in executor_source
|
|
executor_source = executor_source.replace(before, after)
|
|
if use_zsh_adapter:
|
|
executor_source = executor_source.replace(
|
|
'"${!RUN_OWNED_TEMP_PATHS[@]}"',
|
|
'"${(@k)RUN_OWNED_TEMP_PATHS}"',
|
|
).replace(
|
|
'RUN_OWNED_TEMP_PATHS["$1"]=1',
|
|
'RUN_OWNED_TEMP_PATHS[$1]=1',
|
|
).replace('${PAYLOAD_FILES[0]}', '${PAYLOAD_FILES[1]}')
|
|
executor_path = source_stage / EXECUTOR.name
|
|
_write_executable(executor_path, executor_source)
|
|
|
|
verifier_source = VERIFIER.read_text(encoding="utf-8")
|
|
verifier_replacements = {
|
|
'EXPECTED_DESTINATION = Path("/backup/scripts")': f"EXPECTED_DESTINATION = Path({str(destination)!r})",
|
|
'EXPECTED_EXPORTER_DESTINATION = Path("/home/wooo/scripts/backup-health-textfile-exporter.py")': (
|
|
f"EXPECTED_EXPORTER_DESTINATION = Path({str(exporter_root / 'backup-health-textfile-exporter.py')!r})"
|
|
),
|
|
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")': f"RUNTIME_LOCK = Path({str(lock_path)!r})",
|
|
}
|
|
for before, after in verifier_replacements.items():
|
|
assert before in verifier_source
|
|
verifier_source = verifier_source.replace(before, after)
|
|
verifier_path = source_stage / VERIFIER.name
|
|
_write_executable(verifier_path, verifier_source)
|
|
|
|
payload_order = _executor_payload_order(executor_source)
|
|
rows = []
|
|
before_state: dict[str, tuple[bytes, int]] = {}
|
|
for name in payload_order:
|
|
source = (
|
|
ROOT / "scripts" / "ops" / name
|
|
if name == "backup-health-textfile-exporter.py"
|
|
else ROOT / "scripts" / "backup" / name
|
|
)
|
|
staged = source_stage / name
|
|
shutil.copy2(source, staged)
|
|
staged.chmod(0o755)
|
|
rows.append({"name": name, "sha256": _sha256(staged)})
|
|
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
|
target.write_text(f"prestate:{name}\n", encoding="utf-8")
|
|
target.chmod(0o755)
|
|
before_state[name] = (target.read_bytes(), target.stat().st_mode & 0o777)
|
|
|
|
source_revision = "a" * 40
|
|
manifest = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": "b" * 40,
|
|
"runId": run_id,
|
|
"executorSha256": _sha256(executor_path),
|
|
"verifierSha256": _sha256(verifier_path),
|
|
"files": rows,
|
|
}
|
|
(source_stage / "manifest.json").write_text(json.dumps(manifest), encoding="utf-8")
|
|
|
|
_write_executable(fake_bin / "hostname", "#!/bin/sh\nprintf '%s\\n' '192.168.0.110'\n")
|
|
_write_executable(fake_bin / "pgrep", "#!/bin/sh\nexit 1\n")
|
|
_write_executable(fake_bin / "flock", "#!/bin/sh\nexit 0\n")
|
|
_write_executable(fake_bin / "timeout", "#!/bin/sh\nexit 0\n")
|
|
_write_executable(
|
|
fake_bin / "readlink",
|
|
"""#!/usr/bin/env python3
|
|
import pathlib
|
|
import sys
|
|
if len(sys.argv) != 3 or sys.argv[1] != "-f":
|
|
raise SystemExit(2)
|
|
print(pathlib.Path(sys.argv[2]).resolve(strict=True))
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "stat",
|
|
"""#!/usr/bin/env python3
|
|
import grp
|
|
import os
|
|
import pwd
|
|
import sys
|
|
if len(sys.argv) != 4 or sys.argv[1] != "-c":
|
|
raise SystemExit(2)
|
|
row = os.stat(sys.argv[3])
|
|
mode = format(row.st_mode & 0o777, "o")
|
|
values = {
|
|
"%U:%G:%a": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}:{mode}",
|
|
"%U:%G": f"{pwd.getpwuid(row.st_uid).pw_name}:{grp.getgrgid(row.st_gid).gr_name}",
|
|
"%u:%g:%a": f"{row.st_uid}:{row.st_gid}:{mode}",
|
|
}
|
|
if sys.argv[2] not in values:
|
|
raise SystemExit(2)
|
|
print(values[sys.argv[2]])
|
|
""",
|
|
)
|
|
return {
|
|
"executor": executor_path,
|
|
"source_stage": source_stage,
|
|
"source_revision": source_revision,
|
|
"run_id": run_id,
|
|
"destination": destination,
|
|
"exporter_root": exporter_root,
|
|
"status_root": status_root,
|
|
"stage_root": stage_root,
|
|
"rollback_root": rollback_root,
|
|
"fake_bin": fake_bin,
|
|
"shell": shell_path,
|
|
"use_zsh_adapter": use_zsh_adapter,
|
|
"payload_order": payload_order,
|
|
"before_state": before_state,
|
|
}
|
|
|
|
|
|
def _run_executor_harness(harness: dict[str, object], **extra_env: str) -> subprocess.CompletedProcess[str]:
|
|
environment = os.environ.copy()
|
|
for name in (
|
|
"HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT",
|
|
"HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT",
|
|
"HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT",
|
|
"HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL",
|
|
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL",
|
|
"HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT",
|
|
"HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT",
|
|
):
|
|
environment.pop(name, None)
|
|
environment["HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS"] = "1"
|
|
environment.update(extra_env)
|
|
environment["PATH"] = f"{harness['fake_bin']}:{environment['PATH']}"
|
|
command = [str(harness["shell"])]
|
|
if harness["use_zsh_adapter"]:
|
|
environment["ZDOTDIR"] = str(harness["fake_bin"])
|
|
command.append("-f")
|
|
command.extend(
|
|
[
|
|
str(harness["executor"]),
|
|
"--mode",
|
|
"apply",
|
|
"--source-revision",
|
|
str(harness["source_revision"]),
|
|
"--run-id",
|
|
str(harness["run_id"]),
|
|
"--source-stage",
|
|
str(harness["source_stage"]),
|
|
]
|
|
)
|
|
return subprocess.run(
|
|
command,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
)
|
|
|
|
|
|
def _assert_prestate_restored(harness: dict[str, object]) -> None:
|
|
destination = Path(harness["destination"])
|
|
exporter_root = Path(harness["exporter_root"])
|
|
before_state = harness["before_state"]
|
|
assert isinstance(before_state, dict)
|
|
for name in harness["payload_order"]:
|
|
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
|
assert (target.read_bytes(), target.stat().st_mode & 0o777) == before_state[name]
|
|
|
|
|
|
def _assert_payload_committed(harness: dict[str, object]) -> None:
|
|
destination = Path(harness["destination"])
|
|
exporter_root = Path(harness["exporter_root"])
|
|
source_stage = Path(harness["source_stage"])
|
|
for name in harness["payload_order"]:
|
|
target = exporter_root / name if name == "backup-health-textfile-exporter.py" else destination / name
|
|
assert target.read_bytes() == (source_stage / name).read_bytes()
|
|
assert target.stat().st_mode & 0o777 == 0o755
|
|
|
|
|
|
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
|
|
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
|
assert '$TargetHost = "192.168.0.110"' in source
|
|
assert '$TargetUser = "wooo"' in source
|
|
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
|
|
assert 'keys\\agent99_ed25519' in source
|
|
assert '"credential.helper="' in source
|
|
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
|
|
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
|
|
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
|
|
assert 'executorSha256 = $executorDigest' in source
|
|
assert 'verifierSha256 = $verifierDigest' in source
|
|
assert '"scripts/ops/backup-health-textfile-exporter.py"' in source
|
|
assert "/home/wooo/awoooi" not in source
|
|
assert '"StrictHostKeyChecking=yes"' in source
|
|
assert '"NumberOfPasswordPrompts=0"' in source
|
|
assert '"ConnectionAttempts=1"' in source
|
|
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
|
|
assert "taskkill.exe /PID $process.Id /T /F" in source
|
|
assert "$process.Kill()" in source
|
|
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
|
|
assert '[string]$parsed.status -ne "verified"' in source
|
|
assert '-not [bool]$parsed.payloadCommitted' in source
|
|
assert '-not [bool]$parsed.stageCleanupVerified' in source
|
|
assert '-not [bool]$parsed.rollbackPrestateCleanupVerified' in source
|
|
assert '[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps"' in source
|
|
assert '[int64]$parsed.terminalExitCode -ne 0' in source
|
|
assert 'agent99-backup-runtime-terminal-$RunId.json' in source
|
|
assert '$property = $Object.PSObject.Properties[$Name]' in source
|
|
assert 'if ($null -eq $property) { return $false }' in source
|
|
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
|
|
assert 'Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage' in source
|
|
assert 'status = "committed_unknown"' in source
|
|
assert '"committed_verified_terminal_missing"' in source
|
|
assert 'retryApplyAllowed = $false' in source
|
|
assert 'throw "host110_apply_$([string]$apply.status)"' in source
|
|
|
|
|
|
def test_broker_required_field_contract_rejects_missing_fields_before_cast() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
field_guard = _powershell_function(source, "Test-Agent99JsonField")
|
|
common_contract = _powershell_function(source, "Convert-Agent99ExecutorDocument")
|
|
apply_contract = _powershell_function(source, "Assert-Agent99ApplyResultFields")
|
|
verifier_contract = _powershell_function(source, "Assert-Agent99VerifierFields")
|
|
committed_failure = _powershell_function(source, "Get-Agent99CommittedFailureResult")
|
|
|
|
assert '$property = $Object.PSObject.Properties[$Name]' in field_guard
|
|
assert 'if ($null -eq $property) { return $false }' in field_guard
|
|
for field in (
|
|
"schemaVersion",
|
|
"mode",
|
|
"ok",
|
|
"sourceRevision",
|
|
"sourceHead",
|
|
"runId",
|
|
"fileCount",
|
|
"backupScriptFileCount",
|
|
"backupHealthExporterIncluded",
|
|
):
|
|
assert re.search(rf"^\s+{field} = \"", common_contract, re.MULTILINE), field
|
|
for field in (
|
|
"status",
|
|
"payloadCommitted",
|
|
"rollbackAttempted",
|
|
"rollbackPerformed",
|
|
"rollbackVerified",
|
|
"zeroResidueVerified",
|
|
"zeroResidueScope",
|
|
"stageCleanupVerified",
|
|
"rollbackPrestateCleanupVerified",
|
|
"receipt",
|
|
"terminalReceipt",
|
|
"terminalReceiptPublished",
|
|
"terminalExitCode",
|
|
"deferredSignal",
|
|
"independentVerifierVerified",
|
|
"verifier",
|
|
):
|
|
assert re.search(rf"^\s+{field} = ", apply_contract, re.MULTILINE), field
|
|
for field in (
|
|
"schemaVersion",
|
|
"ok",
|
|
"sourceRevision",
|
|
"runId",
|
|
"fileCount",
|
|
"backupScriptFileCount",
|
|
"backupHealthExporterIncluded",
|
|
"remoteWritePerformed",
|
|
"selfIdentityVerified",
|
|
):
|
|
assert re.search(rf"^\s+{field} = \"", verifier_contract, re.MULTILINE), field
|
|
assert 'status -eq "cleanup_pending"' in committed_failure
|
|
assert 'status -eq "committed_signal_deferred"' in committed_failure
|
|
assert 'verifier = $VerifierKind' in apply_contract
|
|
|
|
|
|
def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_write() -> None:
|
|
source = BROKER_CONTRACT_REPLAY.read_text(encoding="utf-8")
|
|
|
|
assert "FunctionDefinitionAst" in source
|
|
for function_name in (
|
|
"Test-Agent99JsonField",
|
|
"Assert-Agent99JsonFields",
|
|
"Assert-Agent99ExactJsonFields",
|
|
"Convert-Agent99ExecutorDocument",
|
|
"Assert-Agent99ApplyResultFields",
|
|
"Assert-Agent99VerifierFields",
|
|
"Get-Agent99ExecutorResult",
|
|
"Get-Agent99CommittedFailureResult",
|
|
"Assert-Agent99VerifierEvidence",
|
|
"Assert-Agent99PreflightEvidence",
|
|
"Get-Agent99TransportReceiptReadback",
|
|
"New-Agent99TransportLossReconciliation",
|
|
"Assert-Agent99ExecutorEvidence",
|
|
"Assert-Agent99ReconciliationEvidence",
|
|
"Assert-Agent99ExistingBrokerEvidence",
|
|
):
|
|
assert f'"{function_name}"' in source
|
|
assert "$mutated.PSObject.Properties.Remove($field)" in source
|
|
assert "$mutated.verifier.PSObject.Properties.Remove($field)" in source
|
|
assert 'status = "cleanup_pending"' in source
|
|
assert 'status = "committed_signal_deferred"' in source
|
|
assert 'status = "terminal_receipt_failed"' in source
|
|
assert 'ok = "false"' in source
|
|
assert 'status -ne "committed_verified_terminal_missing"' in source
|
|
assert 'status -ne "committed_unknown"' in source
|
|
assert "existingEvidenceRejections" in source
|
|
assert "malformed_failure_not_upgraded" in source
|
|
assert "remoteWritePerformed = $false" in source
|
|
assert "secretValuesRead = $false" in source
|
|
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
|
|
assert forbidden not in source
|
|
|
|
|
|
def test_existing_broker_evidence_replay_is_exact_typed_and_nested_fail_closed() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
exact_guard = _powershell_function(source, "Assert-Agent99ExactJsonFields")
|
|
existing = _powershell_function(source, "Assert-Agent99ExistingBrokerEvidence")
|
|
reconciliation = _powershell_function(source, "Assert-Agent99ReconciliationEvidence")
|
|
|
|
assert "$properties.Count -ne $Contract.Count" in exact_guard
|
|
assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard
|
|
for field in (
|
|
"schemaVersion",
|
|
"ok",
|
|
"status",
|
|
"mode",
|
|
"sourceRevision",
|
|
"sourceHead",
|
|
"runId",
|
|
"targetHost",
|
|
"sourceTransport",
|
|
"executor",
|
|
"verifier",
|
|
"decisionProvider",
|
|
"criticProvider",
|
|
"agentAction",
|
|
"check",
|
|
"apply",
|
|
"verify",
|
|
"cleanupVerified",
|
|
"errorCode",
|
|
"elapsedSeconds",
|
|
"evidence",
|
|
"secretValuesRead",
|
|
"rawSessionStored",
|
|
):
|
|
assert re.search(rf"^\s+{field} = \"", existing, re.MULTILINE), field
|
|
assert "Assert-Agent99PreflightEvidence $Existing.check" in existing
|
|
assert "Assert-Agent99ExecutorEvidence $Existing.apply" in existing
|
|
assert "Assert-Agent99ReconciliationEvidence $Existing.apply" in existing
|
|
assert "Assert-Agent99VerifierEvidence $Existing.verify" in existing
|
|
assert 'throw "existing_broker_apply_failure_evidence_missing"' in existing
|
|
assert 'retryApplyAllowed = "Boolean"' in reconciliation
|
|
assert "host110_existing_reconciliation_payload_truth_mismatch" in reconciliation
|
|
|
|
|
|
def test_transport_loss_reconciliation_is_read_only_run_bound_and_never_retries_apply() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
|
|
reconcile = _powershell_function(source, "Invoke-Agent99TransportLossReconciliation")
|
|
classify = _powershell_function(source, "New-Agent99TransportLossReconciliation")
|
|
|
|
for expected in (
|
|
"agent99-backup-runtime-deploy-{args.run_id}.json",
|
|
"agent99-backup-runtime-terminal-{args.run_id}.json",
|
|
'set(document) == set(contract)',
|
|
'payload["status"] == "payload_verified"',
|
|
'terminal["payloadReceipt"] == str(payload_path)',
|
|
'"remoteWritePerformed": False',
|
|
'"secretValuesRead": False',
|
|
):
|
|
assert expected in reader
|
|
assert "Invoke-Agent99BoundedSsh $command 60" in reconcile
|
|
assert "Invoke-Agent99NoWriteVerifier $SourcePackage" in reconcile
|
|
assert 'status = "committed_unknown"' in classify
|
|
assert '"committed_verified_terminal_missing"' in classify
|
|
assert '"committed_verified_transport_lost"' in classify
|
|
assert "retryApplyAllowed = $false" in classify
|
|
for forbidden in ("rm ", "mv ", "install ", "scp", "--mode apply"):
|
|
assert forbidden not in reconcile
|
|
|
|
|
|
def test_transport_receipt_readback_rejects_extra_and_wrong_typed_documents(tmp_path: Path) -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
|
|
status_root = tmp_path / "status"
|
|
status_root.mkdir()
|
|
run_id = "transport-loss-replay"
|
|
source_revision = "a" * 40
|
|
source_head = "b" * 40
|
|
verifier_sha256 = "c" * 64
|
|
payload_path = status_root / f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
terminal_path = status_root / f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
payload = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
|
|
"status": "payload_verified",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": source_head,
|
|
"runId": run_id,
|
|
"fileCount": 18,
|
|
"backupScriptFileCount": 17,
|
|
"backupHealthExporterIncluded": True,
|
|
"rollbackAttempted": False,
|
|
"rollbackPerformed": False,
|
|
"rollbackVerified": False,
|
|
"zeroResidueVerified": True,
|
|
"zeroResidueScope": "run_owned_destination_temps",
|
|
"verifierSha256": verifier_sha256,
|
|
"executorHost": "192.168.0.110",
|
|
"productionServiceRestarted": False,
|
|
"secretValuesRead": False,
|
|
"writtenAt": 1,
|
|
}
|
|
terminal = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
|
|
"status": "verified",
|
|
"ok": True,
|
|
"payloadCommitted": True,
|
|
"payloadReceipt": str(payload_path),
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": source_head,
|
|
"runId": run_id,
|
|
"fileCount": 18,
|
|
"backupScriptFileCount": 17,
|
|
"backupHealthExporterIncluded": True,
|
|
"stageCleanupVerified": True,
|
|
"rollbackPrestateCleanupVerified": True,
|
|
"cleanupVerified": True,
|
|
"independentVerifierVerified": True,
|
|
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
|
|
"deferredSignal": None,
|
|
"terminalExitCode": 0,
|
|
"verifierSha256": verifier_sha256,
|
|
"executorHost": "192.168.0.110",
|
|
"productionServiceRestarted": False,
|
|
"secretValuesRead": False,
|
|
"writtenAt": 2,
|
|
}
|
|
|
|
def run_readback() -> dict[str, object]:
|
|
result = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
"-",
|
|
"--status-root",
|
|
str(status_root),
|
|
"--source-revision",
|
|
source_revision,
|
|
"--source-head",
|
|
source_head,
|
|
"--run-id",
|
|
run_id,
|
|
"--verifier-sha256",
|
|
verifier_sha256,
|
|
],
|
|
input=reader,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
assert result.returncode == 0, result.stderr
|
|
return json.loads(result.stdout)
|
|
|
|
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
|
|
valid = run_readback()
|
|
assert valid["payloadState"] == "valid"
|
|
assert valid["terminalState"] == "valid"
|
|
assert valid["terminalStatus"] == "verified"
|
|
|
|
payload["unexpected"] = True
|
|
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
assert run_readback()["payloadState"] == "invalid"
|
|
payload.pop("unexpected")
|
|
payload["writtenAt"] = "1"
|
|
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
assert run_readback()["payloadState"] == "invalid"
|
|
payload["writtenAt"] = 1
|
|
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
|
terminal["ok"] = "false"
|
|
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
|
|
assert run_readback()["terminalState"] == "invalid"
|
|
|
|
|
|
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
|
|
broker = BROKER.read_text(encoding="utf-8")
|
|
executor = EXECUTOR.read_text(encoding="utf-8")
|
|
|
|
runtime_paths = _powershell_array(broker, "RuntimeRelativePaths")
|
|
broker_payload_basenames = tuple(Path(path).name for path in runtime_paths)
|
|
executor_payload_basenames = _executor_payload_order(executor)
|
|
executor_path = _powershell_string(broker, "ExecutorRelativePath")
|
|
verifier_path = _powershell_string(broker, "VerifierRelativePath")
|
|
manifest_match = re.search(
|
|
r'\$manifestPath\s*=\s*Join-Path\s+\$sourceRoot\s+"([^"]+)"',
|
|
broker,
|
|
)
|
|
assert manifest_match is not None
|
|
manifest_name = manifest_match.group(1)
|
|
|
|
assert len(broker_payload_basenames) == 18
|
|
assert len(executor_payload_basenames) == 18
|
|
assert set(broker_payload_basenames) == set(executor_payload_basenames)
|
|
assert broker_payload_basenames == executor_payload_basenames
|
|
staged_basenames = (
|
|
*broker_payload_basenames,
|
|
Path(executor_path).name,
|
|
Path(verifier_path).name,
|
|
manifest_name,
|
|
)
|
|
assert len(staged_basenames) == 21
|
|
assert len(set(staged_basenames)) == 21
|
|
assert manifest_name == "manifest.json"
|
|
assert '$localPaths += $executorPath, $verifierPath, $ManifestPath' in broker
|
|
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage' in broker
|
|
assert f'$SOURCE_STAGE/{manifest_name}' in executor
|
|
assert f'$SOURCE_STAGE/{Path(verifier_path).name}' in executor
|
|
assert Path(executor_path).name == "host110-backup-runtime-executor.sh"
|
|
|
|
|
|
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
|
|
assert 'RedirectStandardInput = $StandardInputPath' in source
|
|
assert "python3 - --manifest-base64" in source
|
|
assert 'remoteWritePerformed' in source
|
|
assert '[IO.File]::Move($temporary, $Path)' in source
|
|
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
|
|
assert 'run_identity_conflict' in source
|
|
assert 'run_identity_evidence_exists' in source
|
|
assert 'decisionProvider = "deterministic_only"' in source
|
|
assert 'criticProvider = "deterministic_only"' in source
|
|
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
|
|
|
|
|
|
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
|
|
source = EXECUTOR.read_text(encoding="utf-8")
|
|
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
|
|
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
|
|
|
|
assert runtime_files == EXPECTED_RUNTIME_FILES
|
|
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
|
|
assert 'readonly EXPECTED_USER="wooo"' in source
|
|
assert 'readonly DEST_ROOT="/backup/scripts"' in source
|
|
assert 'readonly EXPORTER_ROOT="/home/wooo/scripts"' in source
|
|
assert 'readonly EXPORTER_FILE="backup-health-textfile-exporter.py"' in source
|
|
assert 'readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")' in source
|
|
assert '--source-stage' in source
|
|
assert 'document.get("executorSha256"' in source
|
|
assert 'document.get("verifierSha256"' in source
|
|
assert 'executor_identity_failed' in source
|
|
assert 'verifier_identity_failed' in source
|
|
assert 'flock -x -w 30 9' in source
|
|
assert 'legacy_backup_runtime_active' in source
|
|
assert source.index("for name in \"${PAYLOAD_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
|
|
assert 'rollback_unverified' in source
|
|
assert 'failed_rolled_back' in source
|
|
assert 'ROLLBACK_ATTEMPTED=1' in source
|
|
assert 'ROLLBACK_PERFORMED=1' in source
|
|
assert '"rollbackAttempted": rollback_attempted' in source
|
|
assert '"rollbackPerformed": rollback_performed' in source
|
|
assert '"rollbackVerified": rollback_verified' in source
|
|
assert '"zeroResidueVerified": zero_residue_verified' in source
|
|
assert '"zeroResidueScope": "run_owned_destination_temps"' in source
|
|
assert 'cleanup_run_owned_temps' in source
|
|
assert 'verify_run_owned_temp_absence' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT' in source
|
|
assert 'HOST110_BACKUP_RUNTIME_ENABLE_TEST_FAULTS' in source
|
|
assert '[ "$DEST_ROOT" != "/backup/scripts" ]' in source
|
|
assert 'fsync_rollback_prestate || return 75' in source
|
|
assert 'fsync_destination_state || fail "post_apply_durability_failed"' in source
|
|
assert 'working_digest "$dest_path"' in source
|
|
assert 'PREVIOUS_METADATA' in source
|
|
assert "stat -c '%u:%g:%a'" in source
|
|
assert 'os.link(temporary, path)' in source
|
|
assert 'os.fsync(handle.fileno())' in source
|
|
assert 'os.fsync(directory_fd)' in source
|
|
assert 'hashlib.sha256(readback).hexdigest()' in source
|
|
assert source.index("COMMIT_CRITICAL=1") < source.index('write_receipt "payload_verified"')
|
|
assert source.index('write_receipt "payload_verified"') < source.index("\nAPPLY_COMPLETE=1")
|
|
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex("\nCOMMIT_CRITICAL=0")
|
|
assert source.index("\nAPPLY_COMPLETE=1") < source.rindex(
|
|
'cleanup_postcommit_directory "rollback_prestate"'
|
|
)
|
|
assert 'handle_agent99_signal HUP 129' in source
|
|
assert 'handle_agent99_signal TERM 143' in source
|
|
assert source.index("trap '' HUP INT TERM") < source.index('write_terminal_receipt "$terminal_status"')
|
|
assert 'write_terminal_receipt "$terminal_status"' in source
|
|
assert 'terminal_status="cleanup_pending"' in source
|
|
assert 'terminal_status="committed_signal_deferred"' in source
|
|
assert 'run_identity_receipt_exists' in source
|
|
assert 'run_identity_stage_exists' in source
|
|
assert 'run_identity_rollback_exists' in source
|
|
assert 'verify-host110-backup-runtime.py' in source
|
|
assert 'selfIdentityVerified' in source
|
|
assert '"backupScriptFileCount": 17' in source
|
|
assert '"backupHealthExporterIncluded": True' in source
|
|
assert '"productionServiceRestarted": False' in source
|
|
assert '"secretValuesRead": False' in source
|
|
assert "sudo " not in source
|
|
assert "docker " not in source
|
|
assert "systemctl " not in source
|
|
|
|
|
|
def test_apply_success_publishes_durable_receipt_before_prestate_cleanup(tmp_path: Path) -> None:
|
|
run_id = "apply-success"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(harness)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
output = json.loads(result.stdout)
|
|
assert output["ok"] is True
|
|
assert output["rollbackAttempted"] is False
|
|
assert output["rollbackPerformed"] is False
|
|
assert output["rollbackVerified"] is False
|
|
assert output["zeroResidueVerified"] is True
|
|
assert output["stageCleanupVerified"] is True
|
|
assert output["rollbackPrestateCleanupVerified"] is True
|
|
assert output["status"] == "verified"
|
|
assert output["payloadCommitted"] is True
|
|
assert output["runId"] == run_id
|
|
assert output["terminalExitCode"] == 0
|
|
assert output["terminalReceiptPublished"] is True
|
|
assert output["deferredSignal"] is None
|
|
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
assert receipt["status"] == "payload_verified"
|
|
assert receipt["rollbackAttempted"] is False
|
|
assert receipt["rollbackPerformed"] is False
|
|
assert receipt["rollbackVerified"] is False
|
|
assert receipt["zeroResidueVerified"] is True
|
|
assert receipt["zeroResidueScope"] == "run_owned_destination_temps"
|
|
terminal_path = Path(harness["status_root"]) / f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
terminal = json.loads(terminal_path.read_text(encoding="utf-8"))
|
|
assert terminal["status"] == "verified"
|
|
assert terminal["ok"] is True
|
|
assert terminal["payloadCommitted"] is True
|
|
assert terminal["stageCleanupVerified"] is True
|
|
assert terminal["rollbackPrestateCleanupVerified"] is True
|
|
assert terminal["terminalExitCode"] == 0
|
|
assert terminal["zeroResidueScope"] == "run_owned_destination_temps_and_internal_stage_prestate"
|
|
assert output["terminalReceipt"] == str(terminal_path)
|
|
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
|
assert not (Path(harness["stage_root"]) / run_id).exists()
|
|
_assert_payload_committed(harness)
|
|
|
|
|
|
@pytest.mark.parametrize(("signal_name", "exit_code"), [("TERM", 143), ("HUP", 129), ("INT", 130)])
|
|
def test_commit_window_signal_is_deferred_until_payload_and_terminal_are_consistent(
|
|
tmp_path: Path,
|
|
signal_name: str,
|
|
exit_code: int,
|
|
) -> None:
|
|
run_id = f"commit-signal-{signal_name.lower()}"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_COMMIT_SIGNAL=signal_name,
|
|
)
|
|
|
|
assert result.returncode == exit_code, result.stderr
|
|
output = json.loads(result.stdout)
|
|
assert output["ok"] is False
|
|
assert output["status"] == "committed_signal_deferred"
|
|
assert output["payloadCommitted"] is True
|
|
assert output["runId"] == run_id
|
|
assert output["deferredSignal"] == signal_name
|
|
assert output["terminalExitCode"] == exit_code
|
|
assert output["terminalReceiptPublished"] is True
|
|
assert output["stageCleanupVerified"] is True
|
|
assert output["rollbackPrestateCleanupVerified"] is True
|
|
_assert_payload_committed(harness)
|
|
payload_receipt = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert payload_receipt["status"] == "payload_verified"
|
|
terminal = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert terminal["status"] == "committed_signal_deferred"
|
|
assert terminal["ok"] is False
|
|
assert terminal["deferredSignal"] == signal_name
|
|
assert terminal["terminalExitCode"] == exit_code
|
|
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
|
assert not (Path(harness["stage_root"]) / run_id).exists()
|
|
|
|
|
|
@pytest.mark.parametrize("signal_name", ["TERM", "HUP", "INT"])
|
|
def test_terminal_publication_masks_late_signal_without_receipt_stdout_exit_divergence(
|
|
tmp_path: Path,
|
|
signal_name: str,
|
|
) -> None:
|
|
run_id = f"terminal-signal-{signal_name.lower()}"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_SIGNAL=signal_name,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
output = json.loads(result.stdout)
|
|
assert output["ok"] is True
|
|
assert output["status"] == "verified"
|
|
assert output["payloadCommitted"] is True
|
|
assert output["runId"] == run_id
|
|
assert output["deferredSignal"] is None
|
|
assert output["terminalExitCode"] == 0
|
|
assert output["terminalReceiptPublished"] is True
|
|
terminal = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert terminal["ok"] is True
|
|
assert terminal["status"] == output["status"]
|
|
assert terminal["deferredSignal"] == output["deferredSignal"]
|
|
assert terminal["terminalExitCode"] == output["terminalExitCode"]
|
|
_assert_payload_committed(harness)
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
|
|
def test_terminal_receipt_fault_is_nonzero_with_payload_receipt_only(
|
|
tmp_path: Path,
|
|
fault: str,
|
|
) -> None:
|
|
run_id = f"terminal-receipt-{fault}"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_TERMINAL_RECEIPT_FAULT=fault,
|
|
)
|
|
|
|
assert result.returncode == 76
|
|
assert '"ok": true' not in result.stdout.lower()
|
|
assert "terminal_receipt_failed" in result.stderr
|
|
output = json.loads(result.stdout)
|
|
assert output["ok"] is False
|
|
assert output["status"] == "terminal_receipt_failed"
|
|
assert output["payloadCommitted"] is True
|
|
assert output["runId"] == run_id
|
|
assert output["terminalReceiptPublished"] is False
|
|
assert output["terminalExitCode"] == 76
|
|
_assert_payload_committed(harness)
|
|
payload_receipt = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert payload_receipt["status"] == "payload_verified"
|
|
assert not (
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
).exists()
|
|
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
|
assert not (Path(harness["stage_root"]) / run_id).exists()
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
("fault", "stage_clean", "rollback_clean"),
|
|
[("stage", False, True), ("rollback_prestate", True, False)],
|
|
)
|
|
def test_postcommit_cleanup_failure_is_nonzero_and_never_false_green(
|
|
tmp_path: Path,
|
|
fault: str,
|
|
stage_clean: bool,
|
|
rollback_clean: bool,
|
|
) -> None:
|
|
run_id = f"cleanup-{fault}"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_CLEANUP_FAULT=fault,
|
|
)
|
|
|
|
assert result.returncode == 75, result.stderr
|
|
assert '"ok": true' not in result.stdout.lower()
|
|
output = json.loads(result.stdout)
|
|
assert output["ok"] is False
|
|
assert output["status"] == "cleanup_pending"
|
|
assert output["payloadCommitted"] is True
|
|
assert output["runId"] == run_id
|
|
assert output["stageCleanupVerified"] is stage_clean
|
|
assert output["rollbackPrestateCleanupVerified"] is rollback_clean
|
|
assert output["terminalReceiptPublished"] is True
|
|
_assert_payload_committed(harness)
|
|
payload_receipt = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert payload_receipt["status"] == "payload_verified"
|
|
terminal = json.loads(
|
|
(
|
|
Path(harness["status_root"])
|
|
/ f"agent99-backup-runtime-terminal-{run_id}.json"
|
|
).read_text(encoding="utf-8")
|
|
)
|
|
assert terminal["status"] == "cleanup_pending"
|
|
assert terminal["ok"] is False
|
|
assert terminal["payloadCommitted"] is True
|
|
assert terminal["stageCleanupVerified"] is stage_clean
|
|
assert terminal["rollbackPrestateCleanupVerified"] is rollback_clean
|
|
assert (Path(harness["stage_root"]) / run_id).exists() is (not stage_clean)
|
|
assert (Path(harness["rollback_root"]) / run_id).exists() is (not rollback_clean)
|
|
|
|
|
|
@pytest.mark.parametrize("fault", ["write", "link", "fsync", "readback"])
|
|
def test_receipt_fault_rolls_back_without_false_success_or_premature_prestate_cleanup(
|
|
tmp_path: Path,
|
|
fault: str,
|
|
) -> None:
|
|
run_id = f"receipt-{fault}"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_RECEIPT_FAULT=fault,
|
|
)
|
|
assert result.returncode != 0
|
|
assert '"ok": true' not in result.stdout.lower()
|
|
assert "durable_receipt_failed" in result.stderr
|
|
_assert_prestate_restored(harness)
|
|
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
assert receipt["status"] == "failed_rolled_back"
|
|
assert receipt["rollbackAttempted"] is True
|
|
assert receipt["rollbackPerformed"] is True
|
|
assert receipt["rollbackVerified"] is True
|
|
assert receipt["zeroResidueVerified"] is True
|
|
assert not (Path(harness["rollback_root"]) / run_id).exists()
|
|
assert not (Path(harness["stage_root"]) / run_id).exists()
|
|
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
|
assert not list(parent.glob(f".*.agent99-*{run_id}"))
|
|
|
|
|
|
def test_apply_temp_fault_is_removed_and_zero_residue_is_verified(tmp_path: Path) -> None:
|
|
run_id = "apply-temp-cleanup"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert '"ok": true' not in result.stdout.lower()
|
|
assert "fault_injected_after_temp_install" in result.stderr
|
|
_assert_prestate_restored(harness)
|
|
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
assert receipt["status"] == "failed_rolled_back"
|
|
assert receipt["rollbackAttempted"] is True
|
|
assert receipt["rollbackPerformed"] is True
|
|
assert receipt["rollbackVerified"] is True
|
|
assert receipt["zeroResidueVerified"] is True
|
|
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
|
assert not list(parent.glob(f".*.agent99-*{run_id}"))
|
|
|
|
|
|
def test_temp_cleanup_fault_is_rollback_unverified_and_preserves_prestate(tmp_path: Path) -> None:
|
|
run_id = "temp-residue-unverified"
|
|
harness = _build_executor_harness(tmp_path, run_id)
|
|
|
|
result = _run_executor_harness(
|
|
harness,
|
|
HOST110_BACKUP_RUNTIME_TEST_APPLY_FAULT="after_temp_install",
|
|
HOST110_BACKUP_RUNTIME_TEST_ROLLBACK_FAULT="preserve_first_temp",
|
|
)
|
|
|
|
assert result.returncode != 0
|
|
assert '"ok": true' not in result.stdout.lower()
|
|
_assert_prestate_restored(harness)
|
|
receipt_path = Path(harness["status_root"]) / f"agent99-backup-runtime-deploy-{run_id}.json"
|
|
receipt = json.loads(receipt_path.read_text(encoding="utf-8"))
|
|
assert receipt["status"] == "rollback_unverified"
|
|
assert receipt["rollbackAttempted"] is True
|
|
assert receipt["rollbackPerformed"] is True
|
|
assert receipt["rollbackVerified"] is False
|
|
assert receipt["zeroResidueVerified"] is False
|
|
assert (Path(harness["rollback_root"]) / run_id / "prestate.tsv").is_file()
|
|
residue = []
|
|
for parent in (Path(harness["destination"]), Path(harness["exporter_root"])):
|
|
residue.extend(parent.glob(f".*.agent99-*{run_id}"))
|
|
assert residue
|
|
|
|
|
|
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
|
|
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
|
|
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
|
|
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
|
|
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
|
|
|
for source in (common, restore, offsite):
|
|
assert "/tmp/agent99-host110-backup-runtime.lock" in source
|
|
assert "flock -s -w 60" in source
|
|
assert "realpath -e" in source
|
|
assert "readlink -f" in source
|
|
assert "identity unresolved" in source
|
|
assert "/proc/$$/fd/197" in source
|
|
assert "/dev/fd/197" in source
|
|
assert "inherited fd 197 identity mismatch" in source
|
|
assert source.index("backup_runtime_inherited_fd_status") < source.index("exec 197>>")
|
|
assert 'resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}"' in common
|
|
assert '${PWD}/${raw_path}' in common
|
|
assert '[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0' not in common
|
|
for source in (restore, offsite):
|
|
assert 'resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}"' in source
|
|
assert '&& [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]' not in source
|
|
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
|
|
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" not in archive
|
|
assert 'Path(f"/proc/{os.getpid()}/fd")' in archive
|
|
assert 'Path("/dev/fd/197")' in archive
|
|
assert "os.dup(197)" in archive
|
|
|
|
|
|
def test_common_shared_lock_preserves_resolved_nonproduction_fixture_behavior(tmp_path: Path) -> None:
|
|
gate_source = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8").split(
|
|
"# WOOO AIOps - 備份共用函式庫",
|
|
1,
|
|
)[0]
|
|
gate_fixture = tmp_path / "common-gate.sh"
|
|
_write_executable(gate_fixture, gate_source)
|
|
wrapper = tmp_path / "fixture-entrypoint.sh"
|
|
_write_executable(
|
|
wrapper,
|
|
"#!/usr/bin/env bash\n"
|
|
"set -euo pipefail\n"
|
|
f"source {str(gate_fixture)!r}\n"
|
|
"printf 'NONPRODUCTION_FIXTURE_OK=1\\n'\n",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[str(wrapper)],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
|
|
assert result.returncode == 0, result.stderr
|
|
assert "NONPRODUCTION_FIXTURE_OK=1" in result.stdout
|
|
|
|
|
|
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
|
@pytest.mark.parametrize("invocation", ["absolute", "relative", "symlink"])
|
|
def test_backup_runtime_shared_lock_blocks_absolute_relative_and_symlink_invocation(
|
|
tmp_path: Path,
|
|
entrypoint: str,
|
|
invocation: str,
|
|
) -> None:
|
|
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
|
|
tmp_path,
|
|
entrypoint,
|
|
)
|
|
|
|
command_path = str(script_path)
|
|
cwd = runtime_root
|
|
if invocation == "relative":
|
|
command_path = f"./{script_path.name}"
|
|
elif invocation == "symlink":
|
|
link_path = tmp_path / f"{entrypoint}-entrypoint-link"
|
|
link_path.symlink_to(script_path)
|
|
command_path = str(link_path)
|
|
|
|
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
|
|
lock_handle = lock_path.open("a+")
|
|
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
try:
|
|
result = subprocess.run(
|
|
[str(command_path)],
|
|
cwd=cwd,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
timeout=3,
|
|
)
|
|
finally:
|
|
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
|
|
lock_handle.close()
|
|
|
|
assert result.returncode == 75
|
|
assert "backup runtime deployment is active" in result.stderr
|
|
assert "BODY_REACHED=1" not in result.stdout
|
|
|
|
|
|
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
|
def test_backup_runtime_shared_lock_reuses_exact_inherited_fd_without_reopen(
|
|
tmp_path: Path,
|
|
entrypoint: str,
|
|
) -> None:
|
|
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
|
|
tmp_path,
|
|
entrypoint,
|
|
reject_fd_reopen=True,
|
|
body_sleep_seconds=0.5,
|
|
)
|
|
wrapper = tmp_path / "run-with-inherited-lock.sh"
|
|
_write_executable(
|
|
wrapper,
|
|
"#!/usr/bin/env bash\n"
|
|
"set -euo pipefail\n"
|
|
f"exec 197>>{str(lock_path)!r}\n"
|
|
"flock -s -w 1 197\n"
|
|
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=forged_external_value\n"
|
|
f"exec {str(script_path)!r}\n",
|
|
)
|
|
|
|
process = subprocess.Popen(
|
|
[str(wrapper)],
|
|
cwd=runtime_root,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
text=True,
|
|
env=environment,
|
|
)
|
|
assert process.stdout is not None
|
|
assert process.stdout.readline().strip() == "BODY_REACHED=1"
|
|
contender = lock_path.open("a+")
|
|
try:
|
|
with pytest.raises(BlockingIOError):
|
|
fcntl.flock(contender.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
finally:
|
|
contender.close()
|
|
stdout, stderr = process.communicate(timeout=3)
|
|
|
|
assert process.returncode == 0, stderr
|
|
assert "FD_REOPENED=1" not in stderr
|
|
assert stdout == ""
|
|
|
|
|
|
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
|
def test_backup_runtime_shared_lock_rejects_inherited_fd_for_another_file(
|
|
tmp_path: Path,
|
|
entrypoint: str,
|
|
) -> None:
|
|
script_path, runtime_root, _, environment = _build_shared_lock_gate_fixture(tmp_path, entrypoint)
|
|
wrong_lock = tmp_path / "wrong-runtime.lock"
|
|
wrapper = tmp_path / "run-with-wrong-inherited-lock.sh"
|
|
_write_executable(
|
|
wrapper,
|
|
"#!/usr/bin/env bash\n"
|
|
"set -euo pipefail\n"
|
|
f"exec 197>>{str(wrong_lock)!r}\n"
|
|
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=1\n"
|
|
f"exec {str(script_path)!r}\n",
|
|
)
|
|
|
|
result = subprocess.run(
|
|
[str(wrapper)],
|
|
cwd=runtime_root,
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
timeout=3,
|
|
)
|
|
|
|
assert result.returncode == 69
|
|
assert "inherited fd 197 identity mismatch" in result.stderr
|
|
assert "BODY_REACHED=1" not in result.stdout
|
|
|
|
|
|
def test_host188_archive_lock_ignores_forged_env_and_validates_inherited_fd(tmp_path: Path) -> None:
|
|
runtime_root = tmp_path / "runtime-scripts"
|
|
runtime_root.mkdir()
|
|
lock_path = tmp_path / "runtime.lock"
|
|
archive_path = runtime_root / "verify-host188-products-archive.py"
|
|
archive_source = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
|
archive_source = archive_source.replace('Path("/backup/scripts")', f"Path({str(runtime_root)!r})")
|
|
archive_source = archive_source.replace(
|
|
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")',
|
|
f"RUNTIME_LOCK = Path({str(lock_path)!r})",
|
|
)
|
|
archive_source = archive_source.replace("time.monotonic() + 60", "time.monotonic() + 0.2")
|
|
_write_executable(archive_path, archive_source)
|
|
runner = tmp_path / "archive-lock-replay.py"
|
|
runner.write_text(
|
|
"""import fcntl
|
|
import importlib.util
|
|
import os
|
|
import pathlib
|
|
import sys
|
|
|
|
archive_path = pathlib.Path(sys.argv[1])
|
|
lock_path = pathlib.Path(sys.argv[2])
|
|
mode = sys.argv[3]
|
|
spec = importlib.util.spec_from_file_location("host188_archive_lock_replay", archive_path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
if mode in {"exact", "wrong"}:
|
|
selected = lock_path if mode == "exact" else lock_path.with_name("wrong-runtime.lock")
|
|
selected.touch()
|
|
inherited = selected.open("a+")
|
|
fcntl.flock(inherited.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
|
os.dup2(inherited.fileno(), 197)
|
|
try:
|
|
handle = module.acquire_runtime_lock()
|
|
except RuntimeError as exc:
|
|
print(str(exc))
|
|
raise SystemExit(69)
|
|
else:
|
|
print("LOCK_OK=1")
|
|
if handle is not None:
|
|
handle.close()
|
|
""",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
environment = os.environ.copy()
|
|
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
|
|
lock_handle = lock_path.open("a+")
|
|
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
try:
|
|
forged = subprocess.run(
|
|
[sys.executable, str(runner), str(archive_path), str(lock_path), "forged"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
timeout=3,
|
|
)
|
|
finally:
|
|
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
|
|
lock_handle.close()
|
|
exact = subprocess.run(
|
|
[sys.executable, str(runner), str(archive_path), str(lock_path), "exact"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
timeout=3,
|
|
)
|
|
wrong = subprocess.run(
|
|
[sys.executable, str(runner), str(archive_path), str(lock_path), "wrong"],
|
|
check=False,
|
|
capture_output=True,
|
|
text=True,
|
|
env=environment,
|
|
timeout=3,
|
|
)
|
|
|
|
assert forged.returncode == 69
|
|
assert "backup_runtime_deployment_active" in forged.stdout
|
|
assert exact.returncode == 0, exact.stdout + exact.stderr
|
|
assert "LOCK_OK=1" in exact.stdout
|
|
assert wrong.returncode == 69
|
|
assert "backup_runtime_inherited_fd_mismatch" in wrong.stdout
|
|
|
|
|
|
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
|
|
verifier = _load_verifier()
|
|
destination = tmp_path / "scripts"
|
|
destination.mkdir()
|
|
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
|
rows = []
|
|
for name in verifier.EXPECTED_BACKUP_FILES:
|
|
path = destination / name
|
|
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
|
path.chmod(0o755)
|
|
rows.append({"name": name, "sha256": _sha256(path)})
|
|
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
|
exporter_destination.chmod(0o755)
|
|
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
|
verifier_digest = _sha256(VERIFIER)
|
|
source_revision = "a" * 40
|
|
run_id = "unit-exact"
|
|
manifest = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": "b" * 40,
|
|
"runId": run_id,
|
|
"executorSha256": "c" * 64,
|
|
"verifierSha256": verifier_digest,
|
|
"files": rows,
|
|
}
|
|
manifest_path = tmp_path / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
before = {
|
|
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
|
for path in [*destination.iterdir(), exporter_destination]
|
|
}
|
|
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
|
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
str(VERIFIER),
|
|
"--manifest",
|
|
str(manifest_path),
|
|
"--destination",
|
|
str(destination),
|
|
"--exporter-destination",
|
|
str(exporter_destination),
|
|
"--source-revision",
|
|
source_revision,
|
|
"--run-id",
|
|
run_id,
|
|
"--expected-verifier-sha256",
|
|
verifier_digest,
|
|
],
|
|
)
|
|
|
|
verifier.main()
|
|
|
|
result = json.loads(capsys.readouterr().out)
|
|
after = {
|
|
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
|
for path in [*destination.iterdir(), exporter_destination]
|
|
}
|
|
assert result["ok"] is True
|
|
assert result["fileCount"] == 18
|
|
assert result["backupScriptFileCount"] == 17
|
|
assert result["backupHealthExporterIncluded"] is True
|
|
assert result["remoteWritePerformed"] is False
|
|
assert result["selfIdentityVerified"] is True
|
|
assert before == after
|
|
|
|
|
|
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
|
|
verifier = _load_verifier()
|
|
destination = tmp_path / "scripts"
|
|
destination.mkdir()
|
|
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
|
rows = []
|
|
for name in verifier.EXPECTED_BACKUP_FILES:
|
|
path = destination / name
|
|
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
|
path.chmod(0o755)
|
|
rows.append({"name": name, "sha256": _sha256(path)})
|
|
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
|
exporter_destination.chmod(0o755)
|
|
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
|
(destination / verifier.EXPECTED_BACKUP_FILES[0]).write_text("drifted\n", encoding="utf-8")
|
|
verifier_digest = _sha256(VERIFIER)
|
|
source_revision = "d" * 40
|
|
run_id = "unit-drift"
|
|
manifest = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": "e" * 40,
|
|
"runId": run_id,
|
|
"executorSha256": "f" * 64,
|
|
"verifierSha256": verifier_digest,
|
|
"files": rows,
|
|
}
|
|
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
|
|
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
|
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
str(VERIFIER),
|
|
"--manifest-base64",
|
|
encoded,
|
|
"--destination",
|
|
str(destination),
|
|
"--exporter-destination",
|
|
str(exporter_destination),
|
|
"--source-revision",
|
|
source_revision,
|
|
"--run-id",
|
|
run_id,
|
|
"--expected-verifier-sha256",
|
|
verifier_digest,
|
|
],
|
|
)
|
|
|
|
with pytest.raises(SystemExit):
|
|
verifier.main()
|
|
assert "runtime_file_identity_failed" in capsys.readouterr().err
|
|
|
|
|
|
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
|
|
sources = [
|
|
ROOT / "agent99-bootstrap.ps1",
|
|
ROOT / "agent99-contract-check.ps1",
|
|
ROOT / "agent99-deploy.ps1",
|
|
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
|
|
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
|
|
]
|
|
marker = "agent99-host110-backup-runtime-broker.ps1"
|
|
|
|
for path in sources:
|
|
assert marker in path.read_text(encoding="utf-8"), path
|