from __future__ import annotations import os import signal import subprocess import time from pathlib import Path ROOT = Path(__file__).resolve().parents[3] READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh" def _write_executable(path: Path, body: str) -> None: path.write_text(body, encoding="utf-8") path.chmod(0o755) def _runtime_harness( tmp_path: Path, *, ledger_capacity: int = 128, ) -> tuple[Path, dict[str, str], Path]: source = READINESS.read_text(encoding="utf-8") root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then' assert source.count(root_gate) == 1 assert source.count("timer_ledger_capacity=128") == 1 source = source.replace(root_gate, "if false; then", 1) source = source.replace( "timer_ledger_capacity=128", f"timer_ledger_capacity={ledger_capacity}", 1, ) script = tmp_path / "host112-runtime-harness.sh" _write_executable(script, source) fake_bin = tmp_path / "bin" fake_bin.mkdir() fake_state = tmp_path / "fake-host" fake_state.mkdir() _write_executable( fake_bin / "timeout", """#!/usr/bin/env bash set -u while [ "$#" -gt 0 ]; do case "$1" in --signal=*|--kill-after=*) shift ;; *) break ;; esac done [ "$#" -gt 0 ] || exit 64 shift "$@" """, ) _write_executable( fake_bin / "date", """#!/usr/bin/env bash case "${1:-}" in --iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;; +%s) printf '%s\n' '1000' ;; *) /bin/date "$@" ;; esac """, ) _write_executable( fake_bin / "flock", """#!/usr/bin/env bash exit 0 """, ) _write_executable( fake_bin / "mv", """#!/usr/bin/env bash set -u destination="" for argument in "$@"; do destination="$argument"; done if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then : >"$FAKE_RECEIPT_MV_MARKER" sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}" fi /bin/mv "$@" """, ) _write_executable( fake_bin / "pgrep", """#!/usr/bin/env bash case "$*" in *wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;; *Xorg*|*Xwayland*) exit 0 ;; *) exit 1 ;; esac """, ) _write_executable( fake_bin / "ss", """#!/usr/bin/env bash case "$*" in *-ltn*) for port in 22 1514 1515 55000; do printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port" done ;; *-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;; esac """, ) _write_executable( fake_bin / "df", """#!/usr/bin/env bash printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on' printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /' """, ) _write_executable( fake_bin / "systemctl", """#!/usr/bin/env bash set -u state="${FAKE_HOST_STATE_DIR:?}" cmd="${1:-}" [ "$#" -gt 0 ] && shift case "$cmd" in show) unit="${1:-}"; shift || true property="" while [ "$#" -gt 0 ]; do if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi done case "$property" in LoadState) printf '%s\n' 'loaded' ;; ActiveState) printf '%s\n' 'active' ;; SubState) printf '%s\n' 'running' ;; Result) printf '%s\n' 'success' ;; *) printf '%s\n' 'unknown' ;; esac ;; is-active) unit="${1:-}" case "$unit" in lightdm.service|display-manager.service) if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi ;; *) printf '%s\n' 'active' ;; esac ;; is-enabled) printf '%s\n' 'enabled' ;; is-system-running) if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then : >"$FAKE_FINALIZE_MARKER" sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}" fi printf '%s\n' 'running' ;; get-default) printf '%s\n' 'graphical.target' ;; set-default) printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log" exit "${FAKE_SET_DEFAULT_RC:-0}" ;; start) unit="${1:-}" printf 'start:%s\n' "$unit" >>"$state/mutations.log" if [ "$unit" = "lightdm.service" ]; then if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi rc="${FAKE_LIGHTDM_START_RC:-0}" if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi exit "$rc" fi ;; enable) printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log" ;; reset-failed|stop) ;; *) exit 1 ;; esac """, ) env = os.environ.copy() env.update( { "PATH": f"{fake_bin}:{env['PATH']}", "FAKE_HOST_STATE_DIR": str(fake_state), "HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"), "HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"), "HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"), "HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"), } ) return script, env, fake_state def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]: return subprocess.run( [ "bash", str(script), "--apply", "--trace-id", "trace:runtime-harness", "--run-id", "run:runtime-harness", "--work-item-id", "HOST112-RUNTIME-HARNESS", ], text=True, capture_output=True, env=env, check=False, timeout=15, ) def test_failed_general_run_cannot_be_washed_green_by_external_recovery( tmp_path: Path, ) -> None: script, env, fake_state = _runtime_harness(tmp_path) env["FAKE_LIGHTDM_START_RC"] = "124" env["FAKE_PARTIAL_LIGHTDM_START"] = "1" failed = _apply(script, env) failed_output = failed.stdout + failed.stderr assert failed.returncode == 1 assert "runtime_write_performed=1" in failed_output assert "receipt_status=written_immutable" in failed_output assert "durable_readback_status=written_immutable" in failed_output assert ( "durable_receipt_terminal=execution_failed_or_unattributed_" "idempotent_already_verified_healthy" ) in failed_output assert "guest_ready=1" in failed_output assert "action_failure_observed=1" in failed_output receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) readback = next((tmp_path / "state" / "readbacks").glob("*.txt")) receipt_before = receipt.read_bytes() readback_before = readback.read_bytes() (fake_state / "lightdm_active").touch() env.pop("FAKE_LIGHTDM_START_RC") env.pop("FAKE_PARTIAL_LIGHTDM_START") replay = _apply(script, env) replay_output = replay.stdout + replay.stderr assert replay.returncode == 1 assert "guest_ready=1" in replay_output assert "receipt_status=reused_immutable" in replay_output assert "durable_readback_status=reused_immutable_verified" in replay_output assert "terminal=immutable_receipt_replay" in replay_output assert receipt.read_bytes() == receipt_before assert readback.read_bytes() == readback_before def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded( tmp_path: Path, ) -> None: noop_root = tmp_path / "noop" noop_root.mkdir() script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4) (fake_state / "lightdm_active").touch() for index in range(5): run_env = env | {"INVOCATION_ID": f"noop-{index}"} result = subprocess.run( ["bash", str(script), "--apply-timer"], text=True, capture_output=True, env=run_env, check=False, timeout=15, ) assert result.returncode == 0, result.stdout + result.stderr state = noop_root / "state" assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [ Path("timer-observation.latest") ] assert "repeat_count=5" in (state / "timer-observation.latest").read_text( encoding="utf-8" ) mutation_root = tmp_path / "mutation" mutation_root.mkdir() script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4) env["FAKE_LIGHTDM_START_RC"] = "124" for index in range(12): run_env = env | {"INVOCATION_ID": f"mutation-{index}"} result = subprocess.run( ["bash", str(script), "--apply-timer"], text=True, capture_output=True, env=run_env, check=False, timeout=15, ) output = result.stdout + result.stderr assert result.returncode == 1 assert "runtime_write_performed=1" in output assert "receipt_status=written_bounded_timer_slot" in output assert "durable_readback_status=written_bounded_timer_slot" in output for field in ( "durable_readback_present", "durable_readback_identity_match", "durable_readback_boot_match", "durable_readback_receipt_link_match", "durable_readback_terminal_match", ): assert f"{field}=1" in output assert f"{field}=0" not in output assert "timer_no_runtime_mutation_aggregated" not in output ledger_files = [ path for path in (mutation_root / "state" / "timer-ledger").iterdir() if path.is_file() ] assert len(ledger_files) <= 8 assert not (mutation_root / "state" / "receipts").exists() assert not (mutation_root / "state" / "readbacks").exists() for readback in (mutation_root / "state" / "timer-ledger").glob( "*.readback.txt" ): stored = readback.read_text(encoding="utf-8") for field in ( "durable_readback_present", "durable_readback_identity_match", "durable_readback_boot_match", "durable_readback_receipt_link_match", "durable_readback_terminal_match", ): assert stored.count(f"{field}=1") == 1 assert f"{field}=0" not in stored def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps( tmp_path: Path, ) -> None: script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4) env["FAKE_LIGHTDM_START_RC"] = "124" for index in range(12): marker = tmp_path / f"receipt-mv-{index}.entered" run_env = env | { "INVOCATION_ID": f"sigkill-{index}", "FAKE_RECEIPT_MV_MARKER": str(marker), "FAKE_RECEIPT_MV_SLEEP_SECONDS": "5", } process = subprocess.Popen( ["bash", str(script), "--apply-timer"], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=run_env, start_new_session=True, ) deadline = time.monotonic() + 8 while not marker.exists() and time.monotonic() < deadline: time.sleep(0.01) assert marker.exists() os.killpg(process.pid, signal.SIGKILL) process.communicate(timeout=5) assert process.returncode == -signal.SIGKILL ledger = tmp_path / "state" / "timer-ledger" files = [path for path in ledger.iterdir() if path.is_file()] assert len(files) <= 4 assert all(path.name.endswith(".receipt.txt.tmp") for path in files) def test_signal_during_final_verifier_writes_correlated_failure_chain( tmp_path: Path, ) -> None: script, env, _ = _runtime_harness(tmp_path) marker = tmp_path / "final-verifier.entered" env.update( { "FAKE_FINALIZE_MARKER": str(marker), "FAKE_FINALIZE_SLEEP_SECONDS": "2", } ) process = subprocess.Popen( [ "bash", str(script), "--apply", "--trace-id", "trace:runtime-harness", "--run-id", "run:runtime-harness", "--work-item-id", "HOST112-RUNTIME-HARNESS", ], text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, env=env, ) deadline = time.monotonic() + 8 while not marker.exists() and time.monotonic() < deadline: time.sleep(0.02) assert marker.exists() os.kill(process.pid, signal.SIGTERM) stdout, stderr = process.communicate(timeout=12) assert process.returncode == 75, stdout + stderr receipt = next((tmp_path / "state" / "receipts").glob("*.txt")) signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt")) receipt_text = receipt.read_text(encoding="utf-8") signal_text = signal_readback.read_text(encoding="utf-8") assert "terminal=interrupted_signal_TERM" in receipt_text assert "action_count=1 actions=start_lightdm" in receipt_text assert f"receipt_path={receipt}" in signal_text assert "terminal=interrupted_signal_TERM" in signal_text assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""}) assert replay.returncode == 65 assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in ( replay.stdout + replay.stderr )