a6307e785d
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 3m41s
CD Pipeline / build-and-deploy (push) Successful in 7m17s
CD Pipeline / post-deploy-checks (push) Successful in 1m59s
680 lines
23 KiB
Python
680 lines
23 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
|
|
|
|
|
|
def _write_executable(path: Path, body: str) -> None:
|
|
path.write_text(body, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _runtime_harness(
|
|
tmp_path: Path,
|
|
*,
|
|
ledger_capacity: int = 128,
|
|
) -> tuple[Path, dict[str, str], Path]:
|
|
source = READINESS.read_text(encoding="utf-8")
|
|
root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then'
|
|
assert source.count(root_gate) == 1
|
|
assert source.count("timer_ledger_capacity=128") == 1
|
|
source = source.replace(root_gate, "if false; then", 1)
|
|
source = source.replace(
|
|
"timer_ledger_capacity=128",
|
|
f"timer_ledger_capacity={ledger_capacity}",
|
|
1,
|
|
)
|
|
script = tmp_path / "host112-runtime-harness.sh"
|
|
_write_executable(script, source)
|
|
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
fake_state = tmp_path / "fake-host"
|
|
fake_state.mkdir()
|
|
|
|
_write_executable(
|
|
fake_bin / "timeout",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--signal=*|--kill-after=*) shift ;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
[ "$#" -gt 0 ] || exit 64
|
|
shift
|
|
"$@"
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "date",
|
|
"""#!/usr/bin/env bash
|
|
case "${1:-}" in
|
|
--iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;;
|
|
+%s) printf '%s\n' '1000' ;;
|
|
*) /bin/date "$@" ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "flock",
|
|
"""#!/usr/bin/env bash
|
|
exit 0
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "mv",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
destination=""
|
|
for argument in "$@"; do destination="$argument"; done
|
|
if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then
|
|
: >"$FAKE_RECEIPT_MV_MARKER"
|
|
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
|
|
fi
|
|
/bin/mv "$@"
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "ln",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
destination=""
|
|
for argument in "$@"; do destination="$argument"; done
|
|
if [ "${FAKE_RECEIPT_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */receipts/*.txt ]]; then
|
|
exit 1
|
|
fi
|
|
if [ "${FAKE_READBACK_LINK_FAIL:-0}" = "1" ] && [[ "$destination" == */readbacks/*.txt ]] && [[ "$destination" != *.signal.txt ]]; then
|
|
exit 1
|
|
fi
|
|
/bin/ln "$@"
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "pgrep",
|
|
"""#!/usr/bin/env bash
|
|
case "$*" in
|
|
*wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;;
|
|
*Xorg*|*Xwayland*) exit 0 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "ss",
|
|
"""#!/usr/bin/env bash
|
|
case "$*" in
|
|
*-ltn*)
|
|
for port in 22 1514 1515 55000; do
|
|
printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port"
|
|
done
|
|
;;
|
|
*-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "df",
|
|
"""#!/usr/bin/env bash
|
|
printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
|
|
printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "systemctl",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
state="${FAKE_HOST_STATE_DIR:?}"
|
|
require_wal() {
|
|
if [ ! -f "${HOST112_RECOVERY_STATE_DIR:?}/active-artifact-transaction.txt" ]; then
|
|
: >"$state/mutation_without_wal"
|
|
fi
|
|
}
|
|
cmd="${1:-}"
|
|
[ "$#" -gt 0 ] && shift
|
|
case "$cmd" in
|
|
show)
|
|
unit="${1:-}"; shift || true
|
|
property=""
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi
|
|
done
|
|
case "$property" in
|
|
LoadState) printf '%s\n' 'loaded' ;;
|
|
ActiveState) printf '%s\n' 'active' ;;
|
|
SubState) printf '%s\n' 'running' ;;
|
|
Result) printf '%s\n' 'success' ;;
|
|
*) printf '%s\n' 'unknown' ;;
|
|
esac
|
|
;;
|
|
is-active)
|
|
unit="${1:-}"
|
|
case "$unit" in
|
|
lightdm.service|display-manager.service)
|
|
if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi
|
|
;;
|
|
*) printf '%s\n' 'active' ;;
|
|
esac
|
|
;;
|
|
is-enabled) printf '%s\n' 'enabled' ;;
|
|
is-system-running)
|
|
if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then
|
|
: >"$FAKE_FINALIZE_MARKER"
|
|
sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}"
|
|
fi
|
|
printf '%s\n' 'running'
|
|
;;
|
|
get-default) printf '%s\n' 'graphical.target' ;;
|
|
set-default)
|
|
require_wal
|
|
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
|
|
exit "${FAKE_SET_DEFAULT_RC:-0}"
|
|
;;
|
|
start)
|
|
require_wal
|
|
unit="${1:-}"
|
|
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
|
|
if [ "$unit" = "lightdm.service" ]; then
|
|
if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi
|
|
rc="${FAKE_LIGHTDM_START_RC:-0}"
|
|
if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi
|
|
exit "$rc"
|
|
fi
|
|
;;
|
|
enable)
|
|
require_wal
|
|
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
|
|
;;
|
|
reset-failed|stop) require_wal ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
""",
|
|
)
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{fake_bin}:{env['PATH']}",
|
|
"FAKE_HOST_STATE_DIR": str(fake_state),
|
|
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
|
|
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
|
|
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
|
|
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
|
|
}
|
|
)
|
|
return script, env, fake_state
|
|
|
|
|
|
def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[
|
|
"bash",
|
|
str(script),
|
|
"--apply",
|
|
"--trace-id",
|
|
"trace:runtime-harness",
|
|
"--run-id",
|
|
"run:runtime-harness",
|
|
"--work-item-id",
|
|
"HOST112-RUNTIME-HARNESS",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
|
|
|
|
def _dry_run(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[
|
|
"bash",
|
|
str(script),
|
|
"--dry-run",
|
|
"--trace-id",
|
|
"trace:runtime-harness",
|
|
"--run-id",
|
|
"run:runtime-harness",
|
|
"--work-item-id",
|
|
"HOST112-RUNTIME-HARNESS",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
|
|
|
|
def test_post_apply_dry_run_read_only_verifies_fixed_wal_and_immutable_pair(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, _ = _runtime_harness(tmp_path)
|
|
|
|
applied = _apply(script, env)
|
|
assert applied.returncode == 0, applied.stdout + applied.stderr
|
|
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
|
transaction_before = transaction.read_bytes()
|
|
|
|
verified = _dry_run(script, env)
|
|
output = verified.stdout + verified.stderr
|
|
assert verified.returncode == 0, output
|
|
assert "durable_readback_status=loaded_immutable_verified " in output
|
|
assert "artifact_transaction_status=existing_pair_verified" in output
|
|
assert "artifact_transaction_started=0" in output
|
|
assert "artifact_transaction_blocked=0" in output
|
|
assert "artifact_transaction_write_performed=0" in output
|
|
assert "artifact_transaction_pair_verified=1" in output
|
|
assert "artifact_transaction_prior_pair_verified=1" in output
|
|
assert transaction.read_bytes() == transaction_before
|
|
|
|
|
|
def test_pre_apply_dry_run_blocks_unresolved_prior_transaction(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
state = tmp_path / "state"
|
|
state.mkdir()
|
|
transaction = state / "active-artifact-transaction.txt"
|
|
unresolved = (
|
|
"schema_version=host112_active_artifact_transaction_v1 "
|
|
"updated_at=2026-07-14T12:01:00+08:00 "
|
|
"status=pending_artifact_commit phase=prior_runtime_mutation "
|
|
"trace_id=trace:prior run_id=run:prior "
|
|
"work_item_id=HOST112-PRIOR boot_id=boot:prior "
|
|
f"receipt_path={state / 'receipts' / 'prior.txt'} "
|
|
f"readback_path={state / 'readbacks' / 'prior.txt'} "
|
|
"pair_verified=0\n"
|
|
)
|
|
transaction.write_text(unresolved, encoding="utf-8")
|
|
Path(env["HOST112_MANAGER_LOCK_FILE"]).touch()
|
|
|
|
dry_run = _dry_run(script, env)
|
|
output = dry_run.stdout + dry_run.stderr
|
|
assert dry_run.returncode == 1, output
|
|
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
|
|
assert "artifact_transaction_blocked=1" in output
|
|
assert "runtime_write_performed=0" in output
|
|
assert "receipt_write_performed=0" in output
|
|
assert transaction.read_text(encoding="utf-8") == unresolved
|
|
assert not (fake_state / "mutations.log").exists()
|
|
|
|
|
|
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
env["FAKE_PARTIAL_LIGHTDM_START"] = "1"
|
|
|
|
failed = _apply(script, env)
|
|
failed_output = failed.stdout + failed.stderr
|
|
assert failed.returncode == 1
|
|
assert "runtime_write_performed=1" in failed_output
|
|
assert "receipt_status=written_immutable" in failed_output
|
|
assert "durable_readback_status=written_immutable " in failed_output
|
|
assert (
|
|
"durable_receipt_terminal=execution_failed_or_unattributed_"
|
|
"idempotent_already_verified_healthy"
|
|
) in failed_output
|
|
assert "guest_ready=1" in failed_output
|
|
assert "action_failure_observed=1" in failed_output
|
|
assert "artifact_transaction_status=committed_verified_pair" in failed_output
|
|
assert "artifact_transaction_pair_verified=1" in failed_output
|
|
assert not (fake_state / "mutation_without_wal").exists()
|
|
|
|
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
|
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
|
|
stored_readback = readback.read_text(encoding="utf-8")
|
|
for derived in (
|
|
"durable_readback_present=1",
|
|
"durable_readback_identity_match=1",
|
|
"durable_readback_boot_match=1",
|
|
"durable_readback_receipt_link_match=1",
|
|
"durable_readback_terminal_match=1",
|
|
):
|
|
assert derived not in stored_readback
|
|
receipt_before = receipt.read_bytes()
|
|
readback_before = readback.read_bytes()
|
|
|
|
(fake_state / "lightdm_active").touch()
|
|
env.pop("FAKE_LIGHTDM_START_RC")
|
|
env.pop("FAKE_PARTIAL_LIGHTDM_START")
|
|
replay = _apply(script, env)
|
|
replay_output = replay.stdout + replay.stderr
|
|
assert replay.returncode == 1
|
|
assert "guest_ready=1" in replay_output
|
|
assert "receipt_status=reused_immutable" in replay_output
|
|
assert "durable_readback_status=reused_immutable_verified" in replay_output
|
|
assert "terminal=immutable_receipt_replay" in replay_output
|
|
assert receipt.read_bytes() == receipt_before
|
|
assert readback.read_bytes() == readback_before
|
|
|
|
|
|
def test_immutable_replay_cannot_ignore_newer_unresolved_fixed_transaction(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
|
|
first = _apply(script, env)
|
|
assert first.returncode == 0, first.stdout + first.stderr
|
|
state = tmp_path / "state"
|
|
receipt = next((state / "receipts").glob("*.txt"))
|
|
readback = next((state / "readbacks").glob("*.txt"))
|
|
receipt_before = receipt.read_bytes()
|
|
readback_before = readback.read_bytes()
|
|
transaction = state / "active-artifact-transaction.txt"
|
|
unresolved = (
|
|
"schema_version=host112_active_artifact_transaction_v1 "
|
|
"updated_at=2026-07-14T12:01:00+08:00 "
|
|
"status=pending_artifact_commit phase=newer_runtime_mutation "
|
|
"trace_id=trace:newer run_id=run:newer "
|
|
"work_item_id=HOST112-NEWER boot_id=boot:newer "
|
|
f"receipt_path={state / 'receipts' / 'newer.txt'} "
|
|
f"readback_path={state / 'readbacks' / 'newer.txt'} "
|
|
"pair_verified=0\n"
|
|
)
|
|
transaction.write_text(unresolved, encoding="utf-8")
|
|
mutations = fake_state / "mutations.log"
|
|
mutations_before = mutations.read_bytes()
|
|
|
|
replay = _apply(script, env)
|
|
output = replay.stdout + replay.stderr
|
|
assert replay.returncode == 1, output
|
|
assert "receipt_status=reused_immutable" in output
|
|
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in output
|
|
assert "artifact_transaction_blocked=1" in output
|
|
assert "artifact_transaction_pair_verified=0" in output
|
|
assert transaction.read_text(encoding="utf-8") == unresolved
|
|
assert receipt.read_bytes() == receipt_before
|
|
assert readback.read_bytes() == readback_before
|
|
assert mutations.read_bytes() == mutations_before
|
|
|
|
|
|
def test_receipt_failure_leaves_durable_partial_without_readback_and_blocks_retry(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
env.update(
|
|
{
|
|
"FAKE_LIGHTDM_START_RC": "124",
|
|
"FAKE_RECEIPT_LINK_FAIL": "1",
|
|
}
|
|
)
|
|
|
|
failed = _apply(script, env)
|
|
output = failed.stdout + failed.stderr
|
|
assert failed.returncode == 1
|
|
assert "runtime_write_performed=1" in output
|
|
assert "receipt_status=write_failed" in output
|
|
assert "artifact_transaction_status=partial_pending_public_artifact_commit" in output
|
|
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
|
stored = transaction.read_text(encoding="utf-8")
|
|
assert "status=partial_pending_public_artifact_commit" in stored
|
|
assert "phase=receipt_write_or_disk_verification_failed" in stored
|
|
assert not (tmp_path / "state" / "readbacks").exists()
|
|
assert not (fake_state / "mutation_without_wal").exists()
|
|
|
|
mutations = fake_state / "mutations.log"
|
|
mutations.write_text("", encoding="utf-8")
|
|
retry_env = env.copy()
|
|
retry_env.pop("FAKE_RECEIPT_LINK_FAIL")
|
|
retry = _apply(script, retry_env)
|
|
retry_output = retry.stdout + retry.stderr
|
|
assert retry.returncode == 1
|
|
assert "schema_version=host112_guest_recovery_v2" in retry_output
|
|
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
|
|
retry_output
|
|
)
|
|
assert "terminal=execution_failed_or_unattributed_" in retry_output
|
|
assert "artifact_transaction_blocked" in retry_output
|
|
assert "receipt_write_performed=0" in retry_output
|
|
assert mutations.read_text(encoding="utf-8") == ""
|
|
|
|
|
|
def test_readback_failure_keeps_receipt_and_durable_partial_not_false_success(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
env.update(
|
|
{
|
|
"FAKE_LIGHTDM_START_RC": "124",
|
|
"FAKE_READBACK_LINK_FAIL": "1",
|
|
}
|
|
)
|
|
|
|
failed = _apply(script, env)
|
|
output = failed.stdout + failed.stderr
|
|
assert failed.returncode == 1
|
|
assert "receipt_status=written_immutable" in output
|
|
assert "durable_readback_status=write_failed" in output
|
|
assert "artifact_transaction_pair_verified=0" in output
|
|
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
|
assert "terminal=" in receipt.read_text(encoding="utf-8")
|
|
assert not list((tmp_path / "state" / "readbacks").glob("*.txt"))
|
|
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
|
stored = transaction.read_text(encoding="utf-8")
|
|
assert "status=partial_pending_public_artifact_commit" in stored
|
|
assert "phase=durable_readback_write_or_disk_verification_failed" in stored
|
|
assert not (fake_state / "mutation_without_wal").exists()
|
|
|
|
|
|
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
noop_root = tmp_path / "noop"
|
|
noop_root.mkdir()
|
|
script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4)
|
|
(fake_state / "lightdm_active").touch()
|
|
|
|
for index in range(5):
|
|
run_env = env | {"INVOCATION_ID": f"noop-{index}"}
|
|
result = subprocess.run(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=run_env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
state = noop_root / "state"
|
|
assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [
|
|
Path("timer-observation.latest")
|
|
]
|
|
assert "repeat_count=5" in (state / "timer-observation.latest").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
mutation_root = tmp_path / "mutation"
|
|
mutation_root.mkdir()
|
|
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
mutation_slots: list[str] = []
|
|
# These five identities cover every four-slot ring position and then reuse
|
|
# one slot. That proves both capacity coverage and rollover without twelve
|
|
# expensive full shell-harness invocations.
|
|
for index in range(5):
|
|
run_env = env | {"INVOCATION_ID": f"slot-{index}"}
|
|
result = subprocess.run(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=run_env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
output = result.stdout + result.stderr
|
|
assert result.returncode == 1
|
|
assert "runtime_write_performed=1" in output
|
|
assert "receipt_status=written_bounded_timer_slot" in output
|
|
assert "durable_readback_status=written_bounded_timer_slot " in output
|
|
mutation_slots.append(
|
|
next(
|
|
token.split("=", 1)[1]
|
|
for token in output.split()
|
|
if token.startswith("timer_ledger_slot=")
|
|
)
|
|
)
|
|
for field in (
|
|
"durable_readback_present",
|
|
"durable_readback_identity_match",
|
|
"durable_readback_boot_match",
|
|
"durable_readback_receipt_link_match",
|
|
"durable_readback_terminal_match",
|
|
):
|
|
assert f"{field}=1" in output
|
|
assert f"{field}=0" not in output
|
|
assert "timer_no_runtime_mutation_aggregated" not in output
|
|
|
|
assert len(set(mutation_slots)) == 4
|
|
assert len(mutation_slots) > len(set(mutation_slots))
|
|
|
|
ledger_files = [
|
|
path
|
|
for path in (mutation_root / "state" / "timer-ledger").iterdir()
|
|
if path.is_file()
|
|
]
|
|
assert len(ledger_files) == 8
|
|
assert not (mutation_root / "state" / "receipts").exists()
|
|
assert not (mutation_root / "state" / "readbacks").exists()
|
|
for readback in (mutation_root / "state" / "timer-ledger").glob(
|
|
"*.readback.txt"
|
|
):
|
|
stored = readback.read_text(encoding="utf-8")
|
|
for field in (
|
|
"durable_readback_present",
|
|
"durable_readback_identity_match",
|
|
"durable_readback_boot_match",
|
|
"durable_readback_receipt_link_match",
|
|
"durable_readback_terminal_match",
|
|
):
|
|
assert f"{field}=1" not in stored
|
|
assert f"{field}=0" not in stored
|
|
transaction = mutation_root / "state" / "active-artifact-transaction.txt"
|
|
assert "status=committed_verified_pair" in transaction.read_text(encoding="utf-8")
|
|
|
|
|
|
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
|
|
marker = tmp_path / "receipt-mv.entered"
|
|
run_env = env | {
|
|
"INVOCATION_ID": "sigkill-0",
|
|
"FAKE_RECEIPT_MV_MARKER": str(marker),
|
|
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
|
|
}
|
|
process = subprocess.Popen(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=run_env,
|
|
start_new_session=True,
|
|
)
|
|
deadline = time.monotonic() + 8
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert marker.exists()
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.communicate(timeout=5)
|
|
assert process.returncode == -signal.SIGKILL
|
|
|
|
ledger = tmp_path / "state" / "timer-ledger"
|
|
files = [path for path in ledger.iterdir() if path.is_file()]
|
|
assert len(files) == 1
|
|
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
|
|
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
|
assert "status=pending_artifact_commit" in transaction.read_text(encoding="utf-8")
|
|
|
|
mutations = tmp_path / "fake-host" / "mutations.log"
|
|
before = mutations.read_bytes()
|
|
retry = subprocess.run(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env | {"INVOCATION_ID": "sigkill-retry"},
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
assert retry.returncode == 1
|
|
assert "artifact_transaction_status=blocked_unresolved_prior_transaction" in (
|
|
retry.stdout + retry.stderr
|
|
)
|
|
assert mutations.read_bytes() == before
|
|
|
|
|
|
def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, _ = _runtime_harness(tmp_path)
|
|
marker = tmp_path / "final-verifier.entered"
|
|
env.update(
|
|
{
|
|
"FAKE_FINALIZE_MARKER": str(marker),
|
|
"FAKE_FINALIZE_SLEEP_SECONDS": "2",
|
|
}
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
"bash",
|
|
str(script),
|
|
"--apply",
|
|
"--trace-id",
|
|
"trace:runtime-harness",
|
|
"--run-id",
|
|
"run:runtime-harness",
|
|
"--work-item-id",
|
|
"HOST112-RUNTIME-HARNESS",
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=env,
|
|
)
|
|
deadline = time.monotonic() + 8
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
assert marker.exists()
|
|
os.kill(process.pid, signal.SIGTERM)
|
|
stdout, stderr = process.communicate(timeout=12)
|
|
assert process.returncode == 1, stdout + stderr
|
|
|
|
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
|
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
|
|
receipt_text = receipt.read_text(encoding="utf-8")
|
|
signal_text = signal_readback.read_text(encoding="utf-8")
|
|
assert "terminal=interrupted_signal_TERM" in receipt_text
|
|
assert "action_count=1 actions=start_lightdm" in receipt_text
|
|
assert f"receipt_path={receipt}" in signal_text
|
|
assert "terminal=interrupted_signal_TERM" in signal_text
|
|
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
|
|
transaction = tmp_path / "state" / "active-artifact-transaction.txt"
|
|
transaction_text = transaction.read_text(encoding="utf-8")
|
|
assert "status=partial_pending_public_artifact_commit" in transaction_text
|
|
assert "phase=interrupted_before_durable_readback_TERM" in transaction_text
|
|
assert not [
|
|
path
|
|
for path in (tmp_path / "state" / "readbacks").glob("*.txt")
|
|
if not path.name.endswith(".signal.txt")
|
|
]
|
|
|
|
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
|
|
assert replay.returncode == 65
|
|
assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in (
|
|
replay.stdout + replay.stderr
|
|
)
|