fix(recovery): bind host112 closure artifacts
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m35s
CD Pipeline / build-and-deploy (push) Failing after 7m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m35s
CD Pipeline / build-and-deploy (push) Failing after 7m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
This commit is contained in:
@@ -149,6 +149,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
|
||||
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
|
||||
) == 1
|
||||
assert "manager_attempt_count=1" in source
|
||||
assert 'if [ "$manager_start_exit" = "0" ]; then' in source
|
||||
assert 'manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"' in source
|
||||
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
|
||||
assert 'rollback_terminal="bounded_stop_verified"' in source
|
||||
assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source
|
||||
@@ -171,7 +173,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
|
||||
marker_function = source[
|
||||
source.index("write_epoch_marker() {") : source.index("unit_active() {")
|
||||
]
|
||||
assert 'tmp="${marker}.tmp.$$"' in marker_function
|
||||
assert 'tmp="${marker}.tmp"' in marker_function
|
||||
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
|
||||
assert 'chmod 0644 "$tmp"' in marker_function
|
||||
assert 'mv "$tmp" "$marker"' in marker_function
|
||||
@@ -251,7 +253,15 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No
|
||||
assert "printf '%s\\0%s\\0%s\\0'" in source
|
||||
assert 'boot_id=$boot_id' in source
|
||||
assert 'ln "$tmp" "$receipt_path"' in source
|
||||
assert 'mv "$tmp" "$receipt_path"' not in source
|
||||
receipt_writer = source[
|
||||
source.index("write_public_receipt() {") : source.index(
|
||||
"write_signal_readback() {",
|
||||
)
|
||||
]
|
||||
assert 'if [ "$MODE" = "timer_apply" ]' in receipt_writer
|
||||
assert 'mv "$tmp" "$receipt_path"' in receipt_writer
|
||||
assert 'elif [ "$MODE" != "timer_apply" ]' in receipt_writer
|
||||
assert 'ln "$tmp" "$receipt_path"' in receipt_writer
|
||||
assert 'receipt_status="reused_immutable"' in source
|
||||
assert 'manager_terminal="immutable_receipt_replay"' in source
|
||||
assert "immutable_receipt_identity_or_boot_mismatch" in source
|
||||
@@ -279,6 +289,17 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
|
||||
"boot_id=unknown terminal=verified_healthy\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
readback_dir = tmp_path / "state" / "readbacks"
|
||||
readback_dir.mkdir(parents=True)
|
||||
durable_readback = readback_dir / f"{run_id}-{digest}.txt"
|
||||
receipt_sha = hashlib.sha256(receipt.read_bytes()).hexdigest()
|
||||
durable_readback.write_text(
|
||||
"schema_version=host112_guest_recovery_v2 "
|
||||
f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} "
|
||||
"boot_id=unknown durable_receipt_terminal=verified_healthy "
|
||||
f"receipt_path={receipt} receipt_sha256={receipt_sha}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
original = receipt.read_bytes()
|
||||
env = isolated_env(tmp_path)
|
||||
args = [
|
||||
@@ -301,6 +322,9 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
|
||||
assert "durable_receipt_identity_match=1" in matched_output
|
||||
assert "durable_receipt_boot_match=1" in matched_output
|
||||
assert "durable_receipt_terminal=verified_healthy" in matched_output
|
||||
assert "durable_readback_present=1" in matched_output
|
||||
assert "durable_readback_receipt_link_match=1" in matched_output
|
||||
assert "durable_readback_terminal_match=1" in matched_output
|
||||
assert receipt.read_bytes() == original
|
||||
|
||||
receipt.write_text(
|
||||
@@ -328,12 +352,56 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No
|
||||
assert "trap 'handle_executor_signal TERM' TERM" in source
|
||||
assert "trap 'handle_executor_signal INT' INT" in source
|
||||
assert "trap 'handle_executor_signal HUP' HUP" in source
|
||||
assert "write_public_receipt || true" in source
|
||||
assert "if write_public_receipt && write_signal_readback; then" in source
|
||||
assert "action_count=${#actions[@]} actions=$receipt_action_text" in source
|
||||
assert "schema_version=host112_guest_recovery_signal_readback_v1" in source
|
||||
assert 'ln "$tmp" "$signal_readback_path"' in source
|
||||
assert "signal_readback_status=\"written_immutable\"" in source
|
||||
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source
|
||||
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source
|
||||
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source
|
||||
|
||||
|
||||
def test_timer_noop_uses_one_bounded_aggregate_instead_of_per_minute_artifacts() -> None:
|
||||
source = read(READINESS)
|
||||
function = source[
|
||||
source.index("write_bounded_timer_observation() {") : source.index(
|
||||
'boot_id="$(cat /proc/sys/kernel/random/boot_id',
|
||||
)
|
||||
]
|
||||
|
||||
assert 'timer_observation_path="$STATE_DIR/timer-observation.latest"' in function
|
||||
assert "schema_version=host112_timer_observation_aggregate_v1" in function
|
||||
assert "aggregation=fixed_single_atomic_pointer" in function
|
||||
assert 'mv "$tmp" "$timer_observation_path"' in function
|
||||
assert "repeat_count=$timer_observation_repeat_count" in function
|
||||
assert 'if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then' in source
|
||||
assert 'receipt_status="timer_no_runtime_mutation_aggregated"' in source
|
||||
assert 'durable_readback_status="skipped_bounded_timer_no_runtime_mutation"' in source
|
||||
assert "timer_ledger_capacity=128" in source
|
||||
assert "write_bounded_timer_mutation_readback()" in source
|
||||
assert "retention=rolling_atomic_overwrite" in source
|
||||
assert 'timer_artifact_policy="bounded_ring_128_for_runtime_mutation"' in source
|
||||
assert 'timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"' in source
|
||||
|
||||
|
||||
def test_apply_success_is_bound_to_same_run_durable_terminal() -> None:
|
||||
source = read(READINESS)
|
||||
terminal_gate = source[source.rindex('if [ "$MODE" = "apply" ]') :]
|
||||
|
||||
assert 'successful_terminal="$run_terminal"' in terminal_gate
|
||||
assert 'successful_terminal="$durable_receipt_terminal"' in terminal_gate
|
||||
assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate
|
||||
assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate
|
||||
guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]')
|
||||
receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier)
|
||||
assert guest_verifier < receipt_commit
|
||||
assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source
|
||||
assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source
|
||||
assert 'action_failure_observed=1' in source
|
||||
assert "manager_terminal=$manager_terminal terminal=$run_terminal" in source
|
||||
|
||||
|
||||
def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None:
|
||||
source = read(READINESS)
|
||||
general_start = source.index(
|
||||
@@ -424,10 +492,17 @@ def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() ->
|
||||
assert "$applyReceipt.managerIndependentVerifierReady" in function
|
||||
assert "$applyReceipt.managerRuntimeWritePerformed" in function
|
||||
assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function
|
||||
assert '[string]$applyReceipt.managerStartExit -eq "0"' in function
|
||||
assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function
|
||||
assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function
|
||||
assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function
|
||||
assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function
|
||||
assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function
|
||||
assert "$applyReceipt.durableReadbackTerminalMatch" in function
|
||||
assert "$durableReceiptReadback.durableReadbackReceiptLinkMatch" in function
|
||||
assert "$durableReceiptReadback.durableReadbackTerminalMatch" in function
|
||||
assert "$before.durableReadbackReceiptLinkMatch" in function
|
||||
assert "$before.durableReadbackTerminalMatch" in function
|
||||
assert "$afterManagerVerifierVerified" in function
|
||||
assert 'afterVerifierRole = "independent_second_verifier_only"' in function
|
||||
assert "$after.ready -and" in function
|
||||
|
||||
@@ -0,0 +1,419 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
|
||||
|
||||
|
||||
def _write_executable(path: Path, body: str) -> None:
|
||||
path.write_text(body, encoding="utf-8")
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _runtime_harness(
|
||||
tmp_path: Path,
|
||||
*,
|
||||
ledger_capacity: int = 128,
|
||||
) -> tuple[Path, dict[str, str], Path]:
|
||||
source = READINESS.read_text(encoding="utf-8")
|
||||
root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then'
|
||||
assert source.count(root_gate) == 1
|
||||
assert source.count("timer_ledger_capacity=128") == 1
|
||||
source = source.replace(root_gate, "if false; then", 1)
|
||||
source = source.replace(
|
||||
"timer_ledger_capacity=128",
|
||||
f"timer_ledger_capacity={ledger_capacity}",
|
||||
1,
|
||||
)
|
||||
script = tmp_path / "host112-runtime-harness.sh"
|
||||
_write_executable(script, source)
|
||||
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
fake_state = tmp_path / "fake-host"
|
||||
fake_state.mkdir()
|
||||
|
||||
_write_executable(
|
||||
fake_bin / "timeout",
|
||||
"""#!/usr/bin/env bash
|
||||
set -u
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--signal=*|--kill-after=*) shift ;;
|
||||
*) break ;;
|
||||
esac
|
||||
done
|
||||
[ "$#" -gt 0 ] || exit 64
|
||||
shift
|
||||
"$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "date",
|
||||
"""#!/usr/bin/env bash
|
||||
case "${1:-}" in
|
||||
--iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;;
|
||||
+%s) printf '%s\n' '1000' ;;
|
||||
*) /bin/date "$@" ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "flock",
|
||||
"""#!/usr/bin/env bash
|
||||
exit 0
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "mv",
|
||||
"""#!/usr/bin/env bash
|
||||
set -u
|
||||
destination=""
|
||||
for argument in "$@"; do destination="$argument"; done
|
||||
if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then
|
||||
: >"$FAKE_RECEIPT_MV_MARKER"
|
||||
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
|
||||
fi
|
||||
/bin/mv "$@"
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "pgrep",
|
||||
"""#!/usr/bin/env bash
|
||||
case "$*" in
|
||||
*wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;;
|
||||
*Xorg*|*Xwayland*) exit 0 ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "ss",
|
||||
"""#!/usr/bin/env bash
|
||||
case "$*" in
|
||||
*-ltn*)
|
||||
for port in 22 1514 1515 55000; do
|
||||
printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port"
|
||||
done
|
||||
;;
|
||||
*-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "df",
|
||||
"""#!/usr/bin/env bash
|
||||
printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
|
||||
printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
|
||||
""",
|
||||
)
|
||||
_write_executable(
|
||||
fake_bin / "systemctl",
|
||||
"""#!/usr/bin/env bash
|
||||
set -u
|
||||
state="${FAKE_HOST_STATE_DIR:?}"
|
||||
cmd="${1:-}"
|
||||
[ "$#" -gt 0 ] && shift
|
||||
case "$cmd" in
|
||||
show)
|
||||
unit="${1:-}"; shift || true
|
||||
property=""
|
||||
while [ "$#" -gt 0 ]; do
|
||||
if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi
|
||||
done
|
||||
case "$property" in
|
||||
LoadState) printf '%s\n' 'loaded' ;;
|
||||
ActiveState) printf '%s\n' 'active' ;;
|
||||
SubState) printf '%s\n' 'running' ;;
|
||||
Result) printf '%s\n' 'success' ;;
|
||||
*) printf '%s\n' 'unknown' ;;
|
||||
esac
|
||||
;;
|
||||
is-active)
|
||||
unit="${1:-}"
|
||||
case "$unit" in
|
||||
lightdm.service|display-manager.service)
|
||||
if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi
|
||||
;;
|
||||
*) printf '%s\n' 'active' ;;
|
||||
esac
|
||||
;;
|
||||
is-enabled) printf '%s\n' 'enabled' ;;
|
||||
is-system-running)
|
||||
if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then
|
||||
: >"$FAKE_FINALIZE_MARKER"
|
||||
sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}"
|
||||
fi
|
||||
printf '%s\n' 'running'
|
||||
;;
|
||||
get-default) printf '%s\n' 'graphical.target' ;;
|
||||
set-default)
|
||||
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
|
||||
exit "${FAKE_SET_DEFAULT_RC:-0}"
|
||||
;;
|
||||
start)
|
||||
unit="${1:-}"
|
||||
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
|
||||
if [ "$unit" = "lightdm.service" ]; then
|
||||
if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi
|
||||
rc="${FAKE_LIGHTDM_START_RC:-0}"
|
||||
if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi
|
||||
exit "$rc"
|
||||
fi
|
||||
;;
|
||||
enable)
|
||||
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
|
||||
;;
|
||||
reset-failed|stop) ;;
|
||||
*) exit 1 ;;
|
||||
esac
|
||||
""",
|
||||
)
|
||||
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"PATH": f"{fake_bin}:{env['PATH']}",
|
||||
"FAKE_HOST_STATE_DIR": str(fake_state),
|
||||
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
|
||||
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
|
||||
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
|
||||
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
|
||||
}
|
||||
)
|
||||
return script, env, fake_state
|
||||
|
||||
|
||||
def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(script),
|
||||
"--apply",
|
||||
"--trace-id",
|
||||
"trace:runtime-harness",
|
||||
"--run-id",
|
||||
"run:runtime-harness",
|
||||
"--work-item-id",
|
||||
"HOST112-RUNTIME-HARNESS",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=15,
|
||||
)
|
||||
|
||||
|
||||
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, fake_state = _runtime_harness(tmp_path)
|
||||
env["FAKE_LIGHTDM_START_RC"] = "124"
|
||||
env["FAKE_PARTIAL_LIGHTDM_START"] = "1"
|
||||
|
||||
failed = _apply(script, env)
|
||||
failed_output = failed.stdout + failed.stderr
|
||||
assert failed.returncode == 1
|
||||
assert "runtime_write_performed=1" in failed_output
|
||||
assert "receipt_status=written_immutable" in failed_output
|
||||
assert "durable_readback_status=written_immutable" in failed_output
|
||||
assert (
|
||||
"durable_receipt_terminal=execution_failed_or_unattributed_"
|
||||
"idempotent_already_verified_healthy"
|
||||
) in failed_output
|
||||
assert "guest_ready=1" in failed_output
|
||||
assert "action_failure_observed=1" in failed_output
|
||||
|
||||
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
||||
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
|
||||
receipt_before = receipt.read_bytes()
|
||||
readback_before = readback.read_bytes()
|
||||
|
||||
(fake_state / "lightdm_active").touch()
|
||||
env.pop("FAKE_LIGHTDM_START_RC")
|
||||
env.pop("FAKE_PARTIAL_LIGHTDM_START")
|
||||
replay = _apply(script, env)
|
||||
replay_output = replay.stdout + replay.stderr
|
||||
assert replay.returncode == 1
|
||||
assert "guest_ready=1" in replay_output
|
||||
assert "receipt_status=reused_immutable" in replay_output
|
||||
assert "durable_readback_status=reused_immutable_verified" in replay_output
|
||||
assert "terminal=immutable_receipt_replay" in replay_output
|
||||
assert receipt.read_bytes() == receipt_before
|
||||
assert readback.read_bytes() == readback_before
|
||||
|
||||
|
||||
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
noop_root = tmp_path / "noop"
|
||||
noop_root.mkdir()
|
||||
script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4)
|
||||
(fake_state / "lightdm_active").touch()
|
||||
|
||||
for index in range(5):
|
||||
run_env = env | {"INVOCATION_ID": f"noop-{index}"}
|
||||
result = subprocess.run(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=run_env,
|
||||
check=False,
|
||||
timeout=15,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
state = noop_root / "state"
|
||||
assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [
|
||||
Path("timer-observation.latest")
|
||||
]
|
||||
assert "repeat_count=5" in (state / "timer-observation.latest").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
mutation_root = tmp_path / "mutation"
|
||||
mutation_root.mkdir()
|
||||
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
|
||||
env["FAKE_LIGHTDM_START_RC"] = "124"
|
||||
for index in range(12):
|
||||
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
|
||||
result = subprocess.run(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=run_env,
|
||||
check=False,
|
||||
timeout=15,
|
||||
)
|
||||
output = result.stdout + result.stderr
|
||||
assert result.returncode == 1
|
||||
assert "runtime_write_performed=1" in output
|
||||
assert "receipt_status=written_bounded_timer_slot" in output
|
||||
assert "durable_readback_status=written_bounded_timer_slot" in output
|
||||
for field in (
|
||||
"durable_readback_present",
|
||||
"durable_readback_identity_match",
|
||||
"durable_readback_boot_match",
|
||||
"durable_readback_receipt_link_match",
|
||||
"durable_readback_terminal_match",
|
||||
):
|
||||
assert f"{field}=1" in output
|
||||
assert f"{field}=0" not in output
|
||||
assert "timer_no_runtime_mutation_aggregated" not in output
|
||||
|
||||
ledger_files = [
|
||||
path
|
||||
for path in (mutation_root / "state" / "timer-ledger").iterdir()
|
||||
if path.is_file()
|
||||
]
|
||||
assert len(ledger_files) <= 8
|
||||
assert not (mutation_root / "state" / "receipts").exists()
|
||||
assert not (mutation_root / "state" / "readbacks").exists()
|
||||
for readback in (mutation_root / "state" / "timer-ledger").glob(
|
||||
"*.readback.txt"
|
||||
):
|
||||
stored = readback.read_text(encoding="utf-8")
|
||||
for field in (
|
||||
"durable_readback_present",
|
||||
"durable_readback_identity_match",
|
||||
"durable_readback_boot_match",
|
||||
"durable_readback_receipt_link_match",
|
||||
"durable_readback_terminal_match",
|
||||
):
|
||||
assert stored.count(f"{field}=1") == 1
|
||||
assert f"{field}=0" not in stored
|
||||
|
||||
|
||||
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
|
||||
env["FAKE_LIGHTDM_START_RC"] = "124"
|
||||
|
||||
for index in range(12):
|
||||
marker = tmp_path / f"receipt-mv-{index}.entered"
|
||||
run_env = env | {
|
||||
"INVOCATION_ID": f"sigkill-{index}",
|
||||
"FAKE_RECEIPT_MV_MARKER": str(marker),
|
||||
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
|
||||
}
|
||||
process = subprocess.Popen(
|
||||
["bash", str(script), "--apply-timer"],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=run_env,
|
||||
start_new_session=True,
|
||||
)
|
||||
deadline = time.monotonic() + 8
|
||||
while not marker.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
assert marker.exists()
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
process.communicate(timeout=5)
|
||||
assert process.returncode == -signal.SIGKILL
|
||||
|
||||
ledger = tmp_path / "state" / "timer-ledger"
|
||||
files = [path for path in ledger.iterdir() if path.is_file()]
|
||||
assert len(files) <= 4
|
||||
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
|
||||
|
||||
|
||||
def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
script, env, _ = _runtime_harness(tmp_path)
|
||||
marker = tmp_path / "final-verifier.entered"
|
||||
env.update(
|
||||
{
|
||||
"FAKE_FINALIZE_MARKER": str(marker),
|
||||
"FAKE_FINALIZE_SLEEP_SECONDS": "2",
|
||||
}
|
||||
)
|
||||
process = subprocess.Popen(
|
||||
[
|
||||
"bash",
|
||||
str(script),
|
||||
"--apply",
|
||||
"--trace-id",
|
||||
"trace:runtime-harness",
|
||||
"--run-id",
|
||||
"run:runtime-harness",
|
||||
"--work-item-id",
|
||||
"HOST112-RUNTIME-HARNESS",
|
||||
],
|
||||
text=True,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
env=env,
|
||||
)
|
||||
deadline = time.monotonic() + 8
|
||||
while not marker.exists() and time.monotonic() < deadline:
|
||||
time.sleep(0.02)
|
||||
assert marker.exists()
|
||||
os.kill(process.pid, signal.SIGTERM)
|
||||
stdout, stderr = process.communicate(timeout=12)
|
||||
assert process.returncode == 75, stdout + stderr
|
||||
|
||||
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
||||
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
|
||||
receipt_text = receipt.read_text(encoding="utf-8")
|
||||
signal_text = signal_readback.read_text(encoding="utf-8")
|
||||
assert "terminal=interrupted_signal_TERM" in receipt_text
|
||||
assert "action_count=1 actions=start_lightdm" in receipt_text
|
||||
assert f"receipt_path={receipt}" in signal_text
|
||||
assert "terminal=interrupted_signal_TERM" in signal_text
|
||||
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
|
||||
|
||||
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
|
||||
assert replay.returncode == 65
|
||||
assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in (
|
||||
replay.stdout + replay.stderr
|
||||
)
|
||||
@@ -108,6 +108,7 @@ def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> No
|
||||
assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text
|
||||
assert 'systemctl start awoooi-host112-guest-recovery.service' not in text
|
||||
assert "grep -Fq 'TimeoutStartSec=480'" in text
|
||||
assert "for tool in bash cksum cmp flock" in text
|
||||
assert "TimeoutStartSec=600" not in text
|
||||
assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text
|
||||
assert 'RECOVERY_EXECUTION_PERFORMED=0' in text
|
||||
|
||||
@@ -108,7 +108,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "visudo -cf" in host112_installer
|
||||
assert "--rollback" in host112_installer
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
|
||||
assert "TimeoutStartSec=600" in host112_service
|
||||
assert "TimeoutStartSec=480" in host112_service
|
||||
assert "OnBootSec=20s" in host112_timer
|
||||
assert "OnUnitInactiveSec=60s" in host112_timer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user