fix(recovery): bind host112 closure artifacts
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m35s
CD Pipeline / build-and-deploy (push) Failing after 7m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s

This commit is contained in:
ogt
2026-07-14 12:35:15 +08:00
parent d7a17e58b2
commit c9bb4bbfc6
12 changed files with 978 additions and 69 deletions

View File

@@ -149,6 +149,8 @@ def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
) == 1
assert "manager_attempt_count=1" in source
assert 'if [ "$manager_start_exit" = "0" ]; then' in source
assert 'manager_terminal="manager_start_nonzero_verifier_healthy_unattributed"' in source
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
assert 'rollback_terminal="bounded_stop_verified"' in source
assert "schema_version=host112_wazuh_manager_recovery_receipt_v2" in source
@@ -171,7 +173,7 @@ def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None
marker_function = source[
source.index("write_epoch_marker() {") : source.index("unit_active() {")
]
assert 'tmp="${marker}.tmp.$$"' in marker_function
assert 'tmp="${marker}.tmp"' in marker_function
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
assert 'chmod 0644 "$tmp"' in marker_function
assert 'mv "$tmp" "$marker"' in marker_function
@@ -251,7 +253,15 @@ def test_receipts_and_readbacks_are_immutable_boot_bound_and_replay_safe() -> No
assert "printf '%s\\0%s\\0%s\\0'" in source
assert 'boot_id=$boot_id' in source
assert 'ln "$tmp" "$receipt_path"' in source
assert 'mv "$tmp" "$receipt_path"' not in source
receipt_writer = source[
source.index("write_public_receipt() {") : source.index(
"write_signal_readback() {",
)
]
assert 'if [ "$MODE" = "timer_apply" ]' in receipt_writer
assert 'mv "$tmp" "$receipt_path"' in receipt_writer
assert 'elif [ "$MODE" != "timer_apply" ]' in receipt_writer
assert 'ln "$tmp" "$receipt_path"' in receipt_writer
assert 'receipt_status="reused_immutable"' in source
assert 'manager_terminal="immutable_receipt_replay"' in source
assert "immutable_receipt_identity_or_boot_mismatch" in source
@@ -279,6 +289,17 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
"boot_id=unknown terminal=verified_healthy\n",
encoding="utf-8",
)
readback_dir = tmp_path / "state" / "readbacks"
readback_dir.mkdir(parents=True)
durable_readback = readback_dir / f"{run_id}-{digest}.txt"
receipt_sha = hashlib.sha256(receipt.read_bytes()).hexdigest()
durable_readback.write_text(
"schema_version=host112_guest_recovery_v2 "
f"trace_id={trace_id} run_id={run_id} work_item_id={work_item_id} "
"boot_id=unknown durable_receipt_terminal=verified_healthy "
f"receipt_path={receipt} receipt_sha256={receipt_sha}\n",
encoding="utf-8",
)
original = receipt.read_bytes()
env = isolated_env(tmp_path)
args = [
@@ -301,6 +322,9 @@ def test_check_reads_exact_immutable_receipt_and_rejects_boot_mismatch(
assert "durable_receipt_identity_match=1" in matched_output
assert "durable_receipt_boot_match=1" in matched_output
assert "durable_receipt_terminal=verified_healthy" in matched_output
assert "durable_readback_present=1" in matched_output
assert "durable_readback_receipt_link_match=1" in matched_output
assert "durable_readback_terminal_match=1" in matched_output
assert receipt.read_bytes() == original
receipt.write_text(
@@ -328,12 +352,56 @@ def test_executor_has_absolute_deadline_bounded_reads_and_signal_receipt() -> No
assert "trap 'handle_executor_signal TERM' TERM" in source
assert "trap 'handle_executor_signal INT' INT" in source
assert "trap 'handle_executor_signal HUP' HUP" in source
assert "write_public_receipt || true" in source
assert "if write_public_receipt && write_signal_readback; then" in source
assert "action_count=${#actions[@]} actions=$receipt_action_text" in source
assert "schema_version=host112_guest_recovery_signal_readback_v1" in source
assert 'ln "$tmp" "$signal_readback_path"' in source
assert "signal_readback_status=\"written_immutable\"" in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl show' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" ss -H -ltn' in source
assert 'bounded_timeout "$READ_TIMEOUT_SECONDS" df -Pk' in source
def test_timer_noop_uses_one_bounded_aggregate_instead_of_per_minute_artifacts() -> None:
source = read(READINESS)
function = source[
source.index("write_bounded_timer_observation() {") : source.index(
'boot_id="$(cat /proc/sys/kernel/random/boot_id',
)
]
assert 'timer_observation_path="$STATE_DIR/timer-observation.latest"' in function
assert "schema_version=host112_timer_observation_aggregate_v1" in function
assert "aggregation=fixed_single_atomic_pointer" in function
assert 'mv "$tmp" "$timer_observation_path"' in function
assert "repeat_count=$timer_observation_repeat_count" in function
assert 'if [ "$MODE" = "apply" ] || [ "$runtime_write_performed" -eq 1 ]; then' in source
assert 'receipt_status="timer_no_runtime_mutation_aggregated"' in source
assert 'durable_readback_status="skipped_bounded_timer_no_runtime_mutation"' in source
assert "timer_ledger_capacity=128" in source
assert "write_bounded_timer_mutation_readback()" in source
assert "retention=rolling_atomic_overwrite" in source
assert 'timer_artifact_policy="bounded_ring_128_for_runtime_mutation"' in source
assert 'timer_artifact_policy="bounded_mutable_observation_no_immutable_artifact"' in source
def test_apply_success_is_bound_to_same_run_durable_terminal() -> None:
source = read(READINESS)
terminal_gate = source[source.rindex('if [ "$MODE" = "apply" ]') :]
assert 'successful_terminal="$run_terminal"' in terminal_gate
assert 'successful_terminal="$durable_receipt_terminal"' in terminal_gate
assert '[ "$successful_terminal" = "verified_healthy" ]' in terminal_gate
assert '[ "$successful_terminal" = "idempotent_already_verified_healthy" ]' in terminal_gate
guest_verifier = source.index('guest_ready=0\nif [ "$console_ready" -eq 1 ]')
receipt_commit = source.index("if ! write_public_receipt; then", guest_verifier)
assert guest_verifier < receipt_commit
assert 'run_terminal="guest_verification_failed_${manager_terminal}"' in source
assert 'run_terminal="execution_failed_or_unattributed_${manager_terminal}"' in source
assert 'action_failure_observed=1' in source
assert "manager_terminal=$manager_terminal terminal=$run_terminal" in source
def test_general_guest_repair_includes_recovery_timer_independent_of_manager() -> None:
source = read(READINESS)
general_start = source.index(
@@ -424,10 +492,17 @@ def test_agent99_manager_success_requires_apply_and_immutable_receipt_chain() ->
assert "$applyReceipt.managerIndependentVerifierReady" in function
assert "$applyReceipt.managerRuntimeWritePerformed" in function
assert '[string]$applyReceipt.managerAttemptCount -eq "1"' in function
assert '[string]$applyReceipt.managerStartExit -eq "0"' in function
assert "$durableReceiptReadback.receiptPath -eq $applyReceipt.receiptPath" in function
assert "$durableReceiptReadback.receiptSha256 -eq $applyReceipt.receiptSha256" in function
assert "$durableReceiptReadback.durableReceiptTerminal -eq $applyReceipt.durableReceiptTerminal" in function
assert '$durableReceiptReadback.durableReceiptTerminal -eq "verified_healthy"' in function
assert "$applyReceipt.durableReadbackReceiptLinkMatch" in function
assert "$applyReceipt.durableReadbackTerminalMatch" in function
assert "$durableReceiptReadback.durableReadbackReceiptLinkMatch" in function
assert "$durableReceiptReadback.durableReadbackTerminalMatch" in function
assert "$before.durableReadbackReceiptLinkMatch" in function
assert "$before.durableReadbackTerminalMatch" in function
assert "$afterManagerVerifierVerified" in function
assert 'afterVerifierRole = "independent_second_verifier_only"' in function
assert "$after.ready -and" in function

View File

@@ -0,0 +1,419 @@
from __future__ import annotations
import os
import signal
import subprocess
import time
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
def _write_executable(path: Path, body: str) -> None:
path.write_text(body, encoding="utf-8")
path.chmod(0o755)
def _runtime_harness(
tmp_path: Path,
*,
ledger_capacity: int = 128,
) -> tuple[Path, dict[str, str], Path]:
source = READINESS.read_text(encoding="utf-8")
root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then'
assert source.count(root_gate) == 1
assert source.count("timer_ledger_capacity=128") == 1
source = source.replace(root_gate, "if false; then", 1)
source = source.replace(
"timer_ledger_capacity=128",
f"timer_ledger_capacity={ledger_capacity}",
1,
)
script = tmp_path / "host112-runtime-harness.sh"
_write_executable(script, source)
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
fake_state = tmp_path / "fake-host"
fake_state.mkdir()
_write_executable(
fake_bin / "timeout",
"""#!/usr/bin/env bash
set -u
while [ "$#" -gt 0 ]; do
case "$1" in
--signal=*|--kill-after=*) shift ;;
*) break ;;
esac
done
[ "$#" -gt 0 ] || exit 64
shift
"$@"
""",
)
_write_executable(
fake_bin / "date",
"""#!/usr/bin/env bash
case "${1:-}" in
--iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;;
+%s) printf '%s\n' '1000' ;;
*) /bin/date "$@" ;;
esac
""",
)
_write_executable(
fake_bin / "flock",
"""#!/usr/bin/env bash
exit 0
""",
)
_write_executable(
fake_bin / "mv",
"""#!/usr/bin/env bash
set -u
destination=""
for argument in "$@"; do destination="$argument"; done
if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then
: >"$FAKE_RECEIPT_MV_MARKER"
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
fi
/bin/mv "$@"
""",
)
_write_executable(
fake_bin / "pgrep",
"""#!/usr/bin/env bash
case "$*" in
*wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;;
*Xorg*|*Xwayland*) exit 0 ;;
*) exit 1 ;;
esac
""",
)
_write_executable(
fake_bin / "ss",
"""#!/usr/bin/env bash
case "$*" in
*-ltn*)
for port in 22 1514 1515 55000; do
printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port"
done
;;
*-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;;
esac
""",
)
_write_executable(
fake_bin / "df",
"""#!/usr/bin/env bash
printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
""",
)
_write_executable(
fake_bin / "systemctl",
"""#!/usr/bin/env bash
set -u
state="${FAKE_HOST_STATE_DIR:?}"
cmd="${1:-}"
[ "$#" -gt 0 ] && shift
case "$cmd" in
show)
unit="${1:-}"; shift || true
property=""
while [ "$#" -gt 0 ]; do
if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi
done
case "$property" in
LoadState) printf '%s\n' 'loaded' ;;
ActiveState) printf '%s\n' 'active' ;;
SubState) printf '%s\n' 'running' ;;
Result) printf '%s\n' 'success' ;;
*) printf '%s\n' 'unknown' ;;
esac
;;
is-active)
unit="${1:-}"
case "$unit" in
lightdm.service|display-manager.service)
if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi
;;
*) printf '%s\n' 'active' ;;
esac
;;
is-enabled) printf '%s\n' 'enabled' ;;
is-system-running)
if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then
: >"$FAKE_FINALIZE_MARKER"
sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}"
fi
printf '%s\n' 'running'
;;
get-default) printf '%s\n' 'graphical.target' ;;
set-default)
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
exit "${FAKE_SET_DEFAULT_RC:-0}"
;;
start)
unit="${1:-}"
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
if [ "$unit" = "lightdm.service" ]; then
if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi
rc="${FAKE_LIGHTDM_START_RC:-0}"
if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi
exit "$rc"
fi
;;
enable)
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
;;
reset-failed|stop) ;;
*) exit 1 ;;
esac
""",
)
env = os.environ.copy()
env.update(
{
"PATH": f"{fake_bin}:{env['PATH']}",
"FAKE_HOST_STATE_DIR": str(fake_state),
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
}
)
return script, env, fake_state
def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
"bash",
str(script),
"--apply",
"--trace-id",
"trace:runtime-harness",
"--run-id",
"run:runtime-harness",
"--work-item-id",
"HOST112-RUNTIME-HARNESS",
],
text=True,
capture_output=True,
env=env,
check=False,
timeout=15,
)
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
tmp_path: Path,
) -> None:
script, env, fake_state = _runtime_harness(tmp_path)
env["FAKE_LIGHTDM_START_RC"] = "124"
env["FAKE_PARTIAL_LIGHTDM_START"] = "1"
failed = _apply(script, env)
failed_output = failed.stdout + failed.stderr
assert failed.returncode == 1
assert "runtime_write_performed=1" in failed_output
assert "receipt_status=written_immutable" in failed_output
assert "durable_readback_status=written_immutable" in failed_output
assert (
"durable_receipt_terminal=execution_failed_or_unattributed_"
"idempotent_already_verified_healthy"
) in failed_output
assert "guest_ready=1" in failed_output
assert "action_failure_observed=1" in failed_output
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
receipt_before = receipt.read_bytes()
readback_before = readback.read_bytes()
(fake_state / "lightdm_active").touch()
env.pop("FAKE_LIGHTDM_START_RC")
env.pop("FAKE_PARTIAL_LIGHTDM_START")
replay = _apply(script, env)
replay_output = replay.stdout + replay.stderr
assert replay.returncode == 1
assert "guest_ready=1" in replay_output
assert "receipt_status=reused_immutable" in replay_output
assert "durable_readback_status=reused_immutable_verified" in replay_output
assert "terminal=immutable_receipt_replay" in replay_output
assert receipt.read_bytes() == receipt_before
assert readback.read_bytes() == readback_before
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
tmp_path: Path,
) -> None:
noop_root = tmp_path / "noop"
noop_root.mkdir()
script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4)
(fake_state / "lightdm_active").touch()
for index in range(5):
run_env = env | {"INVOCATION_ID": f"noop-{index}"}
result = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
capture_output=True,
env=run_env,
check=False,
timeout=15,
)
assert result.returncode == 0, result.stdout + result.stderr
state = noop_root / "state"
assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [
Path("timer-observation.latest")
]
assert "repeat_count=5" in (state / "timer-observation.latest").read_text(
encoding="utf-8"
)
mutation_root = tmp_path / "mutation"
mutation_root.mkdir()
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
result = subprocess.run(
["bash", str(script), "--apply-timer"],
text=True,
capture_output=True,
env=run_env,
check=False,
timeout=15,
)
output = result.stdout + result.stderr
assert result.returncode == 1
assert "runtime_write_performed=1" in output
assert "receipt_status=written_bounded_timer_slot" in output
assert "durable_readback_status=written_bounded_timer_slot" in output
for field in (
"durable_readback_present",
"durable_readback_identity_match",
"durable_readback_boot_match",
"durable_readback_receipt_link_match",
"durable_readback_terminal_match",
):
assert f"{field}=1" in output
assert f"{field}=0" not in output
assert "timer_no_runtime_mutation_aggregated" not in output
ledger_files = [
path
for path in (mutation_root / "state" / "timer-ledger").iterdir()
if path.is_file()
]
assert len(ledger_files) <= 8
assert not (mutation_root / "state" / "receipts").exists()
assert not (mutation_root / "state" / "readbacks").exists()
for readback in (mutation_root / "state" / "timer-ledger").glob(
"*.readback.txt"
):
stored = readback.read_text(encoding="utf-8")
for field in (
"durable_readback_present",
"durable_readback_identity_match",
"durable_readback_boot_match",
"durable_readback_receipt_link_match",
"durable_readback_terminal_match",
):
assert stored.count(f"{field}=1") == 1
assert f"{field}=0" not in stored
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
tmp_path: Path,
) -> None:
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
env["FAKE_LIGHTDM_START_RC"] = "124"
for index in range(12):
marker = tmp_path / f"receipt-mv-{index}.entered"
run_env = env | {
"INVOCATION_ID": f"sigkill-{index}",
"FAKE_RECEIPT_MV_MARKER": str(marker),
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
}
process = subprocess.Popen(
["bash", str(script), "--apply-timer"],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=run_env,
start_new_session=True,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.01)
assert marker.exists()
os.killpg(process.pid, signal.SIGKILL)
process.communicate(timeout=5)
assert process.returncode == -signal.SIGKILL
ledger = tmp_path / "state" / "timer-ledger"
files = [path for path in ledger.iterdir() if path.is_file()]
assert len(files) <= 4
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
def test_signal_during_final_verifier_writes_correlated_failure_chain(
tmp_path: Path,
) -> None:
script, env, _ = _runtime_harness(tmp_path)
marker = tmp_path / "final-verifier.entered"
env.update(
{
"FAKE_FINALIZE_MARKER": str(marker),
"FAKE_FINALIZE_SLEEP_SECONDS": "2",
}
)
process = subprocess.Popen(
[
"bash",
str(script),
"--apply",
"--trace-id",
"trace:runtime-harness",
"--run-id",
"run:runtime-harness",
"--work-item-id",
"HOST112-RUNTIME-HARNESS",
],
text=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
)
deadline = time.monotonic() + 8
while not marker.exists() and time.monotonic() < deadline:
time.sleep(0.02)
assert marker.exists()
os.kill(process.pid, signal.SIGTERM)
stdout, stderr = process.communicate(timeout=12)
assert process.returncode == 75, stdout + stderr
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
receipt_text = receipt.read_text(encoding="utf-8")
signal_text = signal_readback.read_text(encoding="utf-8")
assert "terminal=interrupted_signal_TERM" in receipt_text
assert "action_count=1 actions=start_lightdm" in receipt_text
assert f"receipt_path={receipt}" in signal_text
assert "terminal=interrupted_signal_TERM" in signal_text
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
assert replay.returncode == 65
assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in (
replay.stdout + replay.stderr
)

View File

@@ -108,6 +108,7 @@ def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> No
assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text
assert 'systemctl start awoooi-host112-guest-recovery.service' not in text
assert "grep -Fq 'TimeoutStartSec=480'" in text
assert "for tool in bash cksum cmp flock" in text
assert "TimeoutStartSec=600" not in text
assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text
assert 'RECOVERY_EXECUTION_PERFORMED=0' in text

View File

@@ -108,7 +108,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
assert "visudo -cf" in host112_installer
assert "--rollback" in host112_installer
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
assert "TimeoutStartSec=600" in host112_service
assert "TimeoutStartSec=480" in host112_service
assert "OnBootSec=20s" in host112_timer
assert "OnUnitInactiveSec=60s" in host112_timer