c9bb4bbfc6
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m35s
CD Pipeline / build-and-deploy (push) Failing after 7m35s
CD Pipeline / post-deploy-checks (push) Has been skipped
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 16s
420 lines
13 KiB
Python
420 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import os
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
|
|
|
|
|
|
def _write_executable(path: Path, body: str) -> None:
|
|
path.write_text(body, encoding="utf-8")
|
|
path.chmod(0o755)
|
|
|
|
|
|
def _runtime_harness(
|
|
tmp_path: Path,
|
|
*,
|
|
ledger_capacity: int = 128,
|
|
) -> tuple[Path, dict[str, str], Path]:
|
|
source = READINESS.read_text(encoding="utf-8")
|
|
root_gate = 'if [ "${EUID:-$(id -u)}" -ne 0 ]; then'
|
|
assert source.count(root_gate) == 1
|
|
assert source.count("timer_ledger_capacity=128") == 1
|
|
source = source.replace(root_gate, "if false; then", 1)
|
|
source = source.replace(
|
|
"timer_ledger_capacity=128",
|
|
f"timer_ledger_capacity={ledger_capacity}",
|
|
1,
|
|
)
|
|
script = tmp_path / "host112-runtime-harness.sh"
|
|
_write_executable(script, source)
|
|
|
|
fake_bin = tmp_path / "bin"
|
|
fake_bin.mkdir()
|
|
fake_state = tmp_path / "fake-host"
|
|
fake_state.mkdir()
|
|
|
|
_write_executable(
|
|
fake_bin / "timeout",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--signal=*|--kill-after=*) shift ;;
|
|
*) break ;;
|
|
esac
|
|
done
|
|
[ "$#" -gt 0 ] || exit 64
|
|
shift
|
|
"$@"
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "date",
|
|
"""#!/usr/bin/env bash
|
|
case "${1:-}" in
|
|
--iso-8601=seconds) printf '%s\n' '2026-07-14T12:00:00+08:00' ;;
|
|
+%s) printf '%s\n' '1000' ;;
|
|
*) /bin/date "$@" ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "flock",
|
|
"""#!/usr/bin/env bash
|
|
exit 0
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "mv",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
destination=""
|
|
for argument in "$@"; do destination="$argument"; done
|
|
if [ -n "${FAKE_RECEIPT_MV_MARKER:-}" ] && [[ "$destination" == *.receipt.txt ]]; then
|
|
: >"$FAKE_RECEIPT_MV_MARKER"
|
|
sleep "${FAKE_RECEIPT_MV_SLEEP_SECONDS:-0}"
|
|
fi
|
|
/bin/mv "$@"
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "pgrep",
|
|
"""#!/usr/bin/env bash
|
|
case "$*" in
|
|
*wazuh-analysisd*) printf '%s\n' '1'; exit 0 ;;
|
|
*Xorg*|*Xwayland*) exit 0 ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "ss",
|
|
"""#!/usr/bin/env bash
|
|
case "$*" in
|
|
*-ltn*)
|
|
for port in 22 1514 1515 55000; do
|
|
printf 'LISTEN 0 128 0.0.0.0:%s 0.0.0.0:*\n' "$port"
|
|
done
|
|
;;
|
|
*-lun*) printf '%s\n' 'UNCONN 0 0 0.0.0.0:1514 0.0.0.0:*' ;;
|
|
esac
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "df",
|
|
"""#!/usr/bin/env bash
|
|
printf '%s\n' 'Filesystem 1024-blocks Used Available Capacity Mounted on'
|
|
printf '%s\n' '/dev/fake 10000000 1000 9999000 1% /'
|
|
""",
|
|
)
|
|
_write_executable(
|
|
fake_bin / "systemctl",
|
|
"""#!/usr/bin/env bash
|
|
set -u
|
|
state="${FAKE_HOST_STATE_DIR:?}"
|
|
cmd="${1:-}"
|
|
[ "$#" -gt 0 ] && shift
|
|
case "$cmd" in
|
|
show)
|
|
unit="${1:-}"; shift || true
|
|
property=""
|
|
while [ "$#" -gt 0 ]; do
|
|
if [ "$1" = "-p" ]; then property="${2:-}"; shift 2; else shift; fi
|
|
done
|
|
case "$property" in
|
|
LoadState) printf '%s\n' 'loaded' ;;
|
|
ActiveState) printf '%s\n' 'active' ;;
|
|
SubState) printf '%s\n' 'running' ;;
|
|
Result) printf '%s\n' 'success' ;;
|
|
*) printf '%s\n' 'unknown' ;;
|
|
esac
|
|
;;
|
|
is-active)
|
|
unit="${1:-}"
|
|
case "$unit" in
|
|
lightdm.service|display-manager.service)
|
|
if [ -f "$state/lightdm_active" ]; then printf '%s\n' 'active'; else printf '%s\n' 'inactive'; fi
|
|
;;
|
|
*) printf '%s\n' 'active' ;;
|
|
esac
|
|
;;
|
|
is-enabled) printf '%s\n' 'enabled' ;;
|
|
is-system-running)
|
|
if [ -n "${FAKE_FINALIZE_MARKER:-}" ]; then
|
|
: >"$FAKE_FINALIZE_MARKER"
|
|
sleep "${FAKE_FINALIZE_SLEEP_SECONDS:-0}"
|
|
fi
|
|
printf '%s\n' 'running'
|
|
;;
|
|
get-default) printf '%s\n' 'graphical.target' ;;
|
|
set-default)
|
|
printf 'set-default:%s\n' "${1:-}" >>"$state/mutations.log"
|
|
exit "${FAKE_SET_DEFAULT_RC:-0}"
|
|
;;
|
|
start)
|
|
unit="${1:-}"
|
|
printf 'start:%s\n' "$unit" >>"$state/mutations.log"
|
|
if [ "$unit" = "lightdm.service" ]; then
|
|
if [ "${FAKE_PARTIAL_LIGHTDM_START:-0}" = "1" ]; then : >"$state/lightdm_active"; fi
|
|
rc="${FAKE_LIGHTDM_START_RC:-0}"
|
|
if [ "$rc" -eq 0 ]; then : >"$state/lightdm_active"; fi
|
|
exit "$rc"
|
|
fi
|
|
;;
|
|
enable)
|
|
printf 'enable:%s\n' "${1:-}" >>"$state/mutations.log"
|
|
;;
|
|
reset-failed|stop) ;;
|
|
*) exit 1 ;;
|
|
esac
|
|
""",
|
|
)
|
|
|
|
env = os.environ.copy()
|
|
env.update(
|
|
{
|
|
"PATH": f"{fake_bin}:{env['PATH']}",
|
|
"FAKE_HOST_STATE_DIR": str(fake_state),
|
|
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
|
|
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
|
|
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
|
|
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
|
|
}
|
|
)
|
|
return script, env, fake_state
|
|
|
|
|
|
def _apply(script: Path, env: dict[str, str]) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[
|
|
"bash",
|
|
str(script),
|
|
"--apply",
|
|
"--trace-id",
|
|
"trace:runtime-harness",
|
|
"--run-id",
|
|
"run:runtime-harness",
|
|
"--work-item-id",
|
|
"HOST112-RUNTIME-HARNESS",
|
|
],
|
|
text=True,
|
|
capture_output=True,
|
|
env=env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
|
|
|
|
def test_failed_general_run_cannot_be_washed_green_by_external_recovery(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, fake_state = _runtime_harness(tmp_path)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
env["FAKE_PARTIAL_LIGHTDM_START"] = "1"
|
|
|
|
failed = _apply(script, env)
|
|
failed_output = failed.stdout + failed.stderr
|
|
assert failed.returncode == 1
|
|
assert "runtime_write_performed=1" in failed_output
|
|
assert "receipt_status=written_immutable" in failed_output
|
|
assert "durable_readback_status=written_immutable" in failed_output
|
|
assert (
|
|
"durable_receipt_terminal=execution_failed_or_unattributed_"
|
|
"idempotent_already_verified_healthy"
|
|
) in failed_output
|
|
assert "guest_ready=1" in failed_output
|
|
assert "action_failure_observed=1" in failed_output
|
|
|
|
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
|
readback = next((tmp_path / "state" / "readbacks").glob("*.txt"))
|
|
receipt_before = receipt.read_bytes()
|
|
readback_before = readback.read_bytes()
|
|
|
|
(fake_state / "lightdm_active").touch()
|
|
env.pop("FAKE_LIGHTDM_START_RC")
|
|
env.pop("FAKE_PARTIAL_LIGHTDM_START")
|
|
replay = _apply(script, env)
|
|
replay_output = replay.stdout + replay.stderr
|
|
assert replay.returncode == 1
|
|
assert "guest_ready=1" in replay_output
|
|
assert "receipt_status=reused_immutable" in replay_output
|
|
assert "durable_readback_status=reused_immutable_verified" in replay_output
|
|
assert "terminal=immutable_receipt_replay" in replay_output
|
|
assert receipt.read_bytes() == receipt_before
|
|
assert readback.read_bytes() == readback_before
|
|
|
|
|
|
def test_repeated_timer_noop_and_mutation_artifacts_are_actually_bounded(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
noop_root = tmp_path / "noop"
|
|
noop_root.mkdir()
|
|
script, env, fake_state = _runtime_harness(noop_root, ledger_capacity=4)
|
|
(fake_state / "lightdm_active").touch()
|
|
|
|
for index in range(5):
|
|
run_env = env | {"INVOCATION_ID": f"noop-{index}"}
|
|
result = subprocess.run(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=run_env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
assert result.returncode == 0, result.stdout + result.stderr
|
|
|
|
state = noop_root / "state"
|
|
assert [path.relative_to(state) for path in state.rglob("*") if path.is_file()] == [
|
|
Path("timer-observation.latest")
|
|
]
|
|
assert "repeat_count=5" in (state / "timer-observation.latest").read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
mutation_root = tmp_path / "mutation"
|
|
mutation_root.mkdir()
|
|
script, env, _ = _runtime_harness(mutation_root, ledger_capacity=4)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
for index in range(12):
|
|
run_env = env | {"INVOCATION_ID": f"mutation-{index}"}
|
|
result = subprocess.run(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
capture_output=True,
|
|
env=run_env,
|
|
check=False,
|
|
timeout=15,
|
|
)
|
|
output = result.stdout + result.stderr
|
|
assert result.returncode == 1
|
|
assert "runtime_write_performed=1" in output
|
|
assert "receipt_status=written_bounded_timer_slot" in output
|
|
assert "durable_readback_status=written_bounded_timer_slot" in output
|
|
for field in (
|
|
"durable_readback_present",
|
|
"durable_readback_identity_match",
|
|
"durable_readback_boot_match",
|
|
"durable_readback_receipt_link_match",
|
|
"durable_readback_terminal_match",
|
|
):
|
|
assert f"{field}=1" in output
|
|
assert f"{field}=0" not in output
|
|
assert "timer_no_runtime_mutation_aggregated" not in output
|
|
|
|
ledger_files = [
|
|
path
|
|
for path in (mutation_root / "state" / "timer-ledger").iterdir()
|
|
if path.is_file()
|
|
]
|
|
assert len(ledger_files) <= 8
|
|
assert not (mutation_root / "state" / "receipts").exists()
|
|
assert not (mutation_root / "state" / "readbacks").exists()
|
|
for readback in (mutation_root / "state" / "timer-ledger").glob(
|
|
"*.readback.txt"
|
|
):
|
|
stored = readback.read_text(encoding="utf-8")
|
|
for field in (
|
|
"durable_readback_present",
|
|
"durable_readback_identity_match",
|
|
"durable_readback_boot_match",
|
|
"durable_readback_receipt_link_match",
|
|
"durable_readback_terminal_match",
|
|
):
|
|
assert stored.count(f"{field}=1") == 1
|
|
assert f"{field}=0" not in stored
|
|
|
|
|
|
def test_timer_sigkill_during_receipt_commit_leaves_only_bounded_slot_temps(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, _ = _runtime_harness(tmp_path, ledger_capacity=4)
|
|
env["FAKE_LIGHTDM_START_RC"] = "124"
|
|
|
|
for index in range(12):
|
|
marker = tmp_path / f"receipt-mv-{index}.entered"
|
|
run_env = env | {
|
|
"INVOCATION_ID": f"sigkill-{index}",
|
|
"FAKE_RECEIPT_MV_MARKER": str(marker),
|
|
"FAKE_RECEIPT_MV_SLEEP_SECONDS": "5",
|
|
}
|
|
process = subprocess.Popen(
|
|
["bash", str(script), "--apply-timer"],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=run_env,
|
|
start_new_session=True,
|
|
)
|
|
deadline = time.monotonic() + 8
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.01)
|
|
assert marker.exists()
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
process.communicate(timeout=5)
|
|
assert process.returncode == -signal.SIGKILL
|
|
|
|
ledger = tmp_path / "state" / "timer-ledger"
|
|
files = [path for path in ledger.iterdir() if path.is_file()]
|
|
assert len(files) <= 4
|
|
assert all(path.name.endswith(".receipt.txt.tmp") for path in files)
|
|
|
|
|
|
def test_signal_during_final_verifier_writes_correlated_failure_chain(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
script, env, _ = _runtime_harness(tmp_path)
|
|
marker = tmp_path / "final-verifier.entered"
|
|
env.update(
|
|
{
|
|
"FAKE_FINALIZE_MARKER": str(marker),
|
|
"FAKE_FINALIZE_SLEEP_SECONDS": "2",
|
|
}
|
|
)
|
|
process = subprocess.Popen(
|
|
[
|
|
"bash",
|
|
str(script),
|
|
"--apply",
|
|
"--trace-id",
|
|
"trace:runtime-harness",
|
|
"--run-id",
|
|
"run:runtime-harness",
|
|
"--work-item-id",
|
|
"HOST112-RUNTIME-HARNESS",
|
|
],
|
|
text=True,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
env=env,
|
|
)
|
|
deadline = time.monotonic() + 8
|
|
while not marker.exists() and time.monotonic() < deadline:
|
|
time.sleep(0.02)
|
|
assert marker.exists()
|
|
os.kill(process.pid, signal.SIGTERM)
|
|
stdout, stderr = process.communicate(timeout=12)
|
|
assert process.returncode == 75, stdout + stderr
|
|
|
|
receipt = next((tmp_path / "state" / "receipts").glob("*.txt"))
|
|
signal_readback = next((tmp_path / "state" / "readbacks").glob("*.signal.txt"))
|
|
receipt_text = receipt.read_text(encoding="utf-8")
|
|
signal_text = signal_readback.read_text(encoding="utf-8")
|
|
assert "terminal=interrupted_signal_TERM" in receipt_text
|
|
assert "action_count=1 actions=start_lightdm" in receipt_text
|
|
assert f"receipt_path={receipt}" in signal_text
|
|
assert "terminal=interrupted_signal_TERM" in signal_text
|
|
assert "schema_version=host112_guest_recovery_signal_readback_v1" in signal_text
|
|
|
|
replay = _apply(script, env | {"FAKE_FINALIZE_MARKER": ""})
|
|
assert replay.returncode == 65
|
|
assert "BLOCKER durable_readback_missing_or_mismatched_for_receipt" in (
|
|
replay.stdout + replay.stderr
|
|
)
|