fix(recovery): close host112 wazuh manager loop
This commit is contained in:
ogt
@@ -0,0 +1,281 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
READINESS = ROOT / "scripts/reboot-recovery/host112-guest-readiness.sh"
|
||||
INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh"
|
||||
SERVICE = ROOT / "scripts/reboot-recovery/awoooi-host112-guest-recovery.service"
|
||||
CONTROL = ROOT / "agent99-control-plane.ps1"
|
||||
BOOTSTRAP = ROOT / "agent99-bootstrap.ps1"
|
||||
|
||||
|
||||
def read(path: Path) -> str:
|
||||
return path.read_text(encoding="utf-8")
|
||||
|
||||
|
||||
def isolated_env(tmp_path: Path) -> dict[str, str]:
|
||||
env = os.environ.copy()
|
||||
env.update(
|
||||
{
|
||||
"HOST112_RECOVERY_STATE_DIR": str(tmp_path / "state"),
|
||||
"HOST112_INDEXER_RETRY_MARKER": str(tmp_path / "indexer.epoch"),
|
||||
"HOST112_MANAGER_RETRY_MARKER": str(tmp_path / "manager.epoch"),
|
||||
"HOST112_MANAGER_LOCK_FILE": str(tmp_path / "manager.lock"),
|
||||
}
|
||||
)
|
||||
return env
|
||||
|
||||
|
||||
def test_check_and_dry_run_are_real_no_write_execution_paths(tmp_path: Path) -> None:
|
||||
identity = [
|
||||
"--trace-id",
|
||||
"trace:D037-host112-dry-run",
|
||||
"--run-id",
|
||||
"run:D037-host112-dry-run",
|
||||
"--work-item-id",
|
||||
"D037-WAZUH-HOST112",
|
||||
]
|
||||
env = isolated_env(tmp_path)
|
||||
|
||||
check = subprocess.run(
|
||||
["bash", str(READINESS), "--check", *identity],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=20,
|
||||
)
|
||||
dry_run = subprocess.run(
|
||||
["bash", str(READINESS), "--dry-run", *identity],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=20,
|
||||
)
|
||||
|
||||
assert check.returncode in {0, 1}
|
||||
assert dry_run.returncode in {0, 1}
|
||||
for result, mode in ((check, "check"), (dry_run, "dry_run")):
|
||||
output = result.stdout + result.stderr
|
||||
assert f"mode={mode}" in output
|
||||
assert "trace_id=trace:D037-host112-dry-run" in output
|
||||
assert "run_id=run:D037-host112-dry-run" in output
|
||||
assert "work_item_id=D037-WAZUH-HOST112" in output
|
||||
assert "runtime_write_performed=0" in output
|
||||
assert "manager_runtime_write_performed=0" in output
|
||||
assert "receipt_write_performed=0" in output
|
||||
assert "manager_preflight_status=" in output
|
||||
assert "apply_phase_timeout_budget_seconds=500" in output
|
||||
assert not (tmp_path / "state").exists()
|
||||
assert not (tmp_path / "indexer.epoch").exists()
|
||||
assert not (tmp_path / "manager.epoch").exists()
|
||||
assert not (tmp_path / "manager.lock").exists()
|
||||
|
||||
|
||||
def test_unsafe_or_incomplete_identity_fails_before_any_state_path(tmp_path: Path) -> None:
|
||||
env = isolated_env(tmp_path)
|
||||
unsafe = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(READINESS),
|
||||
"--dry-run",
|
||||
"--trace-id",
|
||||
"unsafe value",
|
||||
"--run-id",
|
||||
"run-safe",
|
||||
"--work-item-id",
|
||||
"D037-WAZUH-HOST112",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
incomplete = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(READINESS),
|
||||
"--dry-run",
|
||||
"--trace-id",
|
||||
"trace-safe",
|
||||
],
|
||||
text=True,
|
||||
capture_output=True,
|
||||
env=env,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
|
||||
assert unsafe.returncode == 64
|
||||
assert "BLOCKER unsafe_control_identity" in (unsafe.stdout + unsafe.stderr)
|
||||
assert incomplete.returncode == 64
|
||||
assert "BLOCKER incomplete_control_identity" in (
|
||||
incomplete.stdout + incomplete.stderr
|
||||
)
|
||||
assert not any(tmp_path.iterdir())
|
||||
|
||||
|
||||
def test_manager_preflight_verifier_receipt_and_rollback_are_complete() -> None:
|
||||
source = read(READINESS)
|
||||
|
||||
for field in (
|
||||
"manager_failed_state_observed",
|
||||
"manager_orphan_analysisd",
|
||||
"manager_memory_available_kb",
|
||||
"manager_disk_available_kb",
|
||||
"manager_disk_used_percent",
|
||||
"wazuh_analysisd_process_count",
|
||||
"wazuh_agent_event_port_1514_tcp",
|
||||
"wazuh_agent_event_port_1514_udp",
|
||||
"wazuh_agent_enrollment_port_1515_tcp",
|
||||
"wazuh_manager_api_port_55000_tcp",
|
||||
"manager_independent_verifier_ready",
|
||||
):
|
||||
assert field in source
|
||||
assert 'manager_active_state="$(unit_property wazuh-manager.service ActiveState)"' in source
|
||||
assert 'manager_result="$(unit_property wazuh-manager.service Result)"' in source
|
||||
assert 'analysisd_count="$(process_count wazuh-analysisd)"' in source
|
||||
assert source.count(
|
||||
'timeout "$MANAGER_START_TIMEOUT_SECONDS" systemctl start wazuh-manager.service'
|
||||
) == 1
|
||||
assert "manager_attempt_count=1" in source
|
||||
assert 'timeout "$MANAGER_ROLLBACK_TIMEOUT_SECONDS" systemctl stop wazuh-manager.service' in source
|
||||
assert 'rollback_terminal="bounded_stop_verified"' in source
|
||||
assert "schema_version=host112_wazuh_manager_recovery_receipt_v1" in source
|
||||
assert "trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID" in source
|
||||
assert 'receipt_status="write_failed"' in source
|
||||
assert "prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" in source
|
||||
|
||||
preflight = source[
|
||||
source.index("evaluate_manager_preflight() {") : source.index(
|
||||
"wait_for_manager_verifier() {"
|
||||
)
|
||||
]
|
||||
assert 'elif [ "$(unit_active wazuh-indexer.service)" != "active" ]; then' in preflight
|
||||
assert 'manager_preflight_status="blocked_indexer_dependency_no_write"' in preflight
|
||||
|
||||
|
||||
def test_cooldown_marker_write_is_atomic_and_fails_closed_before_start() -> None:
|
||||
source = read(READINESS)
|
||||
marker_function = source[
|
||||
source.index("write_epoch_marker() {") : source.index("unit_active() {")
|
||||
]
|
||||
assert 'tmp="${marker}.tmp.$$"' in marker_function
|
||||
assert 'printf \'%s\\n\' "$epoch" >"$tmp"' in marker_function
|
||||
assert 'chmod 0644 "$tmp"' in marker_function
|
||||
assert 'mv "$tmp" "$marker"' in marker_function
|
||||
assert "return 1" in marker_function
|
||||
|
||||
manager_start = source.index(
|
||||
' if write_epoch_marker "$MANAGER_RETRY_MARKER" "$now_epoch"; then'
|
||||
)
|
||||
manager_end = source.index(" ;;", manager_start)
|
||||
manager_branch = source[manager_start:manager_end]
|
||||
marker_failure_at = manager_branch.index(
|
||||
' else\n record_action "wazuh_manager_marker_write_failed_no_write"'
|
||||
)
|
||||
success = manager_branch[:marker_failure_at]
|
||||
failure = manager_branch[marker_failure_at:]
|
||||
assert "systemctl reset-failed wazuh-manager.service" in success
|
||||
assert "systemctl start wazuh-manager.service" in success
|
||||
assert 'timeout "$MANAGER_RESET_TIMEOUT_SECONDS" systemctl reset-failed wazuh-manager.service' in success
|
||||
assert "manager_attempt_count=1" in success
|
||||
assert "systemctl reset-failed wazuh-manager.service" not in failure
|
||||
assert "systemctl start wazuh-manager.service" not in failure
|
||||
assert "wazuh_manager_marker_write_failed_no_write" in failure
|
||||
assert 'manager_terminal="no_write_cooldown_marker_write_failed"' in failure
|
||||
|
||||
indexer_start = source.index(
|
||||
' if write_epoch_marker "$INDEXER_RETRY_MARKER" "$now_epoch"; then'
|
||||
)
|
||||
indexer_end = source.index(" else\n record_action", indexer_start)
|
||||
indexer_branch = source[indexer_start:indexer_end]
|
||||
indexer_success, indexer_failure = indexer_branch.split(" else\n", maxsplit=1)
|
||||
assert "systemctl reset-failed wazuh-indexer.service" in indexer_success
|
||||
assert "systemctl start wazuh-indexer.service" in indexer_success
|
||||
assert "systemctl reset-failed wazuh-indexer.service" not in indexer_failure
|
||||
assert "systemctl start wazuh-indexer.service" not in indexer_failure
|
||||
assert "wazuh_indexer_marker_write_failed_no_write" in indexer_failure
|
||||
|
||||
|
||||
def test_phase_budget_and_systemd_outer_timeout_are_aligned() -> None:
|
||||
source = read(READINESS)
|
||||
service = read(SERVICE)
|
||||
control = read(CONTROL)
|
||||
|
||||
assert 'SERVICE_START_TIMEOUT_SECONDS="${HOST112_SERVICE_START_TIMEOUT_SECONDS:-30}"' in source
|
||||
assert 'INDEXER_START_TIMEOUT_SECONDS="${HOST112_INDEXER_START_TIMEOUT_SECONDS:-180}"' in source
|
||||
assert 'MANAGER_START_TIMEOUT_SECONDS="${HOST112_MANAGER_START_TIMEOUT_SECONDS:-120}"' in source
|
||||
assert 'MANAGER_RESET_TIMEOUT_SECONDS="${HOST112_MANAGER_RESET_TIMEOUT_SECONDS:-5}"' in source
|
||||
assert 'MANAGER_VERIFY_TIMEOUT_SECONDS="${HOST112_MANAGER_VERIFY_TIMEOUT_SECONDS:-30}"' in source
|
||||
assert 'MANAGER_ROLLBACK_TIMEOUT_SECONDS="${HOST112_MANAGER_ROLLBACK_TIMEOUT_SECONDS:-45}"' in source
|
||||
assert 'if [ "$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS" -gt 570 ]; then' in source
|
||||
assert "TimeoutStartSec=600" in service
|
||||
assert "Invoke-HostSshText $targetHost $applyCommand 590 1" in control
|
||||
|
||||
|
||||
def test_installer_uses_argument_regex_and_post_install_policy_readback() -> None:
|
||||
source = read(INSTALLER)
|
||||
|
||||
assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in source
|
||||
assert "--run-id [A-Za-z0-9]" in source
|
||||
assert "--work-item-id [A-Za-z0-9]" in source
|
||||
assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in source
|
||||
assert 'visudo -cf "$sudoers_tmp"' in source
|
||||
assert "SUDO_CORRELATED_APPLY_NOPASSWD" in source
|
||||
assert "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" in source
|
||||
assert "SUDO_UNSAFE_IDENTITY_NOPASSWD" in source
|
||||
assert "--trace-id 'unsafe value'" in source
|
||||
assert "--rollback" in source
|
||||
|
||||
|
||||
def test_exact_sudoers_argument_regex_parses_when_visudo_is_available() -> None:
|
||||
visudo = shutil.which("visudo")
|
||||
if not visudo:
|
||||
return
|
||||
candidate = (
|
||||
"kali ALL=(root) NOPASSWD: "
|
||||
"/usr/local/sbin/awoooi-host112-guest-readiness "
|
||||
"^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
|
||||
"--run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} "
|
||||
"--work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$\n"
|
||||
)
|
||||
result = subprocess.run(
|
||||
[visudo, "-cf", "-"],
|
||||
input=candidate,
|
||||
text=True,
|
||||
capture_output=True,
|
||||
check=False,
|
||||
timeout=10,
|
||||
)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
|
||||
def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
|
||||
control = read(CONTROL)
|
||||
bootstrap = read(BOOTSTRAP)
|
||||
function = control[control.index("function Convert-AgentHost112GuestReadback") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
|
||||
assert f"[string]${parameter} = \"\"" in control
|
||||
assert f'[string]`${parameter} = ""' in bootstrap
|
||||
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
|
||||
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
|
||||
assert '$before.managerPreflightStatus -eq "ready"' in function
|
||||
assert 'receiptStatus -eq "written"' in function
|
||||
assert '$after.managerVerified' in function
|
||||
assert 'name = "wazuh_manager_service_active"' in function
|
||||
assert 'name = "wazuh_analysisd_process_present"' in function
|
||||
assert 'name = "wazuh_agent_event_1514_ready"' in function
|
||||
assert 'name = "wazuh_agent_enrollment_1515_tcp"' in function
|
||||
assert 'name = "wazuh_manager_api_55000_tcp"' in function
|
||||
assert "host112_readback_transport_unavailable" in function
|
||||
assert 'terminal = "no_write_control_identity_incomplete_or_unsafe"' in function
|
||||
@@ -107,7 +107,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() ->
|
||||
assert "restrict,command=" in host112_installer
|
||||
assert "visudo -cf" in host112_installer
|
||||
assert "--rollback" in host112_installer
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply" in host112_service
|
||||
assert "ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer" in host112_service
|
||||
assert "TimeoutStartSec=600" in host112_service
|
||||
assert "OnBootSec=20s" in host112_timer
|
||||
assert "OnUnitInactiveSec=60s" in host112_timer
|
||||
|
||||
|
||||
Reference in New Issue
Block a user