158 lines
7.4 KiB
Python
158 lines
7.4 KiB
Python
from __future__ import annotations
|
|
|
|
import subprocess
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
INSTALLER = ROOT / "scripts" / "reboot-recovery" / "install-reboot-auto-recovery-slo-110.sh"
|
|
SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.service"
|
|
TIMER = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.timer"
|
|
EXPORTER = ROOT / "scripts" / "reboot-recovery" / "reboot-auto-recovery-slo-exporter.sh"
|
|
|
|
|
|
def test_installer_dry_run_exposes_apply_rollback_and_verify_contract() -> None:
|
|
result = subprocess.run(
|
|
["bash", str(INSTALLER), "--dry-run"],
|
|
text=True,
|
|
capture_output=True,
|
|
check=True,
|
|
)
|
|
|
|
assert "DRY_RUN=1" in result.stdout
|
|
assert "target_selector=host_110_systemd_timer:awoooi-reboot-auto-recovery-slo.timer" in result.stdout
|
|
assert "rollback_command=bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --rollback" in result.stdout
|
|
assert "verify_command=bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh --verify-only" in result.stdout
|
|
assert "would_include_source_gates=" in result.stdout
|
|
assert "awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in result.stdout
|
|
assert "ops/nginx" in result.stdout
|
|
assert "ops/reboot-recovery" in result.stdout
|
|
assert "wazuh-managed-host-coverage-gate.snapshot.json" in result.stdout
|
|
assert "wazuh-agent-visibility-runtime-gate.snapshot.json" in result.stdout
|
|
assert "would_enable_timer=awoooi-reboot-auto-recovery-slo.timer" in result.stdout
|
|
|
|
|
|
def test_installer_is_limited_to_verifier_timer_not_product_restarts() -> None:
|
|
text = INSTALLER.read_text(encoding="utf-8")
|
|
executable_text = "\n".join(
|
|
line for line in text.splitlines() if not line.lstrip().startswith("#")
|
|
)
|
|
forbidden_fragments = [
|
|
"shutdown",
|
|
"systemctl restart docker",
|
|
"systemctl restart nginx",
|
|
"systemctl restart k3s",
|
|
"systemctl restart postgresql",
|
|
"systemctl restart redis",
|
|
"docker restart",
|
|
"kubectl drain",
|
|
"iptables ",
|
|
"ufw ",
|
|
"DROP ",
|
|
"TRUNCATE ",
|
|
"gh ",
|
|
"github.com",
|
|
]
|
|
|
|
for fragment in forbidden_fragments:
|
|
assert fragment not in executable_text
|
|
assert re.search(r"(?m)^\s*(sudo\s+)?reboot(\s|$)", executable_text) is None
|
|
assert "systemctl enable --now" in text
|
|
assert "systemctl start ${UNIT_NAME}.service" in text
|
|
assert 'payload_dir="$(mktemp -d "${TMPDIR:-/tmp}/${UNIT_NAME}.XXXXXX")"' in text
|
|
assert 'payload="${payload_dir}/payload.tar.gz"' in text
|
|
assert "apps/api/migrations/awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in text
|
|
assert "ops/nginx" in text
|
|
assert "ops/reboot-recovery" in text
|
|
assert "ops/maintenance" in text
|
|
assert "ops/monitoring" in text
|
|
assert "docs/security/wazuh-managed-host-coverage-gate.snapshot.json" in text
|
|
assert "docs/security/wazuh-agent-visibility-runtime-gate.snapshot.json" in text
|
|
assert "PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md" in text
|
|
assert "ServerAliveInterval" in text
|
|
assert "ServerAliveCountMax" in text
|
|
assert "NumberOfPasswordPrompts=0" in text
|
|
|
|
|
|
def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None:
|
|
text = SERVICE.read_text(encoding="utf-8")
|
|
|
|
assert "Type=oneshot" in text
|
|
assert "User=wooo" in text
|
|
assert "Group=wooo" in text
|
|
assert "WorkingDirectory=/home/wooo" in text
|
|
assert "Environment=HOME=/home/wooo" in text
|
|
assert "Environment=ROOT_DIR=/home/wooo/awoooi-reboot-recovery-slo" in text
|
|
assert "Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles" in text
|
|
assert "Environment=LOG_DIR=/home/wooo/reboot-recovery" in text
|
|
assert "ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh" in text
|
|
assert "TimeoutStartSec=600" in text
|
|
|
|
|
|
def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None:
|
|
text = TIMER.read_text(encoding="utf-8")
|
|
|
|
assert "OnBootSec=2min" in text
|
|
assert "OnUnitInactiveSec=5min" in text
|
|
assert "OnUnitActiveSec" not in text
|
|
|
|
|
|
def test_exporter_uses_user_writable_lock_after_creating_log_dir() -> None:
|
|
text = EXPORTER.read_text(encoding="utf-8")
|
|
|
|
assert 'LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"' in text
|
|
assert text.index('mkdir -p "$TEXTFILE_DIR" "$LOG_DIR"') < text.index('exec 9>"$LOCK_FILE"')
|
|
|
|
|
|
def test_exporter_carries_stockplatform_readbacks_into_scorecard() -> None:
|
|
text = EXPORTER.read_text(encoding="utf-8")
|
|
|
|
assert "STOCK_FRESHNESS_URL" in text
|
|
assert "STOCK_INGESTION_URL" in text
|
|
assert 'stock_freshness_file="$artifact_dir/stock-freshness.json"' in text
|
|
assert 'stock_ingestion_file="$artifact_dir/stock-ingestion.json"' in text
|
|
assert "scorecard_args+=(--stock-freshness-file" in text
|
|
assert "scorecard_args+=(--stock-ingestion-file" in text
|
|
assert "manual_db_update" not in text
|
|
assert "systemctl restart" not in text
|
|
|
|
|
|
def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None:
|
|
text = EXPORTER.read_text(encoding="utf-8")
|
|
|
|
assert "windows99-vmware-verify.txt" in text
|
|
assert "collect-windows99-vmware-verify.sh\" --collect" in text
|
|
assert "WINDOWS99_SSH_USERS=\"${WINDOWS99_SSH_USERS:-Administrator}\"" in text
|
|
assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-1}\"" in text
|
|
assert "WINDOWS99_REMOTE_VERIFY_TIMEOUT=\"${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}\"" in text
|
|
assert "WINDOWS99_REQUIRED_VM_ALIASES=\"${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}\"" in text
|
|
assert "windows99_vmware_verify_alias_set_matches_current" in text
|
|
assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" in text
|
|
assert "scorecard_args+=(--windows99-vmware-file" in text
|
|
assert "WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS" in text
|
|
assert "windows99-vmware-verify.current-failed.txt" in text
|
|
assert "windows99_vmware_verify_fallback_applied=1" in text
|
|
assert "windows99_vmware_verify_fallback_max_age_seconds" in text
|
|
assert "find \"$LOG_DIR\" -mindepth 2 -maxdepth 2" in text
|
|
assert "POST_REBOOT_READINESS_TIMEOUT_SECONDS" in text
|
|
assert 'POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:-120}"' in text
|
|
assert "POST_REBOOT_SKIP_RUNNER_GUARD=1" in text
|
|
assert "POST_REBOOT_READINESS_SUMMARY_TIMEOUT=1" in text
|
|
assert "POST_REBOOT_READINESS_PARTIAL_FROM_POST_START=1" in text
|
|
assert "PARTIAL_POST_START_GREEN_SUMMARY_TIMEOUT" in text
|
|
assert "write_partial_post_reboot_summary_from_post_start" in text
|
|
assert "POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS" in text
|
|
assert "post-reboot-readiness-summary.current-failed.txt" in text
|
|
assert "POST_REBOOT_READINESS_SUMMARY_FALLBACK_APPLIED=1" in text
|
|
assert "POST_REBOOT_READINESS_SUMMARY_FALLBACK_MAX_AGE_SECONDS" in text
|
|
assert "awoooi_windows99_vmware_missing_vmx_alias" in text
|
|
assert "awoooi_windows99_vmware_powered_off_alias" in text
|
|
assert "awoooi_windows99_vmware_autostart_config_ready" in text
|
|
assert "active_blocker_metrics" in text
|
|
assert 'for blocker in payload.get("active_blockers") or []' in text
|
|
assert "awoooi_reboot_auto_recovery_slo_active_blocker" in text
|
|
assert "awoooi_reboot_auto_recovery_slo_primary_blocker" in text
|
|
assert 'blocker="{label(blocker)}"' in text
|
|
assert 'target_minutes="{label(target_minutes)}"' in text
|