85d8eeb0db
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 2m5s
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
173 lines
6.4 KiB
Python
173 lines
6.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
SCRIPT = ROOT / "scripts" / "reboot-recovery" / "reboot-event-detector.py"
|
|
|
|
|
|
HOST_PROBE = """\
|
|
AWOOOI_REBOOT_AUTO_RECOVERY_HOST_PROBE=1
|
|
TARGET_HOSTS=99,110,111,112,120,121,188
|
|
HOST_BOOT alias=99 target=192.168.0.99 startup_unit=vmware-host-autostart reachable=1 boot_id=win-boot-2 uptime_seconds=80 systemd_state=windows_exporter startup_enabled=enabled startup_active=active
|
|
HOST_BOOT alias=110 target=wooo@192.168.0.110 startup_unit=awoooi-startup-110.service reachable=1 boot_id=linux-boot-2 uptime_seconds=100 systemd_state=running startup_enabled=enabled startup_active=active
|
|
HOST_BOOT alias=111 target=192.168.0.111 startup_unit=vm-host-boot reachable=1 boot_id=vm111-boot-2 uptime_seconds=110 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown
|
|
HOST_BOOT alias=112 target=192.168.0.112 startup_unit=vm-host-boot reachable=1 boot_id=vm112-boot-2 uptime_seconds=115 systemd_state=node_exporter startup_enabled=unknown startup_active=unknown
|
|
HOST_BOOT alias=120 target=wooo@192.168.0.120 startup_unit=k3s.service reachable=1 boot_id=mon-boot-2 uptime_seconds=120 systemd_state=running startup_enabled=enabled startup_active=active
|
|
HOST_BOOT alias=121 target=wooo@192.168.0.121 startup_unit=k3s.service reachable=1 boot_id=worker-boot-2 uptime_seconds=130 systemd_state=running startup_enabled=enabled startup_active=active
|
|
HOST_BOOT alias=188 target=ollama@192.168.0.188 startup_unit=awoooi-startup.service reachable=1 boot_id=ai-boot-2 uptime_seconds=140 systemd_state=running startup_enabled=enabled startup_active=active
|
|
"""
|
|
|
|
|
|
def test_reboot_detector_detects_all_required_hosts_and_writes_metrics(tmp_path: Path) -> None:
|
|
probe_path = tmp_path / "host-probe.txt"
|
|
state_path = tmp_path / "state.json"
|
|
output_path = tmp_path / "event.json"
|
|
prom_path = tmp_path / "event.prom"
|
|
probe_path.write_text(HOST_PROBE, encoding="utf-8")
|
|
state_path.write_text(
|
|
json.dumps(
|
|
{
|
|
"hosts": {
|
|
"99": {"boot_id": "win-boot-1"},
|
|
"110": {"boot_id": "linux-boot-1"},
|
|
"111": {"boot_id": "vm111-boot-1"},
|
|
"112": {"boot_id": "vm112-boot-1"},
|
|
"120": {"boot_id": "mon-boot-1"},
|
|
"121": {"boot_id": "worker-boot-1"},
|
|
"188": {"boot_id": "ai-boot-1"},
|
|
}
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--host-probe-file",
|
|
str(probe_path),
|
|
"--state-file",
|
|
str(state_path),
|
|
"--target-minutes",
|
|
"10",
|
|
"--generated-at",
|
|
"2026-06-30T18:00:00+08:00",
|
|
"--output",
|
|
str(output_path),
|
|
"--prometheus-output",
|
|
str(prom_path),
|
|
],
|
|
check=True,
|
|
)
|
|
|
|
payload = json.loads(output_path.read_text(encoding="utf-8"))
|
|
metrics = prom_path.read_text(encoding="utf-8")
|
|
|
|
assert payload["required_hosts"] == ["99", "110", "111", "112", "120", "121", "188"]
|
|
assert payload["reboot_detected"] is True
|
|
assert payload["all_required_hosts_in_reboot_window"] is True
|
|
assert payload["target_seconds_remaining"] == 460
|
|
assert 'awoooi_reboot_event_host_rebooted{host="99"} 1' in metrics
|
|
assert 'awoooi_reboot_event_required_host_observed{host="188"} 1' in metrics
|
|
|
|
|
|
def test_reboot_detector_fails_visible_when_windows_or_vm_host_missing(tmp_path: Path) -> None:
|
|
probe_path = tmp_path / "host-probe.txt"
|
|
state_path = tmp_path / "state.json"
|
|
output_path = tmp_path / "event.json"
|
|
probe_path.write_text(
|
|
"\n".join(line for line in HOST_PROBE.splitlines() if "alias=99 " not in line) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--host-probe-file",
|
|
str(probe_path),
|
|
"--state-file",
|
|
str(state_path),
|
|
"--target-minutes",
|
|
"10",
|
|
"--generated-at",
|
|
"2026-06-30T18:00:00+08:00",
|
|
"--output",
|
|
str(output_path),
|
|
"--no-write-state",
|
|
],
|
|
check=True,
|
|
)
|
|
|
|
payload = json.loads(output_path.read_text(encoding="utf-8"))
|
|
|
|
assert "99" in payload["missing_hosts"]
|
|
assert payload["all_required_hosts_observed"] is False
|
|
assert payload["all_required_hosts_in_reboot_window"] is False
|
|
|
|
|
|
def test_reboot_detector_does_not_treat_unknown_uptime_as_fresh_boot(
|
|
tmp_path: Path,
|
|
) -> None:
|
|
probe_path = tmp_path / "host-probe.txt"
|
|
state_path = tmp_path / "state.json"
|
|
output_path = tmp_path / "event.json"
|
|
probe_path.write_text(
|
|
"\n".join(
|
|
[
|
|
"AWOOOI_REBOOT_AUTO_RECOVERY_HOST_PROBE=1",
|
|
"TARGET_HOSTS=99",
|
|
(
|
|
"HOST_BOOT alias=99 target=192.168.0.99 "
|
|
"startup_unit=vmware-host-autostart reachable=1 "
|
|
"boot_id=reachable_unknown_boot uptime_seconds=unknown "
|
|
"systemd_state=ping_reachable startup_enabled=unknown "
|
|
"startup_active=unknown"
|
|
),
|
|
]
|
|
)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
state_path.write_text(
|
|
json.dumps({"hosts": {"99": {"boot_id": "win-boot-1"}}}),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--host-probe-file",
|
|
str(probe_path),
|
|
"--state-file",
|
|
str(state_path),
|
|
"--target-minutes",
|
|
"10",
|
|
"--generated-at",
|
|
"2026-06-30T18:00:00+08:00",
|
|
"--output",
|
|
str(output_path),
|
|
"--required-host",
|
|
"99",
|
|
"--no-write-state",
|
|
],
|
|
check=True,
|
|
)
|
|
|
|
payload = json.loads(output_path.read_text(encoding="utf-8"))
|
|
|
|
assert payload["observed_hosts"] == ["99"]
|
|
assert payload["reboot_detected"] is False
|
|
assert payload["fresh_boot_hosts"] == []
|
|
assert payload["rebooted_hosts"] == []
|
|
assert payload["all_required_hosts_observed"] is True
|
|
assert payload["all_required_hosts_in_reboot_window"] is False
|
|
assert payload["state_written"] is False
|