106 lines
3.6 KiB
Python
106 lines
3.6 KiB
Python
import json
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
SCRIPT = ROOT / "scripts" / "reboot-recovery" / "external-l0-recovery-supervisor.py"
|
|
EXAMPLE_CONFIG = ROOT / "ops" / "reboot-recovery" / "external-l0-recovery-supervisor.example.json"
|
|
|
|
|
|
def write_samples(path: Path, healthy: bool) -> None:
|
|
ids = ("awoooi-api", "awoooi-web", "gitea", "stock-freshness")
|
|
path.write_text(
|
|
json.dumps(
|
|
{
|
|
"target_results": [
|
|
{
|
|
"id": target_id,
|
|
"url": f"https://{target_id}.example.invalid/",
|
|
"status": 200 if healthy else 0,
|
|
"healthy": healthy,
|
|
"error_class": "" if healthy else "timeout",
|
|
"response_body_read": False,
|
|
}
|
|
for target_id in ids
|
|
]
|
|
}
|
|
),
|
|
encoding="utf-8",
|
|
)
|
|
|
|
|
|
def run_once(tmp_path: Path, samples: Path, observed_at: str) -> dict[str, object]:
|
|
completed = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
"--check",
|
|
"--config",
|
|
str(EXAMPLE_CONFIG),
|
|
"--state-file",
|
|
str(tmp_path / "state.json"),
|
|
"--artifact-dir",
|
|
str(tmp_path / "evidence"),
|
|
"--samples-json",
|
|
str(samples),
|
|
"--now",
|
|
observed_at,
|
|
],
|
|
check=True,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
return json.loads(completed.stdout)
|
|
|
|
|
|
def test_external_l0_state_machine_is_deduplicated_and_recovers(tmp_path: Path) -> None:
|
|
failed = tmp_path / "failed.json"
|
|
healthy = tmp_path / "healthy.json"
|
|
write_samples(failed, healthy=False)
|
|
write_samples(healthy, healthy=True)
|
|
|
|
first = run_once(tmp_path, failed, "2026-07-11T08:00:00Z")
|
|
assert first["state"] == "suspected_outage"
|
|
assert first["transition"] == "suspected_outage"
|
|
|
|
second = run_once(tmp_path, failed, "2026-07-11T08:00:15Z")
|
|
assert second["state"] == "outage_confirmed"
|
|
assert second["transition"] == "outage_confirmed"
|
|
assert str(second["incident_id"]).startswith("INC-L0-")
|
|
assert second["runtime_ready"] is False
|
|
|
|
evidence = json.loads(Path(str(second["artifact"])).read_text(encoding="utf-8"))
|
|
assert sorted(evidence["blockers"]) == [
|
|
"callback_not_executable:maintenance_activate",
|
|
"callback_not_executable:power_restore_99",
|
|
"callback_not_executable:telegram_down",
|
|
]
|
|
|
|
third = run_once(tmp_path, failed, "2026-07-11T08:00:30Z")
|
|
assert third["state"] == "outage_confirmed"
|
|
assert third["transition"] is None
|
|
|
|
fourth = run_once(tmp_path, healthy, "2026-07-11T08:00:45Z")
|
|
assert fourth["state"] == "recovering"
|
|
assert fourth["transition"] == "recovering"
|
|
|
|
fifth = run_once(tmp_path, healthy, "2026-07-11T08:01:00Z")
|
|
assert fifth["state"] == "recovered"
|
|
assert fifth["transition"] == "recovered"
|
|
|
|
|
|
def test_external_l0_contract_is_off_lan_and_never_reads_response_bodies() -> None:
|
|
config = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8"))
|
|
text = SCRIPT.read_text(encoding="utf-8")
|
|
|
|
assert len(config["targets"]) == 4
|
|
assert all(target["url"].startswith("https://") for target in config["targets"])
|
|
assert "192.168.0." not in EXAMPLE_CONFIG.read_text(encoding="utf-8")
|
|
assert "response_body_read" in text
|
|
assert "TELEGRAM_BOT_TOKEN" not in text
|
|
assert "shell=True" not in text
|
|
assert "callback_allowlist_root" in text
|
|
assert '"power_restore_99"' in EXAMPLE_CONFIG.read_text(encoding="utf-8")
|