import json import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[3] SCRIPT = ROOT / "scripts" / "reboot-recovery" / "external-l0-recovery-supervisor.py" EXAMPLE_CONFIG = ROOT / "ops" / "reboot-recovery" / "external-l0-recovery-supervisor.example.json" def write_samples(path: Path, healthy: bool) -> None: ids = ("awoooi-api", "awoooi-web", "gitea", "stock-freshness") path.write_text( json.dumps( { "target_results": [ { "id": target_id, "url": f"https://{target_id}.example.invalid/", "status": 200 if healthy else 0, "healthy": healthy, "error_class": "" if healthy else "timeout", "response_body_read": False, } for target_id in ids ] } ), encoding="utf-8", ) def run_once(tmp_path: Path, samples: Path, observed_at: str) -> dict[str, object]: completed = subprocess.run( [ sys.executable, str(SCRIPT), "--check", "--config", str(EXAMPLE_CONFIG), "--state-file", str(tmp_path / "state.json"), "--artifact-dir", str(tmp_path / "evidence"), "--samples-json", str(samples), "--now", observed_at, ], check=True, capture_output=True, text=True, ) return json.loads(completed.stdout) def test_external_l0_state_machine_is_deduplicated_and_recovers(tmp_path: Path) -> None: failed = tmp_path / "failed.json" healthy = tmp_path / "healthy.json" write_samples(failed, healthy=False) write_samples(healthy, healthy=True) first = run_once(tmp_path, failed, "2026-07-11T08:00:00Z") assert first["state"] == "suspected_outage" assert first["transition"] == "suspected_outage" second = run_once(tmp_path, failed, "2026-07-11T08:00:15Z") assert second["state"] == "outage_confirmed" assert second["transition"] == "outage_confirmed" assert str(second["incident_id"]).startswith("INC-L0-") assert second["runtime_ready"] is False evidence = json.loads(Path(str(second["artifact"])).read_text(encoding="utf-8")) assert sorted(evidence["blockers"]) == [ "callback_not_executable:maintenance_activate", "callback_not_executable:power_restore_99", "callback_not_executable:telegram_down", ] third = run_once(tmp_path, failed, "2026-07-11T08:00:30Z") assert third["state"] == "outage_confirmed" assert third["transition"] is None fourth = run_once(tmp_path, healthy, "2026-07-11T08:00:45Z") assert fourth["state"] == "recovering" assert fourth["transition"] == "recovering" fifth = run_once(tmp_path, healthy, "2026-07-11T08:01:00Z") assert fifth["state"] == "recovered" assert fifth["transition"] == "recovered" def test_external_l0_contract_is_off_lan_and_never_reads_response_bodies() -> None: config = json.loads(EXAMPLE_CONFIG.read_text(encoding="utf-8")) text = SCRIPT.read_text(encoding="utf-8") assert len(config["targets"]) == 4 assert all(target["url"].startswith("https://") for target in config["targets"]) assert "192.168.0." not in EXAMPLE_CONFIG.read_text(encoding="utf-8") assert "response_body_read" in text assert "TELEGRAM_BOT_TOKEN" not in text assert "shell=True" not in text assert "callback_allowlist_root" in text assert '"power_restore_99"' in EXAMPLE_CONFIG.read_text(encoding="utf-8")