#!/usr/bin/env python3 """Off-LAN outage state machine for the Windows 99 recovery dependency chain.""" from __future__ import annotations import argparse import json import os import subprocess import sys import urllib.error import urllib.request from datetime import datetime, timezone from pathlib import Path from typing import Any SCHEMA_VERSION = "external_l0_recovery_supervisor_v1" STATE_SCHEMA_VERSION = "external_l0_recovery_state_v1" VALID_LIFECYCLE_STATES = { "healthy", "suspected_outage", "outage_confirmed", "recovering", "recovered", } def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser() parser.add_argument("--config", required=True, type=Path) parser.add_argument("--state-file", required=True, type=Path) parser.add_argument("--artifact-dir", required=True, type=Path) parser.add_argument("--samples-json", type=Path) parser.add_argument("--now") mode = parser.add_mutually_exclusive_group(required=True) mode.add_argument("--check", action="store_true") mode.add_argument("--apply", action="store_true") return parser.parse_args() def now_utc(value: str | None) -> datetime: if not value: return datetime.now(timezone.utc) parsed = datetime.fromisoformat(value.replace("Z", "+00:00")) if parsed.tzinfo is None: parsed = parsed.replace(tzinfo=timezone.utc) return parsed.astimezone(timezone.utc) def isoformat(value: datetime) -> str: return value.replace(microsecond=0).isoformat().replace("+00:00", "Z") def read_json(path: Path) -> dict[str, Any]: payload = json.loads(path.read_text(encoding="utf-8")) if not isinstance(payload, dict): raise ValueError(f"JSON root must be an object: {path}") return payload def write_json(path: Path, payload: dict[str, Any]) -> None: path.parent.mkdir(parents=True, exist_ok=True) temporary = path.with_suffix(path.suffix + ".tmp") temporary.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8") os.replace(temporary, path) def validate_config(config: dict[str, Any]) -> None: if config.get("schema_version") != "external_l0_recovery_supervisor_config_v1": raise ValueError("unsupported config schema_version") targets = config.get("targets") if not isinstance(targets, list) or len(targets) < 2: raise ValueError("at least two external targets are required") target_ids: set[str] = set() for target in targets: if not isinstance(target, dict): raise ValueError("each target must be an object") target_id = str(target.get("id") or "") url = str(target.get("url") or "") if not target_id or target_id in target_ids: raise ValueError("target ids must be present and unique") if not url.startswith("https://"): raise ValueError("external targets must use https") target_ids.add(target_id) thresholds = config.get("thresholds") or {} for key in ( "outage_consecutive_samples", "recovery_consecutive_samples", "minimum_failed_targets", "minimum_healthy_targets", ): value = thresholds.get(key) if not isinstance(value, int) or value < 1: raise ValueError(f"threshold must be a positive integer: {key}") if thresholds["minimum_failed_targets"] > len(targets): raise ValueError("minimum_failed_targets exceeds target count") if thresholds["minimum_healthy_targets"] > len(targets): raise ValueError("minimum_healthy_targets exceeds target count") allow_root = Path(str(config.get("callback_allowlist_root") or "")) if not allow_root.is_absolute(): raise ValueError("callback_allowlist_root must be absolute") callbacks = config.get("callbacks") or {} if not isinstance(callbacks, dict): raise ValueError("callbacks must be an object") for lifecycle, actions in callbacks.items(): if lifecycle not in VALID_LIFECYCLE_STATES: raise ValueError(f"unsupported callback lifecycle: {lifecycle}") if not isinstance(actions, list): raise ValueError("callback lifecycle entries must be arrays") for action in actions: if not isinstance(action, dict) or not action.get("id") or not action.get("path"): raise ValueError("callbacks require id and path") action_path = Path(str(action["path"])) if not action_path.is_absolute() or not action_path.is_relative_to(allow_root): raise ValueError(f"callback path outside allowlist root: {action_path}") def default_state(observed_at: str) -> dict[str, Any]: return { "schema_version": STATE_SCHEMA_VERSION, "lifecycle_state": "healthy", "incident_id": None, "failure_streak": 0, "success_streak": 0, "first_failure_at": None, "last_transition_at": observed_at, "last_observed_at": observed_at, } def load_state(path: Path, observed_at: str) -> dict[str, Any]: if not path.exists(): return default_state(observed_at) state = read_json(path) if state.get("schema_version") != STATE_SCHEMA_VERSION: raise ValueError("unsupported state schema_version") if state.get("lifecycle_state") not in VALID_LIFECYCLE_STATES: raise ValueError("invalid lifecycle state") return state def probe_target(target: dict[str, Any]) -> dict[str, Any]: expected = {int(value) for value in target.get("expected_statuses", [200])} timeout_seconds = float(target.get("timeout_seconds", 5)) request = urllib.request.Request( str(target["url"]), headers={"User-Agent": "AWOOOI-External-L0-Recovery-Supervisor/1.0"}, ) status = 0 error_class = "" try: with urllib.request.urlopen(request, timeout=timeout_seconds) as response: status = int(response.status) except urllib.error.HTTPError as exc: status = int(exc.code) error_class = "http_error" except urllib.error.URLError: error_class = "url_error" except TimeoutError: error_class = "timeout" return { "id": str(target["id"]), "url": str(target["url"]), "status": status, "healthy": status in expected, "error_class": error_class, "response_body_read": False, } def collect_results(config: dict[str, Any], samples_json: Path | None) -> list[dict[str, Any]]: if samples_json: payload = read_json(samples_json) results = payload.get("target_results") if not isinstance(results, list): raise ValueError("samples-json requires target_results array") configured_ids = {str(target["id"]) for target in config["targets"]} result_ids = {str(result.get("id") or "") for result in results if isinstance(result, dict)} if configured_ids != result_ids: raise ValueError("samples-json target ids do not match config") return [dict(result) for result in results] return [probe_target(target) for target in config["targets"]] def advance_state( state: dict[str, Any], results: list[dict[str, Any]], thresholds: dict[str, int], observed_at: str, ) -> tuple[dict[str, Any], str | None, dict[str, Any]]: previous = str(state["lifecycle_state"]) failed = [str(result["id"]) for result in results if not bool(result.get("healthy"))] healthy = [str(result["id"]) for result in results if bool(result.get("healthy"))] outage_sample = len(failed) >= thresholds["minimum_failed_targets"] recovery_sample = len(healthy) >= thresholds["minimum_healthy_targets"] transition: str | None = None if previous in {"healthy", "recovered", "suspected_outage"}: if outage_sample: state["failure_streak"] = int(state.get("failure_streak", 0)) + 1 state["success_streak"] = 0 state["first_failure_at"] = state.get("first_failure_at") or observed_at desired = "outage_confirmed" if state["failure_streak"] >= thresholds["outage_consecutive_samples"] else "suspected_outage" else: state["failure_streak"] = 0 state["success_streak"] = 0 state["first_failure_at"] = None desired = "healthy" if previous != "recovered" else "recovered" else: if recovery_sample: state["success_streak"] = int(state.get("success_streak", 0)) + 1 state["failure_streak"] = 0 desired = "recovered" if state["success_streak"] >= thresholds["recovery_consecutive_samples"] else "recovering" else: state["success_streak"] = 0 desired = "outage_confirmed" if desired != previous: transition = desired state["lifecycle_state"] = desired state["last_transition_at"] = observed_at if desired == "outage_confirmed" and not state.get("incident_id"): compact = observed_at.replace("-", "").replace(":", "").replace("T", "-").replace("Z", "") state["incident_id"] = f"INC-L0-{compact}" if desired == "healthy": state["incident_id"] = None state["last_observed_at"] = observed_at sample = { "outage_candidate": outage_sample, "recovery_candidate": recovery_sample, "failed_target_ids": failed, "healthy_target_ids": healthy, } return state, transition, sample def validate_required_callbacks(config: dict[str, Any], lifecycle: str) -> list[str]: configured = {str(item["id"]) for item in (config.get("callbacks") or {}).get(lifecycle, [])} required = {str(item) for item in (config.get("required_callback_ids") or {}).get(lifecycle, [])} return sorted(required - configured) def run_callbacks( config: dict[str, Any], lifecycle: str, event_file: Path, apply: bool, ) -> tuple[list[dict[str, Any]], list[str]]: receipts: list[dict[str, Any]] = [] blockers: list[str] = [] missing = validate_required_callbacks(config, lifecycle) blockers.extend(f"required_callback_missing:{callback_id}" for callback_id in missing) for action in (config.get("callbacks") or {}).get(lifecycle, []): action_id = str(action["id"]) action_path = Path(str(action["path"])) if not action_path.is_file() or not os.access(action_path, os.X_OK): blockers.append(f"callback_not_executable:{action_id}") receipts.append({"id": action_id, "status": "blocked_not_executable", "path": str(action_path)}) continue if not apply: receipts.append({"id": action_id, "status": "check_only", "path": str(action_path)}) continue try: completed = subprocess.run( [str(action_path), "--event-file", str(event_file)], stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, timeout=int(action.get("timeout_seconds", 30)), check=False, ) status = "executed" if completed.returncode == 0 else "failed" receipts.append({"id": action_id, "status": status, "exit_code": completed.returncode}) if completed.returncode != 0: blockers.append(f"callback_failed:{action_id}") except subprocess.TimeoutExpired: receipts.append({"id": action_id, "status": "timeout"}) blockers.append(f"callback_timeout:{action_id}") return receipts, blockers def main() -> int: args = parse_args() observed = now_utc(args.now) observed_at = isoformat(observed) config = read_json(args.config) validate_config(config) state = load_state(args.state_file, observed_at) previous_state = str(state["lifecycle_state"]) results = collect_results(config, args.samples_json) thresholds = {key: int(value) for key, value in config["thresholds"].items()} state, transition, sample = advance_state(state, results, thresholds, observed_at) run_id = f"external-l0-{observed.strftime('%Y%m%dT%H%M%SZ')}" artifact_file = args.artifact_dir / f"{run_id}.json" event: dict[str, Any] = { "schema_version": SCHEMA_VERSION, "run_id": run_id, "observed_at": observed_at, "mode": "apply" if args.apply else "check", "previous_state": previous_state, "lifecycle_state": state["lifecycle_state"], "transition": transition, "incident_id": state.get("incident_id"), "sample": sample, "target_results": results, "callback_receipts": [], "blockers": [], "secret_value_read": False, "arbitrary_shell_allowed": False, "remote_write_performed": False, } write_json(artifact_file, event) callback_receipts: list[dict[str, Any]] = [] blockers: list[str] = [] if transition: callback_receipts, blockers = run_callbacks(config, transition, artifact_file, args.apply) event["callback_receipts"] = callback_receipts event["blockers"] = blockers event["runtime_ready"] = not blockers event["remote_write_performed"] = args.apply and any( receipt.get("status") == "executed" for receipt in callback_receipts ) write_json(artifact_file, event) write_json(args.state_file, state) print(json.dumps({ "run_id": run_id, "state": state["lifecycle_state"], "transition": transition, "incident_id": state.get("incident_id"), "runtime_ready": event["runtime_ready"], "artifact": str(artifact_file), }, ensure_ascii=True)) return 2 if args.apply and blockers else 0 if __name__ == "__main__": try: raise SystemExit(main()) except (OSError, ValueError, json.JSONDecodeError) as exc: print(f"external_l0_supervisor_error={exc}", file=sys.stderr) raise SystemExit(2)