345 lines
14 KiB
Python
345 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
"""Off-LAN outage state machine for the Windows 99 recovery dependency chain."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
SCHEMA_VERSION = "external_l0_recovery_supervisor_v1"
|
|
STATE_SCHEMA_VERSION = "external_l0_recovery_state_v1"
|
|
VALID_LIFECYCLE_STATES = {
|
|
"healthy",
|
|
"suspected_outage",
|
|
"outage_confirmed",
|
|
"recovering",
|
|
"recovered",
|
|
}
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--config", required=True, type=Path)
|
|
parser.add_argument("--state-file", required=True, type=Path)
|
|
parser.add_argument("--artifact-dir", required=True, type=Path)
|
|
parser.add_argument("--samples-json", type=Path)
|
|
parser.add_argument("--now")
|
|
mode = parser.add_mutually_exclusive_group(required=True)
|
|
mode.add_argument("--check", action="store_true")
|
|
mode.add_argument("--apply", action="store_true")
|
|
return parser.parse_args()
|
|
|
|
|
|
def now_utc(value: str | None) -> datetime:
|
|
if not value:
|
|
return datetime.now(timezone.utc)
|
|
parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
if parsed.tzinfo is None:
|
|
parsed = parsed.replace(tzinfo=timezone.utc)
|
|
return parsed.astimezone(timezone.utc)
|
|
|
|
|
|
def isoformat(value: datetime) -> str:
|
|
return value.replace(microsecond=0).isoformat().replace("+00:00", "Z")
|
|
|
|
|
|
def read_json(path: Path) -> dict[str, Any]:
|
|
payload = json.loads(path.read_text(encoding="utf-8"))
|
|
if not isinstance(payload, dict):
|
|
raise ValueError(f"JSON root must be an object: {path}")
|
|
return payload
|
|
|
|
|
|
def write_json(path: Path, payload: dict[str, Any]) -> None:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
temporary = path.with_suffix(path.suffix + ".tmp")
|
|
temporary.write_text(json.dumps(payload, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
|
|
os.replace(temporary, path)
|
|
|
|
|
|
def validate_config(config: dict[str, Any]) -> None:
|
|
if config.get("schema_version") != "external_l0_recovery_supervisor_config_v1":
|
|
raise ValueError("unsupported config schema_version")
|
|
targets = config.get("targets")
|
|
if not isinstance(targets, list) or len(targets) < 2:
|
|
raise ValueError("at least two external targets are required")
|
|
target_ids: set[str] = set()
|
|
for target in targets:
|
|
if not isinstance(target, dict):
|
|
raise ValueError("each target must be an object")
|
|
target_id = str(target.get("id") or "")
|
|
url = str(target.get("url") or "")
|
|
if not target_id or target_id in target_ids:
|
|
raise ValueError("target ids must be present and unique")
|
|
if not url.startswith("https://"):
|
|
raise ValueError("external targets must use https")
|
|
target_ids.add(target_id)
|
|
|
|
thresholds = config.get("thresholds") or {}
|
|
for key in (
|
|
"outage_consecutive_samples",
|
|
"recovery_consecutive_samples",
|
|
"minimum_failed_targets",
|
|
"minimum_healthy_targets",
|
|
):
|
|
value = thresholds.get(key)
|
|
if not isinstance(value, int) or value < 1:
|
|
raise ValueError(f"threshold must be a positive integer: {key}")
|
|
if thresholds["minimum_failed_targets"] > len(targets):
|
|
raise ValueError("minimum_failed_targets exceeds target count")
|
|
if thresholds["minimum_healthy_targets"] > len(targets):
|
|
raise ValueError("minimum_healthy_targets exceeds target count")
|
|
|
|
allow_root = Path(str(config.get("callback_allowlist_root") or ""))
|
|
if not allow_root.is_absolute():
|
|
raise ValueError("callback_allowlist_root must be absolute")
|
|
callbacks = config.get("callbacks") or {}
|
|
if not isinstance(callbacks, dict):
|
|
raise ValueError("callbacks must be an object")
|
|
for lifecycle, actions in callbacks.items():
|
|
if lifecycle not in VALID_LIFECYCLE_STATES:
|
|
raise ValueError(f"unsupported callback lifecycle: {lifecycle}")
|
|
if not isinstance(actions, list):
|
|
raise ValueError("callback lifecycle entries must be arrays")
|
|
for action in actions:
|
|
if not isinstance(action, dict) or not action.get("id") or not action.get("path"):
|
|
raise ValueError("callbacks require id and path")
|
|
action_path = Path(str(action["path"]))
|
|
if not action_path.is_absolute() or not action_path.is_relative_to(allow_root):
|
|
raise ValueError(f"callback path outside allowlist root: {action_path}")
|
|
|
|
|
|
def default_state(observed_at: str) -> dict[str, Any]:
|
|
return {
|
|
"schema_version": STATE_SCHEMA_VERSION,
|
|
"lifecycle_state": "healthy",
|
|
"incident_id": None,
|
|
"failure_streak": 0,
|
|
"success_streak": 0,
|
|
"first_failure_at": None,
|
|
"last_transition_at": observed_at,
|
|
"last_observed_at": observed_at,
|
|
}
|
|
|
|
|
|
def load_state(path: Path, observed_at: str) -> dict[str, Any]:
|
|
if not path.exists():
|
|
return default_state(observed_at)
|
|
state = read_json(path)
|
|
if state.get("schema_version") != STATE_SCHEMA_VERSION:
|
|
raise ValueError("unsupported state schema_version")
|
|
if state.get("lifecycle_state") not in VALID_LIFECYCLE_STATES:
|
|
raise ValueError("invalid lifecycle state")
|
|
return state
|
|
|
|
|
|
def probe_target(target: dict[str, Any]) -> dict[str, Any]:
|
|
expected = {int(value) for value in target.get("expected_statuses", [200])}
|
|
timeout_seconds = float(target.get("timeout_seconds", 5))
|
|
request = urllib.request.Request(
|
|
str(target["url"]),
|
|
headers={"User-Agent": "AWOOOI-External-L0-Recovery-Supervisor/1.0"},
|
|
)
|
|
status = 0
|
|
error_class = ""
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
|
status = int(response.status)
|
|
except urllib.error.HTTPError as exc:
|
|
status = int(exc.code)
|
|
error_class = "http_error"
|
|
except urllib.error.URLError:
|
|
error_class = "url_error"
|
|
except TimeoutError:
|
|
error_class = "timeout"
|
|
return {
|
|
"id": str(target["id"]),
|
|
"url": str(target["url"]),
|
|
"status": status,
|
|
"healthy": status in expected,
|
|
"error_class": error_class,
|
|
"response_body_read": False,
|
|
}
|
|
|
|
|
|
def collect_results(config: dict[str, Any], samples_json: Path | None) -> list[dict[str, Any]]:
|
|
if samples_json:
|
|
payload = read_json(samples_json)
|
|
results = payload.get("target_results")
|
|
if not isinstance(results, list):
|
|
raise ValueError("samples-json requires target_results array")
|
|
configured_ids = {str(target["id"]) for target in config["targets"]}
|
|
result_ids = {str(result.get("id") or "") for result in results if isinstance(result, dict)}
|
|
if configured_ids != result_ids:
|
|
raise ValueError("samples-json target ids do not match config")
|
|
return [dict(result) for result in results]
|
|
return [probe_target(target) for target in config["targets"]]
|
|
|
|
|
|
def advance_state(
|
|
state: dict[str, Any],
|
|
results: list[dict[str, Any]],
|
|
thresholds: dict[str, int],
|
|
observed_at: str,
|
|
) -> tuple[dict[str, Any], str | None, dict[str, Any]]:
|
|
previous = str(state["lifecycle_state"])
|
|
failed = [str(result["id"]) for result in results if not bool(result.get("healthy"))]
|
|
healthy = [str(result["id"]) for result in results if bool(result.get("healthy"))]
|
|
outage_sample = len(failed) >= thresholds["minimum_failed_targets"]
|
|
recovery_sample = len(healthy) >= thresholds["minimum_healthy_targets"]
|
|
transition: str | None = None
|
|
|
|
if previous in {"healthy", "recovered", "suspected_outage"}:
|
|
if outage_sample:
|
|
state["failure_streak"] = int(state.get("failure_streak", 0)) + 1
|
|
state["success_streak"] = 0
|
|
state["first_failure_at"] = state.get("first_failure_at") or observed_at
|
|
desired = "outage_confirmed" if state["failure_streak"] >= thresholds["outage_consecutive_samples"] else "suspected_outage"
|
|
else:
|
|
state["failure_streak"] = 0
|
|
state["success_streak"] = 0
|
|
state["first_failure_at"] = None
|
|
desired = "healthy" if previous != "recovered" else "recovered"
|
|
else:
|
|
if recovery_sample:
|
|
state["success_streak"] = int(state.get("success_streak", 0)) + 1
|
|
state["failure_streak"] = 0
|
|
desired = "recovered" if state["success_streak"] >= thresholds["recovery_consecutive_samples"] else "recovering"
|
|
else:
|
|
state["success_streak"] = 0
|
|
desired = "outage_confirmed"
|
|
|
|
if desired != previous:
|
|
transition = desired
|
|
state["lifecycle_state"] = desired
|
|
state["last_transition_at"] = observed_at
|
|
if desired == "outage_confirmed" and not state.get("incident_id"):
|
|
compact = observed_at.replace("-", "").replace(":", "").replace("T", "-").replace("Z", "")
|
|
state["incident_id"] = f"INC-L0-{compact}"
|
|
if desired == "healthy":
|
|
state["incident_id"] = None
|
|
state["last_observed_at"] = observed_at
|
|
sample = {
|
|
"outage_candidate": outage_sample,
|
|
"recovery_candidate": recovery_sample,
|
|
"failed_target_ids": failed,
|
|
"healthy_target_ids": healthy,
|
|
}
|
|
return state, transition, sample
|
|
|
|
|
|
def validate_required_callbacks(config: dict[str, Any], lifecycle: str) -> list[str]:
|
|
configured = {str(item["id"]) for item in (config.get("callbacks") or {}).get(lifecycle, [])}
|
|
required = {str(item) for item in (config.get("required_callback_ids") or {}).get(lifecycle, [])}
|
|
return sorted(required - configured)
|
|
|
|
|
|
def run_callbacks(
|
|
config: dict[str, Any],
|
|
lifecycle: str,
|
|
event_file: Path,
|
|
apply: bool,
|
|
) -> tuple[list[dict[str, Any]], list[str]]:
|
|
receipts: list[dict[str, Any]] = []
|
|
blockers: list[str] = []
|
|
missing = validate_required_callbacks(config, lifecycle)
|
|
blockers.extend(f"required_callback_missing:{callback_id}" for callback_id in missing)
|
|
for action in (config.get("callbacks") or {}).get(lifecycle, []):
|
|
action_id = str(action["id"])
|
|
action_path = Path(str(action["path"]))
|
|
if not action_path.is_file() or not os.access(action_path, os.X_OK):
|
|
blockers.append(f"callback_not_executable:{action_id}")
|
|
receipts.append({"id": action_id, "status": "blocked_not_executable", "path": str(action_path)})
|
|
continue
|
|
if not apply:
|
|
receipts.append({"id": action_id, "status": "check_only", "path": str(action_path)})
|
|
continue
|
|
try:
|
|
completed = subprocess.run(
|
|
[str(action_path), "--event-file", str(event_file)],
|
|
stdin=subprocess.DEVNULL,
|
|
stdout=subprocess.DEVNULL,
|
|
stderr=subprocess.DEVNULL,
|
|
timeout=int(action.get("timeout_seconds", 30)),
|
|
check=False,
|
|
)
|
|
status = "executed" if completed.returncode == 0 else "failed"
|
|
receipts.append({"id": action_id, "status": status, "exit_code": completed.returncode})
|
|
if completed.returncode != 0:
|
|
blockers.append(f"callback_failed:{action_id}")
|
|
except subprocess.TimeoutExpired:
|
|
receipts.append({"id": action_id, "status": "timeout"})
|
|
blockers.append(f"callback_timeout:{action_id}")
|
|
return receipts, blockers
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
observed = now_utc(args.now)
|
|
observed_at = isoformat(observed)
|
|
config = read_json(args.config)
|
|
validate_config(config)
|
|
state = load_state(args.state_file, observed_at)
|
|
previous_state = str(state["lifecycle_state"])
|
|
results = collect_results(config, args.samples_json)
|
|
thresholds = {key: int(value) for key, value in config["thresholds"].items()}
|
|
state, transition, sample = advance_state(state, results, thresholds, observed_at)
|
|
run_id = f"external-l0-{observed.strftime('%Y%m%dT%H%M%SZ')}"
|
|
artifact_file = args.artifact_dir / f"{run_id}.json"
|
|
event: dict[str, Any] = {
|
|
"schema_version": SCHEMA_VERSION,
|
|
"run_id": run_id,
|
|
"observed_at": observed_at,
|
|
"mode": "apply" if args.apply else "check",
|
|
"previous_state": previous_state,
|
|
"lifecycle_state": state["lifecycle_state"],
|
|
"transition": transition,
|
|
"incident_id": state.get("incident_id"),
|
|
"sample": sample,
|
|
"target_results": results,
|
|
"callback_receipts": [],
|
|
"blockers": [],
|
|
"secret_value_read": False,
|
|
"arbitrary_shell_allowed": False,
|
|
"remote_write_performed": False,
|
|
}
|
|
write_json(artifact_file, event)
|
|
callback_receipts: list[dict[str, Any]] = []
|
|
blockers: list[str] = []
|
|
if transition:
|
|
callback_receipts, blockers = run_callbacks(config, transition, artifact_file, args.apply)
|
|
event["callback_receipts"] = callback_receipts
|
|
event["blockers"] = blockers
|
|
event["runtime_ready"] = not blockers
|
|
event["remote_write_performed"] = args.apply and any(
|
|
receipt.get("status") == "executed" for receipt in callback_receipts
|
|
)
|
|
write_json(artifact_file, event)
|
|
write_json(args.state_file, state)
|
|
print(json.dumps({
|
|
"run_id": run_id,
|
|
"state": state["lifecycle_state"],
|
|
"transition": transition,
|
|
"incident_id": state.get("incident_id"),
|
|
"runtime_ready": event["runtime_ready"],
|
|
"artifact": str(artifact_file),
|
|
}, ensure_ascii=True))
|
|
return 2 if args.apply and blockers else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
try:
|
|
raise SystemExit(main())
|
|
except (OSError, ValueError, json.JSONDecodeError) as exc:
|
|
print(f"external_l0_supervisor_error={exc}", file=sys.stderr)
|
|
raise SystemExit(2)
|