#!/usr/bin/env bash
# Boot-triggerable 10-minute reboot recovery SLO exporter.
#
# It is read-only: it probes boot state, runs the existing post-reboot summary,
# evaluates the SLO scorecard, and writes node-exporter textfile metrics.

set -uo pipefail

ROOT_DIR="${ROOT_DIR:-/opt/awoooi}"
TEXTFILE_DIR="${TEXTFILE_DIR:-${NODE_EXPORTER_TEXTFILE_DIR:-/home/wooo/node_exporter_textfiles}}"
LOG_DIR="${LOG_DIR:-/home/wooo/reboot-recovery}"
OUTPUT_NAME="${OUTPUT_NAME:-reboot_auto_recovery_slo.prom}"
TARGET_MINUTES="${TARGET_MINUTES:-10}"
MIN_FREE_GIB="${MIN_FREE_GIB:-2}"
LOCK_FILE="${LOCK_FILE:-${LOG_DIR}/reboot_auto_recovery_slo.lock}"
STOCK_FRESHNESS_URL="${STOCK_FRESHNESS_URL:-https://stock.wooo.work/api/v1/system/freshness}"
STOCK_INGESTION_URL="${STOCK_INGESTION_URL:-https://stock.wooo.work/api/v1/system/ingestion}"
STOCK_READBACK_TIMEOUT_SECONDS="${STOCK_READBACK_TIMEOUT_SECONDS:-10}"
PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}"
PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}"

mkdir -p "$TEXTFILE_DIR" "$LOG_DIR"

if command -v flock >/dev/null 2>&1; then
  exec 9>"$LOCK_FILE"
  flock -n 9 || exit 0
fi

run_id="$(date '+%Y%m%d-%H%M%S')"
artifact_dir="$LOG_DIR/reboot-auto-recovery-slo-$run_id"
mkdir -p "$artifact_dir"

host_probe="$artifact_dir/host-probe.txt"
reboot_event_file="$artifact_dir/reboot-event.json"
reboot_event_prom="$artifact_dir/reboot-event.prom"
summary_file="$artifact_dir/summary.txt"
scorecard_file="$artifact_dir/scorecard.json"
stock_freshness_file="$artifact_dir/stock-freshness.json"
stock_ingestion_file="$artifact_dir/stock-ingestion.json"
public_maintenance_file="$artifact_dir/public-maintenance-fallback.json"
public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom"
windows99_management_file="$artifact_dir/windows99-management-channel.json"
reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}"

bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true
python3 "$ROOT_DIR/scripts/reboot-recovery/reboot-event-detector.py" \
  --host-probe-file "$host_probe" \
  --state-file "$reboot_event_state_file" \
  --target-minutes "$TARGET_MINUTES" \
  --output "$reboot_event_file" \
  --prometheus-output "$reboot_event_prom" || true
ARTIFACT_DIR="$artifact_dir/post-reboot-readiness" \
  bash "$ROOT_DIR/scripts/reboot-recovery/post-reboot-readiness-summary.sh" --no-color >"$summary_file" 2>&1 || true

if command -v curl >/dev/null 2>&1; then
  curl -fsS --max-time "$STOCK_READBACK_TIMEOUT_SECONDS" \
    "$STOCK_FRESHNESS_URL" >"$stock_freshness_file" 2>"$artifact_dir/stock-freshness.err" \
    || rm -f "$stock_freshness_file"
  curl -fsS --max-time "$STOCK_READBACK_TIMEOUT_SECONDS" \
    "$STOCK_INGESTION_URL" >"$stock_ingestion_file" 2>"$artifact_dir/stock-ingestion.err" \
    || rm -f "$stock_ingestion_file"
fi

if [ -f "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py" ]; then
  public_probe_args=(
    "$ROOT_DIR/scripts/reboot-recovery/public-maintenance-fallback-probe.py"
    --timeout "$PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS"
    --output "$public_maintenance_file"
    --prometheus-output "$public_maintenance_prom"
  )
  for url in $PUBLIC_MAINTENANCE_URLS; do
    public_probe_args+=(--url "$url")
  done
  python3 "${public_probe_args[@]}" >"$artifact_dir/public-maintenance-fallback.stdout" 2>"$artifact_dir/public-maintenance-fallback.err" \
    || rm -f "$public_maintenance_file" "$public_maintenance_prom"
fi

if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" ]; then
  python3 "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py" \
    --output "$windows99_management_file" >"$artifact_dir/windows99-management-channel.stdout" 2>"$artifact_dir/windows99-management-channel.err" \
    || rm -f "$windows99_management_file"
fi

scorecard_args=(
  "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py"
  --summary-file "$summary_file" \
  --host-probe-file "$host_probe" \
  --reboot-event-file "$reboot_event_file" \
  --target-minutes "$TARGET_MINUTES" \
  --min-free-gib "$MIN_FREE_GIB" \
  --disk-path / \
  --output "$scorecard_file"
)
if [ -s "$stock_freshness_file" ]; then
  scorecard_args+=(--stock-freshness-file "$stock_freshness_file")
fi
if [ -s "$stock_ingestion_file" ]; then
  scorecard_args+=(--stock-ingestion-file "$stock_ingestion_file")
fi
if [ -s "$public_maintenance_file" ]; then
  scorecard_args+=(--public-maintenance-file "$public_maintenance_file")
fi
if [ -s "$windows99_management_file" ]; then
  scorecard_args+=(--windows99-management-file "$windows99_management_file")
fi

python3 "${scorecard_args[@]}" || true

now="$(date +%s)"
ready="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(1 if payload.get("can_claim_all_services_recovered_within_target") else 0)
PY
)"
blocker_count="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(len(payload.get("active_blockers") or []))
PY
)"
max_uptime="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(payload.get("host_boot_detection", {}).get("max_observed_uptime_seconds", 0))
PY
)"
target_seconds_remaining="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(payload.get("sla_recovery_eta", {}).get("target_seconds_remaining", 0) or 0)
PY
)"
public_maintenance_ready="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(1 if payload.get("public_maintenance_fallback", {}).get("ready") else 0)
PY
)"
public_raw_5xx_count="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(payload.get("public_maintenance_fallback", {}).get("raw_5xx_without_fallback_count", 0) or 0)
PY
)"
public_unreachable_count="$(python3 - "$scorecard_file" <<'PY'
import json, sys
payload=json.load(open(sys.argv[1], encoding="utf-8"))
print(payload.get("public_maintenance_fallback", {}).get("route_unreachable_without_external_fallback_count", 0) or 0)
PY
)"
active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY'
import json, sys

payload = json.load(open(sys.argv[1], encoding="utf-8"))
target_minutes = str(sys.argv[2])
scope = "99_110_111_112_120_121_188"
matrix = payload.get("active_blocker_action_matrix") or {}
items = matrix.get("items") if isinstance(matrix, dict) else []
if not isinstance(items, list):
    items = []
actions_by_blocker = {
    str(item.get("blocker") or ""): item
    for item in items
    if isinstance(item, dict) and item.get("blocker")
}


def label(value: object) -> str:
    text = str(value or "")
    return (
        text.replace("\\", "\\\\")
        .replace("\n", "\\n")
        .replace('"', '\\"')
    )


for blocker in payload.get("active_blockers") or []:
    if blocker:
        action = actions_by_blocker.get(str(blocker), {})
        print(
            "awoooi_reboot_auto_recovery_slo_active_blocker"
            f'{{scope="{scope}",target_minutes="{label(target_minutes)}",'
            f'blocker="{label(blocker)}",'
            f'category="{label(action.get("category"))}",'
            f'owner_lane="{label(action.get("owner_lane"))}",'
            f'severity="{label(action.get("telegram_severity"))}",'
            f'primary="{label(str(action.get("is_primary") is True).lower())}"}} 1'
        )

primary_blocker = payload.get("primary_blocker") or ""
if primary_blocker:
    action = actions_by_blocker.get(str(primary_blocker), {})
    print(
        "awoooi_reboot_auto_recovery_slo_primary_blocker"
        f'{{scope="{scope}",target_minutes="{label(target_minutes)}",'
        f'blocker="{label(primary_blocker)}",'
        f'category="{label(action.get("category"))}",'
        f'owner_lane="{label(action.get("owner_lane"))}",'
        f'severity="{label(action.get("telegram_severity"))}"}} 1'
    )
PY
)"

tmp_metric="$(mktemp "$TEXTFILE_DIR/.reboot_auto_recovery_slo.XXXXXX")"
cat >"$tmp_metric" <<METRICS
# HELP awoooi_reboot_auto_recovery_slo_ready Whether all P0 hosts auto-recovered within the target window.
# TYPE awoooi_reboot_auto_recovery_slo_ready gauge
awoooi_reboot_auto_recovery_slo_ready{scope="99_110_111_112_120_121_188",target_minutes="$TARGET_MINUTES"} $ready
# HELP awoooi_reboot_auto_recovery_slo_blocker_count Number of active fail-closed blockers for the reboot recovery SLO.
# TYPE awoooi_reboot_auto_recovery_slo_blocker_count gauge
awoooi_reboot_auto_recovery_slo_blocker_count{scope="99_110_111_112_120_121_188",target_minutes="$TARGET_MINUTES"} $blocker_count
# HELP awoooi_reboot_auto_recovery_slo_active_blocker Active fail-closed reboot SLO blockers projected from scorecard active_blockers with category and owner lane labels.
# TYPE awoooi_reboot_auto_recovery_slo_active_blocker gauge
# HELP awoooi_reboot_auto_recovery_slo_primary_blocker Primary reboot SLO blocker selected by the fixed SOP priority order with category and owner lane labels.
# TYPE awoooi_reboot_auto_recovery_slo_primary_blocker gauge
# HELP awoooi_reboot_auto_recovery_slo_max_host_uptime_seconds Max observed host uptime in the boot probe.
# TYPE awoooi_reboot_auto_recovery_slo_max_host_uptime_seconds gauge
awoooi_reboot_auto_recovery_slo_max_host_uptime_seconds{scope="99_110_111_112_120_121_188"} $max_uptime
# HELP awoooi_reboot_auto_recovery_slo_target_seconds_remaining Seconds remaining before the 10-minute target expires.
# TYPE awoooi_reboot_auto_recovery_slo_target_seconds_remaining gauge
awoooi_reboot_auto_recovery_slo_target_seconds_remaining{scope="99_110_111_112_120_121_188",target_minutes="$TARGET_MINUTES"} $target_seconds_remaining
# HELP awoooi_reboot_auto_recovery_slo_last_run_timestamp Last SLO exporter run timestamp.
# TYPE awoooi_reboot_auto_recovery_slo_last_run_timestamp gauge
awoooi_reboot_auto_recovery_slo_last_run_timestamp{scope="99_110_111_112_120_121_188"} $now
# HELP awoooi_public_maintenance_fallback_ready Whether public routes avoid raw 5xx exposure.
# TYPE awoooi_public_maintenance_fallback_ready gauge
awoooi_public_maintenance_fallback_ready{scope="public_routes"} $public_maintenance_ready
# HELP awoooi_public_route_raw_5xx_without_maintenance_fallback Raw public 5xx responses without maintenance fallback.
# TYPE awoooi_public_route_raw_5xx_without_maintenance_fallback gauge
awoooi_public_route_raw_5xx_without_maintenance_fallback{scope="public_routes"} $public_raw_5xx_count
# HELP awoooi_public_route_unreachable_without_external_l1_fallback Public routes unreachable without external fallback proof.
# TYPE awoooi_public_route_unreachable_without_external_l1_fallback gauge
awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_routes"} $public_unreachable_count
METRICS
if [ -n "$active_blocker_metrics" ]; then
  printf '%s\n' "$active_blocker_metrics" >>"$tmp_metric"
fi
if [ -s "$reboot_event_prom" ]; then
  cat "$reboot_event_prom" >>"$tmp_metric"
fi
chmod 0644 "$tmp_metric"
mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
