From 10df0e9c957ed84a77e01dc40710acb54b2f8211 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 15:58:45 +0800 Subject: [PATCH] feat(recovery): add independent reboot drill observer --- .../full-host-reboot-drill-observer.sh | 169 ++++++++++++++++++ ...ull_host_reboot_drill_observer_contract.py | 36 ++++ 2 files changed, 205 insertions(+) create mode 100755 scripts/reboot-recovery/full-host-reboot-drill-observer.sh create mode 100644 scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py diff --git a/scripts/reboot-recovery/full-host-reboot-drill-observer.sh b/scripts/reboot-recovery/full-host-reboot-drill-observer.sh new file mode 100755 index 000000000..7113d9043 --- /dev/null +++ b/scripts/reboot-recovery/full-host-reboot-drill-observer.sh @@ -0,0 +1,169 @@ +#!/usr/bin/env bash +set -u + +DURATION_SECONDS="${DURATION_SECONDS:-900}" +INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}" +CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-3}" +RUN_ID="${RUN_ID:-$(date -u '+%Y%m%dT%H%M%SZ')}" +ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-full-host-reboot-drill/${RUN_ID}}" + +HOST_SPECS=( + "99|192.168.0.99|22" + "110|192.168.0.110|22" + "111|192.168.0.111|22" + "112|192.168.0.112|22" + "120|192.168.0.120|22" + "121|192.168.0.121|22" + "188|192.168.0.188|22" +) +ROUTE_SPECS=( + "awoooi-api|https://awoooi.wooo.work/api/v1/health" + "awoooi-web|https://awoooi.wooo.work/" + "stock|https://stock.wooo.work/api/v1/system/freshness" + "momo|https://mo.wooo.work/health" + "gitea|https://gitea.wooo.work/" + "harbor|https://harbor.wooo.work/" +) + +usage() { + printf '%s\n' "usage: $0 [--duration SECONDS] [--interval SECONDS] [--artifact-dir DIR]" +} + +is_positive_integer() { + [[ "$1" =~ ^[1-9][0-9]*$ ]] +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --duration) + shift + DURATION_SECONDS="${1:-}" + ;; + --interval) + shift + INTERVAL_SECONDS="${1:-}" + ;; + --artifact-dir) + shift + ARTIFACT_DIR="${1:-}" + ;; + --help|-h) + usage + exit 0 + ;; + *) + printf 'error=unknown_argument:%s\n' "$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +if ! is_positive_integer "$DURATION_SECONDS" || ! is_positive_integer "$INTERVAL_SECONDS"; then + printf '%s\n' "error=duration_and_interval_must_be_positive_integers" >&2 + exit 64 +fi + +mkdir -p "$ARTIFACT_DIR/state" +SAMPLES_TSV="$ARTIFACT_DIR/samples.tsv" +EVENTS_TSV="$ARTIFACT_DIR/events.tsv" +SUMMARY_TXT="$ARTIFACT_DIR/summary.txt" +printf 'observed_at\tkind\ttarget\tstate\tdetail\n' >"$SAMPLES_TSV" +printf 'observed_at\tkind\ttarget\tprevious_state\tstate\tdetail\n' >"$EVENTS_TSV" + +sanitize_name() { + printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_' +} + +probe_host() { + local observed_at="$1" alias="$2" address="$3" port="$4" output="$5" + if nc -z -w "$CONNECT_TIMEOUT_SECONDS" "$address" "$port" >/dev/null 2>&1; then + printf '%s\thost\t%s\tup\ttcp_%s_open\n' "$observed_at" "$alias" "$port" >"$output" + else + printf '%s\thost\t%s\tdown\ttcp_%s_closed_or_unreachable\n' "$observed_at" "$alias" "$port" >"$output" + fi +} + +probe_route() { + local observed_at="$1" name="$2" url="$3" output="$4" + local code="000" + code="$(curl --silent --show-error --location --max-time "$CONNECT_TIMEOUT_SECONDS" --output /dev/null --write-out '%{http_code}' "$url" 2>/dev/null || true)" + case "$code" in + 2??|3??|401) + printf '%s\troute\t%s\tup\thttp_%s\n' "$observed_at" "$name" "$code" >"$output" + ;; + *) + printf '%s\troute\t%s\tdown\thttp_%s\n' "$observed_at" "$name" "${code:-000}" >"$output" + ;; + esac +} + +record_transition() { + local observed_at="$1" kind="$2" target="$3" state="$4" detail="$5" + local state_file="$ARTIFACT_DIR/state/$(sanitize_name "${kind}-${target}").state" + local previous="unknown" + if [[ -f "$state_file" ]]; then + previous="$(<"$state_file")" + fi + if [[ "$state" != "$previous" ]]; then + printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$observed_at" "$kind" "$target" "$previous" "$state" "$detail" >>"$EVENTS_TSV" + printf '%s' "$state" >"$state_file" + fi +} + +START_EPOCH="$(date +%s)" +DEADLINE_EPOCH="$((START_EPOCH + DURATION_SECONDS))" + +while [[ "$(date +%s)" -lt "$DEADLINE_EPOCH" ]]; do + observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" + sample_dir="$ARTIFACT_DIR/sample-$(date +%s)-$$" + mkdir -p "$sample_dir" + + for spec in "${HOST_SPECS[@]}"; do + IFS='|' read -r alias address port <<<"$spec" + probe_host "$observed_at" "$alias" "$address" "$port" "$sample_dir/host-$alias.tsv" & + done + for spec in "${ROUTE_SPECS[@]}"; do + IFS='|' read -r name url <<<"$spec" + probe_route "$observed_at" "$name" "$url" "$sample_dir/route-$name.tsv" & + done + wait + + for sample in "$sample_dir"/*.tsv; do + IFS=$'\t' read -r timestamp kind target state detail <"$sample" + printf '%s\t%s\t%s\t%s\t%s\n' "$timestamp" "$kind" "$target" "$state" "$detail" >>"$SAMPLES_TSV" + record_transition "$timestamp" "$kind" "$target" "$state" "$detail" + done + rm -rf "$sample_dir" + sleep "$INTERVAL_SECONDS" +done + +host_down_samples="$(awk -F '\t' '$2=="host" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")" +route_down_samples="$(awk -F '\t' '$2=="route" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")" +down_transitions="$(awk -F '\t' '$5=="down" {count++} END {print count+0}' "$EVENTS_TSV")" +recovery_transitions="$(awk -F '\t' '$4=="down" && $5=="up" {count++} END {print count+0}' "$EVENTS_TSV")" +first_down_at="$(awk -F '\t' '$5=="down" {print $1; exit}' "$EVENTS_TSV")" +last_recovered_at="$(awk -F '\t' '$4=="down" && $5=="up" {value=$1} END {print value}' "$EVENTS_TSV")" + +{ + printf 'schema_version=full_host_reboot_drill_observer_v1\n' + printf 'run_id=%s\n' "$RUN_ID" + printf 'duration_seconds=%s\n' "$DURATION_SECONDS" + printf 'interval_seconds=%s\n' "$INTERVAL_SECONDS" + printf 'host_count=%s\n' "${#HOST_SPECS[@]}" + printf 'route_count=%s\n' "${#ROUTE_SPECS[@]}" + printf 'host_down_samples=%s\n' "$host_down_samples" + printf 'route_down_samples=%s\n' "$route_down_samples" + printf 'down_transitions=%s\n' "$down_transitions" + printf 'recovery_transitions=%s\n' "$recovery_transitions" + printf 'first_down_at=%s\n' "${first_down_at:-none}" + printf 'last_recovered_at=%s\n' "${last_recovered_at:-none}" + printf 'samples=%s\n' "$SAMPLES_TSV" + printf 'events=%s\n' "$EVENTS_TSV" + printf 'secret_value_read=false\n' + printf 'remote_write_performed=false\n' + printf 'reboot_command_executed=false\n' +} >"$SUMMARY_TXT" + +cat "$SUMMARY_TXT" diff --git a/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py b/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py new file mode 100644 index 000000000..c628e42f7 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_full_host_reboot_drill_observer_contract.py @@ -0,0 +1,36 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts" / "reboot-recovery" / "full-host-reboot-drill-observer.sh" + + +def test_observer_covers_all_hosts_and_core_routes() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + for alias in ("99", "110", "111", "112", "120", "121", "188"): + assert f'"{alias}|192.168.0.{alias}|22"' in text + for route in ( + "awoooi.wooo.work/api/v1/health", + "stock.wooo.work/api/v1/system/freshness", + "mo.wooo.work/health", + "gitea.wooo.work", + "harbor.wooo.work", + ): + assert route in text + + +def test_observer_is_read_only_and_records_transitions() -> None: + text = SCRIPT.read_text(encoding="utf-8") + + assert "probe_host" in text + assert "probe_route" in text + assert "record_transition" in text + assert "samples.tsv" in text + assert "events.tsv" in text + assert "secret_value_read=false" in text + assert "remote_write_performed=false" in text + assert "reboot_command_executed=false" in text + assert "shutdown" not in text + assert "Restart-Computer" not in text + assert "TELEGRAM_BOT_TOKEN" not in text