feat(recovery): add independent reboot drill observer
This commit is contained in:
ogt
169
scripts/reboot-recovery/full-host-reboot-drill-observer.sh
Executable file
169
scripts/reboot-recovery/full-host-reboot-drill-observer.sh
Executable file
@@ -0,0 +1,169 @@
|
||||
#!/usr/bin/env bash
|
||||
set -u
|
||||
|
||||
DURATION_SECONDS="${DURATION_SECONDS:-900}"
|
||||
INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}"
|
||||
CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-3}"
|
||||
RUN_ID="${RUN_ID:-$(date -u '+%Y%m%dT%H%M%SZ')}"
|
||||
ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-full-host-reboot-drill/${RUN_ID}}"
|
||||
|
||||
HOST_SPECS=(
|
||||
"99|192.168.0.99|22"
|
||||
"110|192.168.0.110|22"
|
||||
"111|192.168.0.111|22"
|
||||
"112|192.168.0.112|22"
|
||||
"120|192.168.0.120|22"
|
||||
"121|192.168.0.121|22"
|
||||
"188|192.168.0.188|22"
|
||||
)
|
||||
ROUTE_SPECS=(
|
||||
"awoooi-api|https://awoooi.wooo.work/api/v1/health"
|
||||
"awoooi-web|https://awoooi.wooo.work/"
|
||||
"stock|https://stock.wooo.work/api/v1/system/freshness"
|
||||
"momo|https://mo.wooo.work/health"
|
||||
"gitea|https://gitea.wooo.work/"
|
||||
"harbor|https://harbor.wooo.work/"
|
||||
)
|
||||
|
||||
usage() {
|
||||
printf '%s\n' "usage: $0 [--duration SECONDS] [--interval SECONDS] [--artifact-dir DIR]"
|
||||
}
|
||||
|
||||
is_positive_integer() {
|
||||
[[ "$1" =~ ^[1-9][0-9]*$ ]]
|
||||
}
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--duration)
|
||||
shift
|
||||
DURATION_SECONDS="${1:-}"
|
||||
;;
|
||||
--interval)
|
||||
shift
|
||||
INTERVAL_SECONDS="${1:-}"
|
||||
;;
|
||||
--artifact-dir)
|
||||
shift
|
||||
ARTIFACT_DIR="${1:-}"
|
||||
;;
|
||||
--help|-h)
|
||||
usage
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'error=unknown_argument:%s\n' "$1" >&2
|
||||
usage >&2
|
||||
exit 64
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
if ! is_positive_integer "$DURATION_SECONDS" || ! is_positive_integer "$INTERVAL_SECONDS"; then
|
||||
printf '%s\n' "error=duration_and_interval_must_be_positive_integers" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
mkdir -p "$ARTIFACT_DIR/state"
|
||||
SAMPLES_TSV="$ARTIFACT_DIR/samples.tsv"
|
||||
EVENTS_TSV="$ARTIFACT_DIR/events.tsv"
|
||||
SUMMARY_TXT="$ARTIFACT_DIR/summary.txt"
|
||||
printf 'observed_at\tkind\ttarget\tstate\tdetail\n' >"$SAMPLES_TSV"
|
||||
printf 'observed_at\tkind\ttarget\tprevious_state\tstate\tdetail\n' >"$EVENTS_TSV"
|
||||
|
||||
sanitize_name() {
|
||||
printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_'
|
||||
}
|
||||
|
||||
probe_host() {
|
||||
local observed_at="$1" alias="$2" address="$3" port="$4" output="$5"
|
||||
if nc -z -w "$CONNECT_TIMEOUT_SECONDS" "$address" "$port" >/dev/null 2>&1; then
|
||||
printf '%s\thost\t%s\tup\ttcp_%s_open\n' "$observed_at" "$alias" "$port" >"$output"
|
||||
else
|
||||
printf '%s\thost\t%s\tdown\ttcp_%s_closed_or_unreachable\n' "$observed_at" "$alias" "$port" >"$output"
|
||||
fi
|
||||
}
|
||||
|
||||
probe_route() {
|
||||
local observed_at="$1" name="$2" url="$3" output="$4"
|
||||
local code="000"
|
||||
code="$(curl --silent --show-error --location --max-time "$CONNECT_TIMEOUT_SECONDS" --output /dev/null --write-out '%{http_code}' "$url" 2>/dev/null || true)"
|
||||
case "$code" in
|
||||
2??|3??|401)
|
||||
printf '%s\troute\t%s\tup\thttp_%s\n' "$observed_at" "$name" "$code" >"$output"
|
||||
;;
|
||||
*)
|
||||
printf '%s\troute\t%s\tdown\thttp_%s\n' "$observed_at" "$name" "${code:-000}" >"$output"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
record_transition() {
|
||||
local observed_at="$1" kind="$2" target="$3" state="$4" detail="$5"
|
||||
local state_file="$ARTIFACT_DIR/state/$(sanitize_name "${kind}-${target}").state"
|
||||
local previous="unknown"
|
||||
if [[ -f "$state_file" ]]; then
|
||||
previous="$(<"$state_file")"
|
||||
fi
|
||||
if [[ "$state" != "$previous" ]]; then
|
||||
printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$observed_at" "$kind" "$target" "$previous" "$state" "$detail" >>"$EVENTS_TSV"
|
||||
printf '%s' "$state" >"$state_file"
|
||||
fi
|
||||
}
|
||||
|
||||
START_EPOCH="$(date +%s)"
|
||||
DEADLINE_EPOCH="$((START_EPOCH + DURATION_SECONDS))"
|
||||
|
||||
while [[ "$(date +%s)" -lt "$DEADLINE_EPOCH" ]]; do
|
||||
observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')"
|
||||
sample_dir="$ARTIFACT_DIR/sample-$(date +%s)-$$"
|
||||
mkdir -p "$sample_dir"
|
||||
|
||||
for spec in "${HOST_SPECS[@]}"; do
|
||||
IFS='|' read -r alias address port <<<"$spec"
|
||||
probe_host "$observed_at" "$alias" "$address" "$port" "$sample_dir/host-$alias.tsv" &
|
||||
done
|
||||
for spec in "${ROUTE_SPECS[@]}"; do
|
||||
IFS='|' read -r name url <<<"$spec"
|
||||
probe_route "$observed_at" "$name" "$url" "$sample_dir/route-$name.tsv" &
|
||||
done
|
||||
wait
|
||||
|
||||
for sample in "$sample_dir"/*.tsv; do
|
||||
IFS=$'\t' read -r timestamp kind target state detail <"$sample"
|
||||
printf '%s\t%s\t%s\t%s\t%s\n' "$timestamp" "$kind" "$target" "$state" "$detail" >>"$SAMPLES_TSV"
|
||||
record_transition "$timestamp" "$kind" "$target" "$state" "$detail"
|
||||
done
|
||||
rm -rf "$sample_dir"
|
||||
sleep "$INTERVAL_SECONDS"
|
||||
done
|
||||
|
||||
host_down_samples="$(awk -F '\t' '$2=="host" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")"
|
||||
route_down_samples="$(awk -F '\t' '$2=="route" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")"
|
||||
down_transitions="$(awk -F '\t' '$5=="down" {count++} END {print count+0}' "$EVENTS_TSV")"
|
||||
recovery_transitions="$(awk -F '\t' '$4=="down" && $5=="up" {count++} END {print count+0}' "$EVENTS_TSV")"
|
||||
first_down_at="$(awk -F '\t' '$5=="down" {print $1; exit}' "$EVENTS_TSV")"
|
||||
last_recovered_at="$(awk -F '\t' '$4=="down" && $5=="up" {value=$1} END {print value}' "$EVENTS_TSV")"
|
||||
|
||||
{
|
||||
printf 'schema_version=full_host_reboot_drill_observer_v1\n'
|
||||
printf 'run_id=%s\n' "$RUN_ID"
|
||||
printf 'duration_seconds=%s\n' "$DURATION_SECONDS"
|
||||
printf 'interval_seconds=%s\n' "$INTERVAL_SECONDS"
|
||||
printf 'host_count=%s\n' "${#HOST_SPECS[@]}"
|
||||
printf 'route_count=%s\n' "${#ROUTE_SPECS[@]}"
|
||||
printf 'host_down_samples=%s\n' "$host_down_samples"
|
||||
printf 'route_down_samples=%s\n' "$route_down_samples"
|
||||
printf 'down_transitions=%s\n' "$down_transitions"
|
||||
printf 'recovery_transitions=%s\n' "$recovery_transitions"
|
||||
printf 'first_down_at=%s\n' "${first_down_at:-none}"
|
||||
printf 'last_recovered_at=%s\n' "${last_recovered_at:-none}"
|
||||
printf 'samples=%s\n' "$SAMPLES_TSV"
|
||||
printf 'events=%s\n' "$EVENTS_TSV"
|
||||
printf 'secret_value_read=false\n'
|
||||
printf 'remote_write_performed=false\n'
|
||||
printf 'reboot_command_executed=false\n'
|
||||
} >"$SUMMARY_TXT"
|
||||
|
||||
cat "$SUMMARY_TXT"
|
||||
@@ -0,0 +1,36 @@
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "reboot-recovery" / "full-host-reboot-drill-observer.sh"
|
||||
|
||||
|
||||
def test_observer_covers_all_hosts_and_core_routes() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
for alias in ("99", "110", "111", "112", "120", "121", "188"):
|
||||
assert f'"{alias}|192.168.0.{alias}|22"' in text
|
||||
for route in (
|
||||
"awoooi.wooo.work/api/v1/health",
|
||||
"stock.wooo.work/api/v1/system/freshness",
|
||||
"mo.wooo.work/health",
|
||||
"gitea.wooo.work",
|
||||
"harbor.wooo.work",
|
||||
):
|
||||
assert route in text
|
||||
|
||||
|
||||
def test_observer_is_read_only_and_records_transitions() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert "probe_host" in text
|
||||
assert "probe_route" in text
|
||||
assert "record_transition" in text
|
||||
assert "samples.tsv" in text
|
||||
assert "events.tsv" in text
|
||||
assert "secret_value_read=false" in text
|
||||
assert "remote_write_performed=false" in text
|
||||
assert "reboot_command_executed=false" in text
|
||||
assert "shutdown" not in text
|
||||
assert "Restart-Computer" not in text
|
||||
assert "TELEGRAM_BOT_TOKEN" not in text
|
||||
Reference in New Issue
Block a user