#!/usr/bin/env bash set -u DURATION_SECONDS="${DURATION_SECONDS:-900}" INTERVAL_SECONDS="${INTERVAL_SECONDS:-5}" CONNECT_TIMEOUT_SECONDS="${CONNECT_TIMEOUT_SECONDS:-3}" TCP_PROBE_PYTHON="${TCP_PROBE_PYTHON:-python3}" RUN_ID="${RUN_ID:-$(date -u '+%Y%m%dT%H%M%SZ')}" ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-full-host-reboot-drill/${RUN_ID}}" HOST_SPECS=( "99|192.168.0.99|22" "110|192.168.0.110|22" "111|192.168.0.111|22" "112|192.168.0.112|22" "120|192.168.0.120|22" "121|192.168.0.121|22" "188|192.168.0.188|22" ) ROUTE_SPECS=( "awoooi-api|https://awoooi.wooo.work/api/v1/health" "awoooi-web|https://awoooi.wooo.work/" "stock|https://stock.wooo.work/api/v1/system/freshness" "momo|https://mo.wooo.work/health" "gitea|https://gitea.wooo.work/" "harbor|https://harbor.wooo.work/" ) usage() { printf '%s\n' "usage: $0 [--duration SECONDS] [--interval SECONDS] [--artifact-dir DIR]" } is_positive_integer() { [[ "$1" =~ ^[1-9][0-9]*$ ]] } while [[ $# -gt 0 ]]; do case "$1" in --duration) shift DURATION_SECONDS="${1:-}" ;; --interval) shift INTERVAL_SECONDS="${1:-}" ;; --artifact-dir) shift ARTIFACT_DIR="${1:-}" ;; --help|-h) usage exit 0 ;; *) printf 'error=unknown_argument:%s\n' "$1" >&2 usage >&2 exit 64 ;; esac shift done if ! is_positive_integer "$DURATION_SECONDS" || ! is_positive_integer "$INTERVAL_SECONDS"; then printf '%s\n' "error=duration_and_interval_must_be_positive_integers" >&2 exit 64 fi if ! command -v "$TCP_PROBE_PYTHON" >/dev/null 2>&1; then printf 'error=tcp_probe_python_not_found:%s\n' "$TCP_PROBE_PYTHON" >&2 exit 69 fi mkdir -p "$ARTIFACT_DIR/state" SAMPLES_TSV="$ARTIFACT_DIR/samples.tsv" EVENTS_TSV="$ARTIFACT_DIR/events.tsv" SUMMARY_TXT="$ARTIFACT_DIR/summary.txt" printf 'observed_at\tkind\ttarget\tstate\tdetail\n' >"$SAMPLES_TSV" printf 'observed_at\tkind\ttarget\tprevious_state\tstate\tdetail\n' >"$EVENTS_TSV" sanitize_name() { printf '%s' "$1" | tr -c 'A-Za-z0-9._-' '_' } probe_host() { local observed_at="$1" alias="$2" address="$3" port="$4" output="$5" if "$TCP_PROBE_PYTHON" - "$address" "$port" "$CONNECT_TIMEOUT_SECONDS" >/dev/null 2>&1 <<'PY'; then import socket import sys address = sys.argv[1] port = int(sys.argv[2]) timeout_seconds = float(sys.argv[3]) with socket.create_connection((address, port), timeout=timeout_seconds): pass PY printf '%s\thost\t%s\tup\ttcp_%s_open\n' "$observed_at" "$alias" "$port" >"$output" else printf '%s\thost\t%s\tdown\ttcp_%s_closed_or_unreachable\n' "$observed_at" "$alias" "$port" >"$output" fi } probe_route() { local observed_at="$1" name="$2" url="$3" output="$4" local code="000" code="$(curl --silent --show-error --location --max-time "$CONNECT_TIMEOUT_SECONDS" --output /dev/null --write-out '%{http_code}' "$url" 2>/dev/null || true)" case "$code" in 2??|3??|401) printf '%s\troute\t%s\tup\thttp_%s\n' "$observed_at" "$name" "$code" >"$output" ;; *) printf '%s\troute\t%s\tdown\thttp_%s\n' "$observed_at" "$name" "${code:-000}" >"$output" ;; esac } record_transition() { local observed_at="$1" kind="$2" target="$3" state="$4" detail="$5" local state_file="$ARTIFACT_DIR/state/$(sanitize_name "${kind}-${target}").state" local previous="unknown" if [[ -f "$state_file" ]]; then previous="$(<"$state_file")" fi if [[ "$state" != "$previous" ]]; then printf '%s\t%s\t%s\t%s\t%s\t%s\n' "$observed_at" "$kind" "$target" "$previous" "$state" "$detail" >>"$EVENTS_TSV" printf '%s' "$state" >"$state_file" fi } START_EPOCH="$(date +%s)" DEADLINE_EPOCH="$((START_EPOCH + DURATION_SECONDS))" while [[ "$(date +%s)" -lt "$DEADLINE_EPOCH" ]]; do observed_at="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" sample_dir="$ARTIFACT_DIR/sample-$(date +%s)-$$" mkdir -p "$sample_dir" for spec in "${HOST_SPECS[@]}"; do IFS='|' read -r alias address port <<<"$spec" probe_host "$observed_at" "$alias" "$address" "$port" "$sample_dir/host-$alias.tsv" & done for spec in "${ROUTE_SPECS[@]}"; do IFS='|' read -r name url <<<"$spec" probe_route "$observed_at" "$name" "$url" "$sample_dir/route-$name.tsv" & done wait for sample in "$sample_dir"/*.tsv; do IFS=$'\t' read -r timestamp kind target state detail <"$sample" printf '%s\t%s\t%s\t%s\t%s\n' "$timestamp" "$kind" "$target" "$state" "$detail" >>"$SAMPLES_TSV" record_transition "$timestamp" "$kind" "$target" "$state" "$detail" done rm -rf "$sample_dir" sleep "$INTERVAL_SECONDS" done host_down_samples="$(awk -F '\t' '$2=="host" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")" route_down_samples="$(awk -F '\t' '$2=="route" && $4=="down" {count++} END {print count+0}' "$SAMPLES_TSV")" down_transitions="$(awk -F '\t' '$5=="down" {count++} END {print count+0}' "$EVENTS_TSV")" recovery_transitions="$(awk -F '\t' '$4=="down" && $5=="up" {count++} END {print count+0}' "$EVENTS_TSV")" first_down_at="$(awk -F '\t' '$5=="down" {print $1; exit}' "$EVENTS_TSV")" last_recovered_at="$(awk -F '\t' '$4=="down" && $5=="up" {value=$1} END {print value}' "$EVENTS_TSV")" { printf 'schema_version=full_host_reboot_drill_observer_v1\n' printf 'run_id=%s\n' "$RUN_ID" printf 'duration_seconds=%s\n' "$DURATION_SECONDS" printf 'interval_seconds=%s\n' "$INTERVAL_SECONDS" printf 'host_count=%s\n' "${#HOST_SPECS[@]}" printf 'route_count=%s\n' "${#ROUTE_SPECS[@]}" printf 'host_down_samples=%s\n' "$host_down_samples" printf 'route_down_samples=%s\n' "$route_down_samples" printf 'down_transitions=%s\n' "$down_transitions" printf 'recovery_transitions=%s\n' "$recovery_transitions" printf 'first_down_at=%s\n' "${first_down_at:-none}" printf 'last_recovered_at=%s\n' "${last_recovered_at:-none}" printf 'samples=%s\n' "$SAMPLES_TSV" printf 'events=%s\n' "$EVENTS_TSV" printf 'secret_value_read=false\n' printf 'remote_write_performed=false\n' printf 'reboot_command_executed=false\n' } >"$SUMMARY_TXT" cat "$SUMMARY_TXT"