6d2d59c027
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m17s
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / build-and-deploy (push) Has been cancelled
308 lines
11 KiB
Bash
Executable File
308 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
# Full-host reboot recovery orchestrator.
|
|
# This is intentionally a coordinator: irreversible actions stay out of scope,
|
|
# while bounded service-start/recovery hooks are delegated to audited scripts.
|
|
|
|
MODE="check"
|
|
ARTIFACT_ROOT="${ARTIFACT_ROOT:-/tmp/awoooi-full-host-reboot-recovery}"
|
|
RUN_ID="${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}"
|
|
HOSTS="${HOSTS:-99 110 111 112 120 121 188}"
|
|
PUBLIC_ROUTES="${PUBLIC_ROUTES:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/healthz https://stock.wooo.work/api/healthz https://gitea.wooo.work/ https://harbor.wooo.work/ https://registry.wooo.work/v2/}"
|
|
SSH_TIMEOUT_SECONDS="${SSH_TIMEOUT_SECONDS:-12}"
|
|
RECOVERY_SLO_SECONDS="${RECOVERY_SLO_SECONDS:-600}"
|
|
TELEGRAM_NOTIFY_SCRIPT="${TELEGRAM_NOTIFY_SCRIPT:-}"
|
|
DISABLE_TELEGRAM="${DISABLE_TELEGRAM:-0}"
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ARTIFACT_DIR="${ARTIFACT_ROOT%/}/${RUN_ID}"
|
|
EVENTS_TSV="${ARTIFACT_DIR}/events.tsv"
|
|
FINDINGS_TSV="${ARTIFACT_DIR}/findings.tsv"
|
|
ACTIONS_TSV="${ARTIFACT_DIR}/actions.tsv"
|
|
SUMMARY_TXT="${ARTIFACT_DIR}/summary.txt"
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage:
|
|
full-host-reboot-orchestrator.sh [--check|--apply] [--artifact-dir DIR]
|
|
|
|
Modes:
|
|
--check Read-only orchestration and evidence collection.
|
|
--apply Run controlled apply hooks that are already allowlisted.
|
|
|
|
Environment:
|
|
HOSTS Space-separated host list, default: 99 110 111 112 120 121 188
|
|
PUBLIC_ROUTES Space-separated route readback list.
|
|
SSH_TIMEOUT_SECONDS SSH connect/command timeout, default: 12.
|
|
RECOVERY_SLO_SECONDS Full recovery target, default: 600.
|
|
TELEGRAM_NOTIFY_SCRIPT Optional no-secret wrapper script for Telegram notification.
|
|
DISABLE_TELEGRAM=1 Suppress Telegram wrapper even when configured.
|
|
USAGE
|
|
}
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--check)
|
|
MODE="check"
|
|
shift
|
|
;;
|
|
--apply)
|
|
MODE="apply"
|
|
shift
|
|
;;
|
|
--artifact-dir)
|
|
ARTIFACT_DIR="$2"
|
|
ARTIFACT_ROOT="$(dirname "$2")"
|
|
RUN_ID="$(basename "$2")"
|
|
shift 2
|
|
;;
|
|
--help|-h)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown argument: $1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
mkdir -p "$ARTIFACT_DIR"
|
|
: >"$EVENTS_TSV"
|
|
: >"$FINDINGS_TSV"
|
|
: >"$ACTIONS_TSV"
|
|
|
|
event() {
|
|
local level="$1" component="$2" message="$3"
|
|
printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$level" "$component" "$message" | tee -a "$EVENTS_TSV" >/dev/null
|
|
}
|
|
|
|
finding() {
|
|
local severity="$1" component="$2" code="$3" message="$4"
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$severity" "$component" "$code" "$message" | tee -a "$FINDINGS_TSV" >/dev/null
|
|
}
|
|
|
|
action() {
|
|
local mode="$1" component="$2" code="$3" message="$4"
|
|
printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$mode" "$component" "$code" "$message" | tee -a "$ACTIONS_TSV" >/dev/null
|
|
}
|
|
|
|
notify() {
|
|
local phase="$1" status="$2" message="$3"
|
|
if [[ "$DISABLE_TELEGRAM" == "1" || -z "$TELEGRAM_NOTIFY_SCRIPT" ]]; then
|
|
event "INFO" "telegram" "notify skipped phase=${phase} status=${status}: ${message}"
|
|
return 0
|
|
fi
|
|
if [[ ! -x "$TELEGRAM_NOTIFY_SCRIPT" ]]; then
|
|
finding "P1" "telegram" "TELEGRAM_WRAPPER_NOT_EXECUTABLE" "$TELEGRAM_NOTIFY_SCRIPT"
|
|
return 0
|
|
fi
|
|
"$TELEGRAM_NOTIFY_SCRIPT" --phase "$phase" --status "$status" --artifact-dir "$ARTIFACT_DIR" --message "$message" || {
|
|
finding "P1" "telegram" "TELEGRAM_NOTIFY_FAILED" "phase=${phase} status=${status}"
|
|
}
|
|
}
|
|
|
|
host_ip() {
|
|
case "$1" in
|
|
99) printf '192.168.0.99' ;;
|
|
110) printf '192.168.0.110' ;;
|
|
111) printf '192.168.0.111' ;;
|
|
112) printf '192.168.0.112' ;;
|
|
120) printf '192.168.0.120' ;;
|
|
121) printf '192.168.0.121' ;;
|
|
188) printf '192.168.0.188' ;;
|
|
*) printf '%s' "$1" ;;
|
|
esac
|
|
}
|
|
|
|
ssh_user() {
|
|
case "$1" in
|
|
188) printf 'ollama' ;;
|
|
*) printf 'wooo' ;;
|
|
esac
|
|
}
|
|
|
|
probe_host_tcp() {
|
|
local host="$1" ip
|
|
ip="$(host_ip "$host")"
|
|
if nc -z -w "$SSH_TIMEOUT_SECONDS" "$ip" 22 >/dev/null 2>&1; then
|
|
event "INFO" "host-${host}" "ssh tcp open ${ip}:22"
|
|
return 0
|
|
fi
|
|
finding "P0" "host-${host}" "HOST_SSH_PORT_DOWN" "${ip}:22 not reachable"
|
|
return 1
|
|
}
|
|
|
|
probe_host_command() {
|
|
local host="$1" ip user output_file
|
|
ip="$(host_ip "$host")"
|
|
user="$(ssh_user "$host")"
|
|
output_file="${ARTIFACT_DIR}/host-${host}-uptime.txt"
|
|
if timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'printf "host=%s\n" "$(hostname)"; uptime -s; uptime' >"$output_file" 2>&1; then
|
|
event "INFO" "host-${host}" "ssh command ok ${user}@${ip}"
|
|
return 0
|
|
fi
|
|
if grep -Eqi 'Permission denied|publickey|authentication' "$output_file"; then
|
|
finding "P0" "host-${host}" "SSH_AUTH_FAILED" "${user}@${ip}"
|
|
elif grep -Eqi 'timed out|Timeout|not responding' "$output_file"; then
|
|
finding "P0" "host-${host}" "SSH_COMMAND_SESSION_TIMEOUT" "${user}@${ip}"
|
|
else
|
|
finding "P0" "host-${host}" "SSH_COMMAND_FAILED" "${user}@${ip}; see ${output_file}"
|
|
fi
|
|
return 1
|
|
}
|
|
|
|
detect_recent_reboot() {
|
|
local host="$1" ip user output_file boot_epoch now_epoch age
|
|
ip="$(host_ip "$host")"
|
|
user="$(ssh_user "$host")"
|
|
output_file="${ARTIFACT_DIR}/host-${host}-boot.txt"
|
|
if ! timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'date +%s; cat /proc/stat | awk "/btime/ {print \$2}"' >"$output_file" 2>&1; then
|
|
event "WARN" "host-${host}" "recent reboot detection unavailable"
|
|
return 0
|
|
fi
|
|
now_epoch="$(sed -n '1p' "$output_file")"
|
|
boot_epoch="$(sed -n '2p' "$output_file")"
|
|
if [[ "$now_epoch" =~ ^[0-9]+$ && "$boot_epoch" =~ ^[0-9]+$ ]]; then
|
|
age=$((now_epoch - boot_epoch))
|
|
printf '%s\t%s\t%s\n' "$host" "$boot_epoch" "$age" >>"${ARTIFACT_DIR}/boot-age.tsv"
|
|
if (( age <= RECOVERY_SLO_SECONDS )); then
|
|
event "INFO" "host-${host}" "recent reboot detected age_seconds=${age}"
|
|
else
|
|
event "INFO" "host-${host}" "host already past reboot window age_seconds=${age}"
|
|
fi
|
|
fi
|
|
}
|
|
|
|
probe_public_routes() {
|
|
local route status output_file safe_name
|
|
for route in $PUBLIC_ROUTES; do
|
|
safe_name="$(printf '%s' "$route" | sed 's#[^A-Za-z0-9._-]#_#g')"
|
|
output_file="${ARTIFACT_DIR}/route-${safe_name}.txt"
|
|
status="$(curl -k -L --max-time 12 -o "$output_file" -w '%{http_code}' "$route" 2>"${output_file}.err" || true)"
|
|
printf '%s\t%s\n' "$route" "$status" >>"${ARTIFACT_DIR}/public-routes.tsv"
|
|
case "$status" in
|
|
200|204|301|302|307|308|401)
|
|
event "INFO" "route" "${route} status=${status}"
|
|
;;
|
|
502|503|504)
|
|
finding "P0" "route" "PUBLIC_ROUTE_UPSTREAM_FAILED" "${route} status=${status}"
|
|
;;
|
|
000)
|
|
finding "P0" "route" "PUBLIC_ROUTE_UNREACHABLE" "${route} status=${status}"
|
|
;;
|
|
*)
|
|
finding "P1" "route" "PUBLIC_ROUTE_UNEXPECTED_STATUS" "${route} status=${status}"
|
|
;;
|
|
esac
|
|
done
|
|
}
|
|
|
|
run_ai_log_triage() {
|
|
local triage_dir="${ARTIFACT_DIR}/ai-log-triage"
|
|
local rc=0
|
|
if [[ ! -x "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" ]]; then
|
|
finding "P0" "recovery" "AI_LOG_TRIAGE_SCRIPT_MISSING" "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh"
|
|
return 1
|
|
fi
|
|
action "$MODE" "ai-log-triage" "RUN_AI_LOG_TRIAGE" "artifact_dir=${triage_dir}"
|
|
"${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" "--${MODE}" --artifact-dir "$triage_dir" || rc=$?
|
|
if [[ "$rc" -ne 0 && "$rc" -ne 2 ]]; then
|
|
finding "P0" "ai-log-triage" "AI_LOG_TRIAGE_FAILED" "see ${triage_dir}"
|
|
return 1
|
|
fi
|
|
if [[ "$rc" -eq 2 ]]; then
|
|
event "INFO" "ai-log-triage" "AI_LOG_TRIAGE_REPORTED_FINDINGS rc=2"
|
|
fi
|
|
if [[ -s "${triage_dir}/findings.tsv" ]]; then
|
|
awk -v now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -F '\t' \
|
|
'{print now "\t" $2 "\tai-log-triage\t" $1 "\thost=" $3 " evidence=" $4 " next_action=" $5}' \
|
|
"${triage_dir}/findings.tsv" >>"$FINDINGS_TSV" || true
|
|
fi
|
|
}
|
|
|
|
run_scorecard() {
|
|
local scorecard_file="${ARTIFACT_DIR}/scorecard.txt"
|
|
if [[ ! -f "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" ]]; then
|
|
finding "P0" "scorecard" "SCORECARD_SCRIPT_MISSING" "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py"
|
|
return 1
|
|
fi
|
|
action "check" "scorecard" "RUN_REBOOT_SCORECARD" "$scorecard_file"
|
|
python3 "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" >"$scorecard_file" 2>&1 || {
|
|
finding "P0" "scorecard" "SCORECARD_FAILED" "see ${scorecard_file}"
|
|
return 1
|
|
}
|
|
}
|
|
|
|
run_stock_upstream_gate_hint() {
|
|
local stock_gate="/Users/ogt/codex-workspaces/gitea-restored-worktrees-20260701/stockplatform-v2/scripts/ops/stockplatform-api-upstream-recovery-gate.sh"
|
|
if [[ ! -x "$stock_gate" ]]; then
|
|
event "WARN" "stockplatform" "stock upstream recovery gate not executable or missing: ${stock_gate}"
|
|
return 0
|
|
fi
|
|
action "check" "stockplatform" "VERIFY_STOCK_API_UPSTREAM_GATE" "$stock_gate"
|
|
STOCKPLATFORM_DIRECT_EDGE_URL="${STOCKPLATFORM_DIRECT_EDGE_URL:-http://127.0.0.1:31235}" \
|
|
STOCKPLATFORM_PUBLIC_WEB_URL="${STOCKPLATFORM_PUBLIC_WEB_URL:-https://stock.wooo.work}" \
|
|
"$stock_gate" --verify-only >"${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt" 2>&1 || {
|
|
finding "P0" "stockplatform" "STOCK_API_UPSTREAM_GATE_BLOCKED" "see ${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt"
|
|
}
|
|
}
|
|
|
|
write_summary() {
|
|
local p0_count p1_count elapsed status
|
|
elapsed=$(( $(date +%s) - START_EPOCH ))
|
|
p0_count="$(awk -F '\t' '$2=="P0"{c++} END{print c+0}' "$FINDINGS_TSV")"
|
|
p1_count="$(awk -F '\t' '$2=="P1"{c++} END{print c+0}' "$FINDINGS_TSV")"
|
|
if (( p0_count == 0 )); then
|
|
status="RECOVERY_READY_FOR_RUNTIME_CONFIRMATION"
|
|
else
|
|
status="RECOVERY_BLOCKED_P0"
|
|
fi
|
|
{
|
|
printf 'run_id=%s\n' "$RUN_ID"
|
|
printf 'mode=%s\n' "$MODE"
|
|
printf 'status=%s\n' "$status"
|
|
printf 'elapsed_seconds=%s\n' "$elapsed"
|
|
printf 'slo_seconds=%s\n' "$RECOVERY_SLO_SECONDS"
|
|
printf 'p0_count=%s\n' "$p0_count"
|
|
printf 'p1_count=%s\n' "$p1_count"
|
|
printf 'artifact_dir=%s\n' "$ARTIFACT_DIR"
|
|
printf '\n[P0 findings]\n'
|
|
awk -F '\t' '$2=="P0"{print}' "$FINDINGS_TSV" || true
|
|
printf '\n[Actions]\n'
|
|
cat "$ACTIONS_TSV"
|
|
} >"$SUMMARY_TXT"
|
|
cat "$SUMMARY_TXT"
|
|
}
|
|
|
|
START_EPOCH="$(date +%s)"
|
|
|
|
event "INFO" "orchestrator" "start mode=${MODE} run_id=${RUN_ID}"
|
|
notify "start" "running" "full-host reboot recovery started mode=${MODE}"
|
|
|
|
for host in $HOSTS; do
|
|
probe_host_tcp "$host" || true
|
|
done
|
|
|
|
for host in $HOSTS; do
|
|
probe_host_command "$host" || true
|
|
detect_recent_reboot "$host" || true
|
|
done
|
|
|
|
probe_public_routes
|
|
run_ai_log_triage || true
|
|
run_scorecard || true
|
|
run_stock_upstream_gate_hint || true
|
|
|
|
write_summary
|
|
|
|
if awk -F '\t' '$2=="P0"{found=1} END{exit found ? 0 : 1}' "$FINDINGS_TSV"; then
|
|
notify "finish" "blocked" "full-host reboot recovery has P0 blockers; artifact=${ARTIFACT_DIR}"
|
|
exit 1
|
|
fi
|
|
|
|
notify "finish" "ok" "full-host reboot recovery has no P0 blockers; artifact=${ARTIFACT_DIR}"
|
|
exit 0
|