c9ef44e050
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 30s
CD Pipeline / build-and-deploy (push) Failing after 8s
CD Pipeline / post-deploy-checks (push) Has been skipped
263 lines
8.3 KiB
Bash
Executable File
263 lines
8.3 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Read-only scorecard for reboot recovery and backup DR readiness.
|
|
|
|
set -euo pipefail
|
|
|
|
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"
|
|
PROMETHEUS_URL="${PROMETHEUS_URL:-http://192.168.0.110:9090}"
|
|
ALERTMANAGER_URL="${ALERTMANAGER_URL:-http://192.168.0.110:9093}"
|
|
REGISTRY_HTTPS_URL="${REGISTRY_HTTPS_URL:-https://192.168.0.110:5000/v2/}"
|
|
REGISTRY_HTTP_URL="${REGISTRY_HTTP_URL:-http://192.168.0.110:5000/v2/}"
|
|
SSH_BATCH_MODE="${SSH_BATCH_MODE:-yes}"
|
|
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
|
|
REMOTE_COMMAND_TIMEOUT_SECONDS="${REMOTE_COMMAND_TIMEOUT_SECONDS:-45}"
|
|
REQUIRE_CORE=0
|
|
REQUIRE_DR=0
|
|
|
|
usage() {
|
|
cat <<'USAGE'
|
|
Usage: bash scripts/reboot-recovery/full-stack-recovery-scorecard.sh [--require-core] [--require-dr] [--require-all]
|
|
|
|
Read-only scorecard for reboot recovery and DR readiness.
|
|
|
|
Options:
|
|
--require-core Exit non-zero unless core cold-start recovery is ready.
|
|
--require-dr Exit non-zero unless rclone/offsite + escrow + full marker are ready.
|
|
--require-all Require both core and DR readiness.
|
|
|
|
Environment:
|
|
REMOTE_110, PROMETHEUS_URL, ALERTMANAGER_URL, SSH_BATCH_MODE,
|
|
SSH_STRICT_HOST_KEY_CHECKING.
|
|
USAGE
|
|
}
|
|
|
|
while [ "$#" -gt 0 ]; do
|
|
case "$1" in
|
|
--require-core)
|
|
REQUIRE_CORE=1
|
|
shift
|
|
;;
|
|
--require-dr)
|
|
REQUIRE_DR=1
|
|
shift
|
|
;;
|
|
--require-all)
|
|
REQUIRE_CORE=1
|
|
REQUIRE_DR=1
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
usage
|
|
exit 0
|
|
;;
|
|
*)
|
|
echo "Unknown argument: $1" >&2
|
|
usage >&2
|
|
exit 2
|
|
;;
|
|
esac
|
|
done
|
|
|
|
ssh_opts=(
|
|
-o BatchMode="$SSH_BATCH_MODE"
|
|
-o ConnectTimeout=6
|
|
-o ConnectionAttempts=1
|
|
-o ServerAliveInterval=5
|
|
-o ServerAliveCountMax=1
|
|
-o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING"
|
|
)
|
|
|
|
status_value() {
|
|
local key="$1"
|
|
local value="$2"
|
|
printf '%s=%s\n' "$key" "$value"
|
|
}
|
|
|
|
metric_value() {
|
|
local expr="$1"
|
|
PROMETHEUS_URL="$PROMETHEUS_URL" EXPR="$expr" python3 - <<'PY'
|
|
import json
|
|
import os
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
base = os.environ["PROMETHEUS_URL"].rstrip("/")
|
|
expr = os.environ["EXPR"]
|
|
url = base + "/api/v1/query?" + urllib.parse.urlencode({"query": expr})
|
|
payload = json.load(urllib.request.urlopen(url, timeout=8))
|
|
rows = payload.get("data", {}).get("result") or []
|
|
if not rows:
|
|
print("0")
|
|
else:
|
|
value = rows[0].get("value") or [0, "0"]
|
|
print(value[1])
|
|
PY
|
|
}
|
|
|
|
bool_metric() {
|
|
local expr="$1"
|
|
local value
|
|
value="$(metric_value "$expr" 2>/dev/null || echo 0)"
|
|
python3 - "$value" <<'PY'
|
|
import sys
|
|
try:
|
|
print(1 if float(sys.argv[1]) > 0 else 0)
|
|
except Exception:
|
|
print(0)
|
|
PY
|
|
}
|
|
|
|
http_code() {
|
|
local url="$1"
|
|
curl -s -o /dev/null -w "%{http_code}" --max-time 8 "$url" 2>/dev/null || true
|
|
}
|
|
|
|
http_code_insecure() {
|
|
local url="$1"
|
|
curl -ks -o /dev/null -w "%{http_code}" --max-time 8 "$url" 2>/dev/null || true
|
|
}
|
|
|
|
ready_http_code() {
|
|
local code="$1"
|
|
[ "$code" = "200" ] || [ "$code" = "401" ]
|
|
}
|
|
|
|
classify_readback_blocker() {
|
|
local path="$1"
|
|
if [ ! -s "$path" ]; then
|
|
echo "none"
|
|
return
|
|
fi
|
|
if grep -Eiq 'Connection .* timed out|Operation timed out|timed out' "$path"; then
|
|
echo "ssh_control_channel_timeout"
|
|
return
|
|
fi
|
|
if grep -Eiq 'Permission denied|publickey|Authentication failed' "$path"; then
|
|
echo "ssh_control_channel_auth_denied"
|
|
return
|
|
fi
|
|
echo "readback_command_failed"
|
|
}
|
|
|
|
remote_110_read() {
|
|
local command="$1"
|
|
local quoted_command=""
|
|
printf -v quoted_command '%q' "$command"
|
|
ssh "${ssh_opts[@]}" "$REMOTE_110" "if command -v timeout >/dev/null 2>&1; then timeout ${REMOTE_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi"
|
|
}
|
|
|
|
echo "AWOOOI full-stack recovery scorecard"
|
|
date '+%Y-%m-%d %H:%M:%S %Z'
|
|
echo
|
|
|
|
cold_green="$(bool_metric 'awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"}')"
|
|
cold_warn="$(metric_value 'awoooi_cold_start_warn_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
|
|
cold_blocked="$(metric_value 'awoooi_cold_start_blocked_gates{host="110",scope="110_120_121_188"}' 2>/dev/null || echo 999)"
|
|
cold_alerts="$(metric_value 'count(ALERTS{alertname=~"ColdStart.*",alertstate="firing"})' 2>/dev/null || echo 999)"
|
|
|
|
status_value CORE_COLD_START_GREEN "$cold_green"
|
|
status_value CORE_COLD_START_WARN_GATES "$cold_warn"
|
|
status_value CORE_COLD_START_BLOCKED_GATES "$cold_blocked"
|
|
status_value CORE_COLD_START_FIRING_ALERTS "$cold_alerts"
|
|
|
|
registry_https_code="$(http_code_insecure "$REGISTRY_HTTPS_URL")"
|
|
registry_http_code="$(http_code "$REGISTRY_HTTP_URL")"
|
|
status_value CORE_REGISTRY_HTTPS_CODE "${registry_https_code:-000}"
|
|
status_value CORE_REGISTRY_HTTP_CODE "${registry_http_code:-000}"
|
|
if ready_http_code "${registry_https_code:-000}" || ready_http_code "${registry_http_code:-000}"; then
|
|
status_value CORE_REGISTRY_READY 1
|
|
status_value CORE_REGISTRY_BLOCKER none
|
|
else
|
|
status_value CORE_REGISTRY_READY 0
|
|
status_value CORE_REGISTRY_BLOCKER registry_5000_unreachable_or_not_ready
|
|
fi
|
|
|
|
cold_start_parity_log="/tmp/awoooi-scorecard-cold-start-parity.log"
|
|
if bash "$ROOT_DIR/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh" >"$cold_start_parity_log" 2>&1; then
|
|
status_value CORE_COLD_START_DEPLOY_PARITY 1
|
|
status_value CORE_COLD_START_DEPLOY_PARITY_BLOCKER none
|
|
else
|
|
status_value CORE_COLD_START_DEPLOY_PARITY 0
|
|
status_value CORE_COLD_START_DEPLOY_PARITY_BLOCKER "$(classify_readback_blocker "$cold_start_parity_log")"
|
|
fi
|
|
|
|
if python3 "$ROOT_DIR/scripts/ops/backup-alert-live-visibility-check.py" \
|
|
--prometheus-url "$PROMETHEUS_URL" \
|
|
--alertmanager-url "$ALERTMANAGER_URL" \
|
|
>/tmp/awoooi-scorecard-backup-alert-visibility.log 2>&1; then
|
|
status_value BACKUP_GAP_ALERT_VISIBILITY 1
|
|
else
|
|
status_value BACKUP_GAP_ALERT_VISIBILITY 0
|
|
fi
|
|
|
|
offsite_evidence_err="/tmp/awoooi-scorecard-offsite-evidence.err"
|
|
evidence_report="$(remote_110_read '/backup/scripts/offsite-escrow-evidence-report.sh --no-color' 2>"$offsite_evidence_err" || true)"
|
|
if [ -n "$evidence_report" ]; then
|
|
status_value DR_OFFSITE_EVIDENCE_READBACK 1
|
|
status_value DR_OFFSITE_EVIDENCE_BLOCKER none
|
|
else
|
|
status_value DR_OFFSITE_EVIDENCE_READBACK 0
|
|
status_value DR_OFFSITE_EVIDENCE_BLOCKER "$(classify_readback_blocker "$offsite_evidence_err")"
|
|
fi
|
|
|
|
extract_report_value() {
|
|
local key="$1"
|
|
awk -F= -v key="$key" '$1 == key {print $2; found=1; exit} END {if (!found) print ""}' <<<"$evidence_report"
|
|
}
|
|
|
|
offsite_configured="$(extract_report_value OFFSITE_CONFIGURED)"
|
|
rclone_configured="$(extract_report_value RCLONE_CONFIGURED)"
|
|
b2_configured="$(extract_report_value B2_CONFIGURED)"
|
|
escrow_missing="$(extract_report_value ESCROW_MISSING_COUNT)"
|
|
partial_marker="$(extract_report_value PARTIAL_MARKER_PRESENT)"
|
|
full_marker="$(extract_report_value FULL_MARKER_PRESENT)"
|
|
next_step="$(extract_report_value NEXT_STEP)"
|
|
|
|
status_value OFFSITE_CONFIGURED "${offsite_configured:-${b2_configured:-unknown}}"
|
|
status_value OFFSITE_RCLONE_CONFIGURED "${rclone_configured:-unknown}"
|
|
status_value OFFSITE_B2_LEGACY_CONFIGURED "${b2_configured:-unknown}"
|
|
status_value OFFSITE_PARTIAL_MARKER_PRESENT "${partial_marker:-unknown}"
|
|
status_value OFFSITE_FULL_MARKER_PRESENT "${full_marker:-unknown}"
|
|
status_value ESCROW_MISSING_COUNT "${escrow_missing:-unknown}"
|
|
status_value NEXT_STEP "${next_step:-unknown}"
|
|
|
|
if [ "$cold_green" = "1" ] \
|
|
&& [ "${cold_warn%.*}" = "0" ] \
|
|
&& [ "${cold_blocked%.*}" = "0" ] \
|
|
&& [ "${cold_alerts%.*}" = "0" ] \
|
|
&& { ready_http_code "${registry_https_code:-000}" || ready_http_code "${registry_http_code:-000}"; }; then
|
|
core_state="CORE_READY"
|
|
else
|
|
core_state="CORE_NOT_READY"
|
|
fi
|
|
|
|
if [ "${offsite_configured:-${b2_configured:-0}}" = "1" ] \
|
|
&& [ "${escrow_missing:-999}" = "0" ] \
|
|
&& [ "${full_marker:-0}" = "1" ]; then
|
|
dr_state="DR_OFFSITE_READY"
|
|
else
|
|
dr_state="DR_OFFSITE_PENDING"
|
|
fi
|
|
|
|
status_value RECOVERY_STATE "${core_state}_${dr_state}"
|
|
|
|
echo
|
|
echo "Artifacts:"
|
|
echo "- /tmp/awoooi-scorecard-cold-start-parity.log"
|
|
echo "- /tmp/awoooi-scorecard-backup-alert-visibility.log"
|
|
echo "- /tmp/awoooi-scorecard-offsite-evidence.err"
|
|
|
|
exit_code=0
|
|
if [ "$REQUIRE_CORE" = "1" ] && [ "$core_state" != "CORE_READY" ]; then
|
|
echo "BLOCKED require-core failed: ${core_state}" >&2
|
|
exit_code=1
|
|
fi
|
|
|
|
if [ "$REQUIRE_DR" = "1" ] && [ "$dr_state" != "DR_OFFSITE_READY" ]; then
|
|
echo "BLOCKED require-dr failed: ${dr_state}; NEXT_STEP=${next_step:-unknown}" >&2
|
|
exit_code=1
|
|
fi
|
|
|
|
exit "$exit_code"
|