227 lines
7.4 KiB
Bash
Executable File
227 lines
7.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
||
# 120 root filesystem 維護前只讀檢查。
|
||
# 本腳本不會 reboot、drain、cordon、fsck、刪檔或修改遠端狀態。
|
||
|
||
set -uo pipefail
|
||
|
||
REMOTE_120="${REMOTE_120:-wooo@192.168.0.120}"
|
||
REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"
|
||
SSH_BATCH_MODE="${SSH_BATCH_MODE:-yes}"
|
||
SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
|
||
NO_COLOR=0
|
||
|
||
usage() {
|
||
cat <<'USAGE'
|
||
Usage: bash scripts/reboot-recovery/120-fsck-maintenance-checklist.sh [--no-color]
|
||
|
||
Read-only pre-maintenance checklist for host 120 filesystem repair.
|
||
It prints evidence and manual console steps only; it never runs fsck online.
|
||
|
||
Environment:
|
||
REMOTE_120=wooo@192.168.0.120
|
||
REMOTE_110=wooo@192.168.0.110
|
||
SSH_BATCH_MODE=yes
|
||
SSH_STRICT_HOST_KEY_CHECKING=accept-new
|
||
USAGE
|
||
}
|
||
|
||
while [ "$#" -gt 0 ]; do
|
||
case "$1" in
|
||
--no-color)
|
||
NO_COLOR=1
|
||
shift
|
||
;;
|
||
-h|--help)
|
||
usage
|
||
exit 0
|
||
;;
|
||
*)
|
||
echo "Unknown argument: $1" >&2
|
||
usage >&2
|
||
exit 64
|
||
;;
|
||
esac
|
||
done
|
||
|
||
if [ "$NO_COLOR" = "1" ]; then
|
||
green=""
|
||
yellow=""
|
||
red=""
|
||
blue=""
|
||
reset=""
|
||
else
|
||
green="$(printf '\033[32m')"
|
||
yellow="$(printf '\033[33m')"
|
||
red="$(printf '\033[31m')"
|
||
blue="$(printf '\033[34m')"
|
||
reset="$(printf '\033[0m')"
|
||
fi
|
||
|
||
PASS=0
|
||
WARN=0
|
||
BLOCKED=0
|
||
|
||
ssh_opts=(-o BatchMode="$SSH_BATCH_MODE" -o ConnectTimeout=8 -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING")
|
||
|
||
section() {
|
||
printf "\n%s=== %s ===%s\n" "$blue" "$1" "$reset"
|
||
}
|
||
|
||
ok() {
|
||
PASS=$((PASS + 1))
|
||
printf "%sOK%s %s\n" "$green" "$reset" "$*"
|
||
}
|
||
|
||
warn() {
|
||
WARN=$((WARN + 1))
|
||
printf "%sWARN%s %s\n" "$yellow" "$reset" "$*"
|
||
}
|
||
|
||
blocked() {
|
||
BLOCKED=$((BLOCKED + 1))
|
||
printf "%sBLOCKED%s %s\n" "$red" "$reset" "$*"
|
||
}
|
||
|
||
ssh_cmd() {
|
||
local target="$1"
|
||
local command="$2"
|
||
ssh "${ssh_opts[@]}" "$target" "$command"
|
||
}
|
||
|
||
echo "AWOOOI 120 filesystem maintenance checklist"
|
||
date '+%Y-%m-%d %H:%M:%S %Z'
|
||
echo "Scope: 120 root LV fsck readiness. 112 Kali is intentionally skipped."
|
||
|
||
section "120 host state"
|
||
if out=$(ssh_cmd "$REMOTE_120" '
|
||
hostname
|
||
uptime
|
||
systemctl is-system-running || true
|
||
findmnt -n -o SOURCE,FSTYPE,OPTIONS /
|
||
test -r /proc/mounts && awk "\$2 == \"/\" {print \"ROOT_MOUNT_OPTIONS \" \$4}" /proc/mounts
|
||
' 2>&1); then
|
||
echo "$out"
|
||
ok "120 SSH and host state readable"
|
||
grep -q ' rw,' <<<"$out" && ok "120 root filesystem currently writable" || blocked "120 root filesystem is not confirmed writable"
|
||
else
|
||
blocked "120 host state unavailable"
|
||
echo "$out"
|
||
fi
|
||
|
||
section "K3s service and API state"
|
||
if out=$(ssh_cmd "$REMOTE_120" '
|
||
kcmd() {
|
||
sudo -n kubectl "$@"
|
||
}
|
||
echo "K3S_ACTIVE $(systemctl is-active k3s 2>/dev/null || true)"
|
||
echo "KEEPALIVED_ACTIVE $(systemctl is-active keepalived 2>/dev/null || true)"
|
||
kcmd get nodes -o wide
|
||
non_running="$(kcmd get pods -A --field-selector=status.phase!=Running,status.phase!=Succeeded --no-headers 2>/dev/null || true)"
|
||
echo "NON_RUNNING_PODS $(printf "%s\n" "$non_running" | awk "NF {count++} END {print count+0}")"
|
||
printf "%s\n" "$non_running"
|
||
kcmd get --raw /readyz >/dev/null && echo "READYZ ok" || echo "READYZ failed"
|
||
kcmd get --raw /livez >/dev/null && echo "LIVEZ ok" || echo "LIVEZ failed"
|
||
' 2>&1); then
|
||
echo "$out"
|
||
grep -q 'K3S_ACTIVE active' <<<"$out" && ok "120 k3s active" || blocked "120 k3s not active"
|
||
grep -q 'KEEPALIVED_ACTIVE active' <<<"$out" && ok "120 keepalived active" || warn "120 keepalived not active"
|
||
grep -q 'NON_RUNNING_PODS 0' <<<"$out" && ok "K3s has no non-running/non-succeeded pods" || warn "K3s has non-running/non-succeeded pods"
|
||
grep -q 'READYZ ok' <<<"$out" && ok "K3s readyz passed" || blocked "K3s readyz failed"
|
||
grep -q 'LIVEZ ok' <<<"$out" && ok "K3s livez passed" || blocked "K3s livez failed"
|
||
else
|
||
blocked "K3s API check unavailable"
|
||
echo "$out"
|
||
fi
|
||
|
||
section "120 filesystem blocker evidence"
|
||
if out=$(ssh_cmd "$REMOTE_120" '
|
||
kcmd() {
|
||
sudo -n kubectl "$@"
|
||
}
|
||
events="$(kcmd get events -A --field-selector involvedObject.kind=Node --sort-by=.lastTimestamp --no-headers 2>/dev/null | grep -Ei "EXT4-fs error|Buffer I/O error|I/O error|Structure needs cleaning|deleted inode" || true)"
|
||
echo "NODE_FS_ERROR_EVENTS $(printf "%s\n" "$events" | awk "NF {count++} END {print count+0}")"
|
||
printf "%s\n" "$events"
|
||
' 2>&1); then
|
||
echo "$out"
|
||
if grep -q 'NODE_FS_ERROR_EVENTS 0' <<<"$out"; then
|
||
ok "K3s Node filesystem error events absent"
|
||
else
|
||
blocked "120 still has K3s Node filesystem error events; do not declare reboot safe before offline fsck"
|
||
fi
|
||
else
|
||
blocked "120 filesystem event evidence unavailable"
|
||
echo "$out"
|
||
fi
|
||
|
||
section "Backup and restore evidence"
|
||
if out=$(ssh_cmd "$REMOTE_120" '
|
||
kcmd() {
|
||
sudo -n kubectl "$@"
|
||
}
|
||
kcmd get schedules,backups -n velero 2>/dev/null || true
|
||
' 2>&1); then
|
||
echo "$out"
|
||
grep -q 'schedule.velero.io/daily-awoooi-prod' <<<"$out" && ok "Velero daily schedule exists" || warn "Velero daily schedule not confirmed"
|
||
grep -Eq 'daily-awoooi-prod-[0-9]+' <<<"$out" && ok "Velero backup history visible" || warn "Velero backup history not confirmed"
|
||
else
|
||
warn "Velero evidence unavailable"
|
||
echo "$out"
|
||
fi
|
||
|
||
if out=$(ssh_cmd "$REMOTE_110" '
|
||
test -x /backup/scripts/offsite-escrow-evidence-report.sh
|
||
/backup/scripts/offsite-escrow-evidence-report.sh --no-color
|
||
' 2>&1); then
|
||
echo "$out"
|
||
grep -q 'FULL_MARKER_PRESENT=1' <<<"$out" && ok "110 offsite full marker present" || warn "110 offsite full marker not confirmed"
|
||
grep -q 'ESCROW_MISSING_COUNT=0' <<<"$out" && ok "credential escrow complete" || warn "credential escrow still has manual gaps"
|
||
else
|
||
warn "110 offsite/escrow evidence unavailable"
|
||
echo "$out"
|
||
fi
|
||
|
||
section "Public route smoke check"
|
||
route_fail=0
|
||
for domain in awoooi.wooo.work mo.wooo.work gitea.wooo.work harbor.wooo.work registry.wooo.work sentry.wooo.work signoz.wooo.work; do
|
||
code="$(curl -LsS -o /dev/null -w '%{http_code}' --max-time 12 "https://${domain}/" 2>/dev/null || echo 000)"
|
||
printf 'PUBLIC_ROUTE_TLS %s %s\n' "$domain" "$code"
|
||
case "$code" in
|
||
2??|3??|4??) ;;
|
||
*) route_fail=1 ;;
|
||
esac
|
||
done
|
||
if [ "$route_fail" -eq 0 ]; then
|
||
ok "public HTTPS routes respond with verified TLS"
|
||
else
|
||
blocked "one or more public HTTPS routes failed verified TLS check"
|
||
fi
|
||
|
||
section "Manual console-only fsck procedure"
|
||
cat <<'STEPS'
|
||
Do not run fsck against the mounted root filesystem.
|
||
|
||
Maintenance window sequence:
|
||
1. 確認本腳本的 Public route、Velero、offsite marker 與 K3s API 證據已保存。
|
||
2. 暫停非必要 deploy / runner / AI auto-repair full execution，只保留 observe-only 告警。
|
||
3. 透過主機 console、rescue mode 或 initramfs 停在 120，不要在線上 root mount 狀態執行 fsck。
|
||
4. 在 console/rescue 執行：
|
||
fsck -f /dev/mapper/ubuntu--vg-ubuntu--lv
|
||
5. 若 fsck 要求互動修復，逐項確認；完成後 reboot 120。
|
||
6. 回到 SSH 後執行：
|
||
SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color --watch --interval 10 --max-attempts 6
|
||
SSH_BATCH_MODE=yes bash scripts/reboot-recovery/reboot-recovery-readiness-audit.sh --live --no-color
|
||
7. 只有在 NODE_FS_ERROR_EVENTS=0、public TLS gate 通過、Prometheus scorecard core ready 後，才解除維護狀態。
|
||
STEPS
|
||
|
||
echo
|
||
echo "PASS=$PASS WARN=$WARN BLOCKED=$BLOCKED"
|
||
if [ "$BLOCKED" -gt 0 ]; then
|
||
echo "Result: MAINTENANCE REQUIRED. 120 filesystem risk is still blocking reboot confidence."
|
||
exit 1
|
||
fi
|
||
if [ "$WARN" -gt 0 ]; then
|
||
echo "Result: READY WITH WARNINGS for scheduled manual fsck."
|
||
exit 0
|
||
fi
|
||
echo "Result: READY for scheduled manual fsck."
|