77 lines
3.1 KiB
Bash
Executable File
77 lines
3.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Restore the 188 MinIO endpoint used by Velero, then optionally create a
|
|
# one-off Velero backup to re-establish backup freshness evidence.
|
|
|
|
set -euo pipefail
|
|
|
|
MINIO_HOST="${MINIO_HOST:-ollama@192.168.0.188}"
|
|
K3S_HOST="${K3S_HOST:-wooo@192.168.0.120}"
|
|
BACKUP_HEALTH_HOST="${BACKUP_HEALTH_HOST:-wooo@192.168.0.110}"
|
|
MINIO_COMPOSE_FILE="${MINIO_COMPOSE_FILE:-/home/ollama/minio/docker-compose.yml}"
|
|
MINIO_OVERRIDE_FILE="${MINIO_OVERRIDE_FILE:-/home/ollama/minio/docker-compose.override.yml}"
|
|
VELERO_NAMESPACE="${VELERO_NAMESPACE:-velero}"
|
|
VELERO_TARGET_NAMESPACE="${VELERO_TARGET_NAMESPACE:-awoooi-prod}"
|
|
CREATE_VELERO_BACKUP="${CREATE_VELERO_BACKUP:-false}"
|
|
REFRESH_BACKUP_HEALTH="${REFRESH_BACKUP_HEALTH:-false}"
|
|
BACKUP_NAME="${VELERO_BACKUP_NAME:-reboot-recovery-$(date -u +%Y%m%d%H%M)}"
|
|
|
|
ssh "$MINIO_HOST" "test -f '$MINIO_COMPOSE_FILE'"
|
|
|
|
ssh "$MINIO_HOST" "cat > '$MINIO_OVERRIDE_FILE' <<'EOF'
|
|
services:
|
|
minio:
|
|
userns_mode: host
|
|
EOF"
|
|
|
|
ssh "$MINIO_HOST" "docker compose -f '$MINIO_COMPOSE_FILE' -f '$MINIO_OVERRIDE_FILE' up -d"
|
|
|
|
ssh "$MINIO_HOST" "for i in \$(seq 1 30); do curl -fsS --max-time 3 http://127.0.0.1:9000/minio/health/live >/dev/null && exit 0; sleep 2; done; docker logs --tail=80 minio >&2; exit 1"
|
|
echo "MINIO_188_HEALTHY endpoint=192.168.0.188:9000"
|
|
|
|
ssh "$K3S_HOST" "sudo -n k3s kubectl -n '$VELERO_NAMESPACE' get backupstoragelocations.velero.io default -o jsonpath='{.status.phase}'" | grep -qx Available
|
|
echo "VELERO_BACKUP_STORAGE_LOCATION_AVAILABLE namespace=$VELERO_NAMESPACE"
|
|
|
|
if [ "$CREATE_VELERO_BACKUP" = "true" ]; then
|
|
ssh "$K3S_HOST" "printf '%s\n' \
|
|
'apiVersion: velero.io/v1' \
|
|
'kind: Backup' \
|
|
'metadata:' \
|
|
' name: $BACKUP_NAME' \
|
|
' namespace: $VELERO_NAMESPACE' \
|
|
' labels:' \
|
|
' awoooi.wooo.work/source: reboot-recovery' \
|
|
'spec:' \
|
|
' includedNamespaces:' \
|
|
' - $VELERO_TARGET_NAMESPACE' \
|
|
' storageLocation: default' \
|
|
' ttl: 720h0m0s' \
|
|
| sudo -n k3s kubectl apply -f -"
|
|
|
|
for _ in $(seq 1 60); do
|
|
phase="$(ssh "$K3S_HOST" "sudo -n k3s kubectl -n '$VELERO_NAMESPACE' get backup '$BACKUP_NAME' -o jsonpath='{.status.phase}'" || true)"
|
|
case "$phase" in
|
|
Completed)
|
|
echo "VELERO_BACKUP_COMPLETED name=$BACKUP_NAME"
|
|
break
|
|
;;
|
|
Failed|PartiallyFailed)
|
|
ssh "$K3S_HOST" "sudo -n k3s kubectl -n '$VELERO_NAMESPACE' get backup '$BACKUP_NAME' -o yaml" >&2 || true
|
|
echo "VELERO_BACKUP_FAILED name=$BACKUP_NAME phase=$phase" >&2
|
|
exit 1
|
|
;;
|
|
esac
|
|
sleep 5
|
|
done
|
|
|
|
phase="$(ssh "$K3S_HOST" "sudo -n k3s kubectl -n '$VELERO_NAMESPACE' get backup '$BACKUP_NAME' -o jsonpath='{.status.phase}'" || true)"
|
|
if [ "$phase" != "Completed" ]; then
|
|
echo "VELERO_BACKUP_TIMEOUT name=$BACKUP_NAME phase=${phase:-unknown}" >&2
|
|
exit 1
|
|
fi
|
|
fi
|
|
|
|
if [ "$REFRESH_BACKUP_HEALTH" = "true" ]; then
|
|
ssh "$BACKUP_HEALTH_HOST" "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin AIOPS_HOST_LABEL=110 NODE_EXPORTER_TEXTFILE_DIR=/home/wooo/node_exporter_textfiles /home/wooo/scripts/backup-health-textfile-exporter.py"
|
|
echo "BACKUP_HEALTH_TEXTFILE_REFRESHED host=110"
|
|
fi
|