4af6487ba6
All checks were successful
CD Pipeline / workflow-shape (push) Successful in 2s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Successful in 2m31s
CD Pipeline / build-and-deploy (push) Successful in 15m18s
AWOOOI Harbor 110 Local Repair / workflow-shape (push) Successful in 1s
AWOOOI Harbor 110 Local Repair / harbor-110-local-repair (push) Successful in 22s
CD Pipeline / post-deploy-checks (push) Successful in 5m35s
155 lines
6.4 KiB
Bash
Executable File
155 lines
6.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Retire only the stale host110 Nginx listener that formerly proxied Ollama.
|
|
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="${TRACE_ID:?TRACE_ID is required}"
|
|
RUN_ID="${RUN_ID:?RUN_ID is required}"
|
|
WORK_ITEM_ID="${WORK_ITEM_ID:?WORK_ITEM_ID is required}"
|
|
TARGET_HOST="${TARGET_HOST:-wooo@192.168.0.110}"
|
|
MODE="${1:---check}"
|
|
|
|
case "${MODE}" in
|
|
apply|--check) ;;
|
|
*) printf 'usage: %s [--check|apply]\n' "$0" >&2; exit 64 ;;
|
|
esac
|
|
|
|
for value in "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"; do
|
|
case "${value}" in
|
|
*[!A-Za-z0-9._-]*|'')
|
|
printf 'trace/run/work item identifiers must be non-empty and shell-safe\n' >&2
|
|
exit 64
|
|
;;
|
|
esac
|
|
done
|
|
|
|
ssh -o BatchMode=yes -o ConnectTimeout=8 "${TARGET_HOST}" \
|
|
bash -s -- "${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" <<'REMOTE'
|
|
set -euo pipefail
|
|
|
|
TRACE_ID="$1"
|
|
RUN_ID="$2"
|
|
WORK_ITEM_ID="$3"
|
|
MODE="$4"
|
|
CONFIG_PATH="/etc/nginx/sites-enabled/110-ollama-proxy.conf"
|
|
BACKUP_PATH="/var/backups/awoooi/110-ollama-proxy.conf.${RUN_ID}"
|
|
LOCK_PATH="/tmp/awoooi-host110-ollama-proxy-retirement.lock"
|
|
APPLY_STARTED=0
|
|
VERIFIED=0
|
|
SOURCE_SHA=""
|
|
|
|
exec 9>"${LOCK_PATH}"
|
|
if ! flock -n 9; then
|
|
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=blocked_with_safe_next_action reason=retirement_apply_in_progress\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" >&2
|
|
exit 75
|
|
fi
|
|
|
|
rollback() {
|
|
local rollback_rc=0
|
|
if [ "${APPLY_STARTED}" -eq 1 ] && [ "${VERIFIED}" -eq 0 ]; then
|
|
sudo -n test -f "${BACKUP_PATH}" || rollback_rc=1
|
|
if [ "${rollback_rc}" -eq 0 ]; then
|
|
sudo -n install -o root -g root -m 0644 \
|
|
"${BACKUP_PATH}" "${CONFIG_PATH}" || rollback_rc=1
|
|
fi
|
|
if [ "${rollback_rc}" -eq 0 ]; then
|
|
sudo -n nginx -t >/dev/null 2>&1 || rollback_rc=1
|
|
sudo -n systemctl reload nginx || rollback_rc=1
|
|
fi
|
|
if [ "${rollback_rc}" -eq 0 ]; then
|
|
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=restored source_sha=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}"
|
|
return 0
|
|
fi
|
|
printf 'rollback_receipt trace_id=%s run_id=%s work_item_id=%s terminal=rollback_failed backup=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${BACKUP_PATH}" >&2
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
on_exit() {
|
|
local rc=$?
|
|
local rollback_rc=0
|
|
trap - EXIT
|
|
set +e
|
|
if [ "${rc}" -ne 0 ]; then
|
|
rollback || rollback_rc=$?
|
|
fi
|
|
if [ "${rollback_rc}" -ne 0 ]; then
|
|
exit 70
|
|
fi
|
|
exit "${rc}"
|
|
}
|
|
trap on_exit EXIT
|
|
|
|
sudo -n true
|
|
if ! sudo -n test -e "${CONFIG_PATH}"; then
|
|
test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
|
|
test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
|
|
sudo -n systemctl is-active --quiet nginx
|
|
sudo -n nginx -t >/dev/null 2>&1
|
|
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s state=already_retired listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass persistent_writes_performed=0\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}"
|
|
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed_already_converged_no_write\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
|
|
exit 0
|
|
fi
|
|
sudo -n test -f "${CONFIG_PATH}"
|
|
test "$(sudo -n stat -c '%F' "${CONFIG_PATH}")" = "regular file"
|
|
test "$(sudo -n stat -c '%U:%G:%a' "${CONFIG_PATH}")" = "root:root:644"
|
|
test "$(sudo -n grep -Ec '^[[:space:]]*listen[[:space:]]+.*11435' "${CONFIG_PATH}")" = "2"
|
|
PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+http://[^;]+:11434' "${CONFIG_PATH}" || true)"
|
|
ALL_PROXY_PASS_COUNT="$(sudo -n grep -Ec '^[[:space:]]*proxy_pass[[:space:]]+' "${CONFIG_PATH}" || true)"
|
|
test "${PROXY_PASS_COUNT}" -ge 1
|
|
test "${PROXY_PASS_COUNT}" = "${ALL_PROXY_PASS_COUNT}"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "2"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
|
|
test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
|
|
SOURCE_SHA="$(sudo -n sha256sum "${CONFIG_PATH}" | awk '{print $1}')"
|
|
sudo -n nginx -t >/dev/null 2>&1
|
|
|
|
printf 'check_receipt trace_id=%s run_id=%s work_item_id=%s mode=%s exact_path=%s source_sha=%s listener_11435=2 listener_11434=0 ollama_containers=0 nginx_test=pass persistent_writes_performed=0\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${MODE}" \
|
|
"${CONFIG_PATH}" "${SOURCE_SHA}"
|
|
|
|
if [ "${MODE}" = "--check" ]; then
|
|
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=check_pass_no_write\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
|
|
exit 0
|
|
fi
|
|
|
|
sudo -n install -d -o root -g root -m 0700 /var/backups/awoooi
|
|
sudo -n install -o root -g root -m 0600 "${CONFIG_PATH}" "${BACKUP_PATH}"
|
|
test "$(sudo -n sha256sum "${BACKUP_PATH}" | awk '{print $1}')" = "${SOURCE_SHA}"
|
|
APPLY_STARTED=1
|
|
sudo -n rm -- "${CONFIG_PATH}"
|
|
sudo -n nginx -t >/dev/null 2>&1
|
|
sudo -n systemctl reload nginx
|
|
|
|
for _ in $(seq 1 10); do
|
|
if [ "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0" ]; then
|
|
break
|
|
fi
|
|
sleep 1
|
|
done
|
|
|
|
sudo -n test ! -e "${CONFIG_PATH}"
|
|
test "$(sudo -n grep -RIl --include='*.conf' --include='nginx.conf' '11435' /etc/nginx 2>/dev/null | wc -l | tr -d ' ')" = "0"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11435' | wc -l | tr -d ' ')" = "0"
|
|
test "$(sudo -n ss -H -ltn 'sport = :11434' | wc -l | tr -d ' ')" = "0"
|
|
test "$(docker ps --format '{{.Names}}' | grep -Eic 'ollama' || true)" = "0"
|
|
sudo -n systemctl is-active --quiet nginx
|
|
sudo -n nginx -t >/dev/null 2>&1
|
|
VERIFIED=1
|
|
|
|
printf 'execution_receipt trace_id=%s run_id=%s work_item_id=%s action=retire_host110_ollama_proxy source_sha=%s backup=%s\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}" "${SOURCE_SHA}" "${BACKUP_PATH}"
|
|
printf 'post_verifier trace_id=%s run_id=%s work_item_id=%s listener_11435=0 listener_11434=0 ollama_containers=0 stale_config_paths=0 nginx_active=true nginx_test=pass\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
|
|
printf 'terminal trace_id=%s run_id=%s work_item_id=%s status=completed\n' \
|
|
"${TRACE_ID}" "${RUN_ID}" "${WORK_ITEM_ID}"
|
|
REMOTE
|