1431abba54
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / build-and-deploy (push) Has been cancelled
CD Pipeline / post-deploy-checks (push) Has been cancelled
CD Pipeline / tests (push) Has been cancelled
289 lines
11 KiB
Bash
Executable File
289 lines
11 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
MODE="check"
|
|
TRACE_ID=""
|
|
RUN_ID=""
|
|
WORK_ITEM_ID=""
|
|
|
|
HOST_ID="192.168.0.188"
|
|
N8N_IMAGE="harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5"
|
|
N8N_IMAGE_ID="sha256:19c1ad26c0c285fddd7fc1300ff670b7977afebf52a21de4dfb2a5a2b2a75e2b"
|
|
OPEN_WEBUI_IMAGE="harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915"
|
|
OPEN_WEBUI_IMAGE_ID="sha256:5bddf2d93bc307b99c1fde8b8d0c2dc3c563e0e24fc3eebb6488c539fa55d080"
|
|
BACKUP_IMAGE="harbor.wooo.work/dockerhub-cache/library/alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
|
|
N8N_DIR="/home/ollama/n8n"
|
|
N8N_OVERRIDE="${N8N_DIR}/docker-compose.agent99.yml"
|
|
OPEN_WEBUI_DIR="/home/ollama/open-webui"
|
|
OPEN_WEBUI_COMPOSE="${OPEN_WEBUI_DIR}/docker-compose.yml"
|
|
RECEIPT_ROOT="/home/ollama/agent99-runtime-recovery"
|
|
|
|
usage() {
|
|
printf 'usage: %s --check | --apply --trace-id ID --run-id ID --work-item-id ID\n' "$0" >&2
|
|
exit 64
|
|
}
|
|
|
|
valid_id() {
|
|
[[ "$1" =~ ^[A-Za-z0-9._:-]+$ ]]
|
|
}
|
|
|
|
while (($#)); do
|
|
case "$1" in
|
|
--check)
|
|
MODE="check"
|
|
shift
|
|
;;
|
|
--apply)
|
|
MODE="apply"
|
|
shift
|
|
;;
|
|
--trace-id)
|
|
(($# >= 2)) || usage
|
|
TRACE_ID="$2"
|
|
shift 2
|
|
;;
|
|
--run-id)
|
|
(($# >= 2)) || usage
|
|
RUN_ID="$2"
|
|
shift 2
|
|
;;
|
|
--work-item-id)
|
|
(($# >= 2)) || usage
|
|
WORK_ITEM_ID="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [[ "$MODE" == "apply" ]]; then
|
|
[[ -n "$TRACE_ID" && -n "$RUN_ID" && -n "$WORK_ITEM_ID" ]] || usage
|
|
valid_id "$TRACE_ID" && valid_id "$RUN_ID" && valid_id "$WORK_ITEM_ID" || usage
|
|
fi
|
|
|
|
bool_json() {
|
|
if [[ "$1" == "true" ]]; then printf 'true'; else printf 'false'; fi
|
|
}
|
|
|
|
http_status() {
|
|
local url="$1"
|
|
local resolve_arg="${2:-}"
|
|
local status
|
|
if [[ -n "$resolve_arg" ]]; then
|
|
status="$(curl -k -sS -L -o /dev/null -w '%{http_code}' --resolve "$resolve_arg" --max-time 12 "$url" 2>/dev/null || true)"
|
|
else
|
|
status="$(curl -sS -L -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
|
|
fi
|
|
[[ "$status" =~ ^[0-9]{3}$ ]] || status="000"
|
|
printf '%s' "$status"
|
|
}
|
|
|
|
non_5xx() {
|
|
local status="$1"
|
|
[[ "$status" =~ ^[0-9]{3}$ ]] && ((10#$status >= 200 && 10#$status < 500))
|
|
}
|
|
|
|
container_value() {
|
|
local name="$1"
|
|
local format="$2"
|
|
docker inspect "$name" --format "$format" 2>/dev/null || true
|
|
}
|
|
|
|
collect_state() {
|
|
local runtime_write="${1:-false}"
|
|
local terminal="${2:-observed}"
|
|
local receipt_path="${3:-}"
|
|
|
|
local n8n_status n8n_image n8n_restart n8n_restarts n8n_local n8n_public
|
|
local webui_status webui_health webui_image webui_restart webui_restarts webui_local webui_public
|
|
local n8n_config_ok=false webui_config_ok=false n8n_ok=false webui_ok=false overall=false
|
|
|
|
n8n_status="$(container_value n8n '{{.State.Status}}')"
|
|
n8n_image="$(container_value n8n '{{.Image}}')"
|
|
n8n_restart="$(container_value n8n '{{.HostConfig.RestartPolicy.Name}}')"
|
|
n8n_restarts="$(container_value n8n '{{.RestartCount}}')"
|
|
[[ "$n8n_restarts" =~ ^[0-9]+$ ]] || n8n_restarts=0
|
|
n8n_local="$(http_status 'http://127.0.0.1:5678/')"
|
|
n8n_public="$(http_status 'https://n8n.wooo.work/' 'n8n.wooo.work:443:127.0.0.1')"
|
|
|
|
webui_status="$(container_value open-webui '{{.State.Status}}')"
|
|
webui_health="$(container_value open-webui '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')"
|
|
webui_image="$(container_value open-webui '{{.Image}}')"
|
|
webui_restart="$(container_value open-webui '{{.HostConfig.RestartPolicy.Name}}')"
|
|
webui_restarts="$(container_value open-webui '{{.RestartCount}}')"
|
|
[[ "$webui_restarts" =~ ^[0-9]+$ ]] || webui_restarts=0
|
|
webui_local="$(http_status 'http://127.0.0.1:3010/')"
|
|
webui_public="$(http_status 'https://ollama.wooo.work/' 'ollama.wooo.work:443:127.0.0.1')"
|
|
|
|
if [[ -f "$N8N_OVERRIDE" ]] && grep -Fqx " image: ${N8N_IMAGE}" "$N8N_OVERRIDE"; then
|
|
n8n_config_ok=true
|
|
fi
|
|
if [[ -f "$OPEN_WEBUI_COMPOSE" ]] && grep -Fqx " image: ${OPEN_WEBUI_IMAGE}" "$OPEN_WEBUI_COMPOSE"; then
|
|
webui_config_ok=true
|
|
fi
|
|
|
|
if [[ "$n8n_config_ok" == true && "$n8n_status" == running && "$n8n_image" == "$N8N_IMAGE_ID" && "$n8n_restart" == always ]] && non_5xx "$n8n_local" && non_5xx "$n8n_public"; then
|
|
n8n_ok=true
|
|
fi
|
|
if [[ "$webui_config_ok" == true && "$webui_status" == running && "$webui_health" == healthy && "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" && "$webui_restart" == always ]] && non_5xx "$webui_local" && non_5xx "$webui_public"; then
|
|
webui_ok=true
|
|
fi
|
|
if [[ "$n8n_ok" == true && "$webui_ok" == true ]]; then
|
|
overall=true
|
|
fi
|
|
|
|
printf '{"schemaVersion":"agent99_host188_edge_services_v1","mode":"%s","traceId":"%s","runId":"%s","workItemId":"%s","host":"%s","overallHealthy":%s,"runtimeWritePerformed":%s,"secretValueRead":false,"rawDataRead":false,"terminal":"%s","receiptPath":"%s","services":[{"name":"n8n","publicUrl":"https://n8n.wooo.work","containerStatus":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s},{"name":"open-webui","publicUrl":"https://ollama.wooo.work","containerStatus":"%s","health":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s}],"rollback":{"automatic":false,"snapshotPreserved":true,"procedure":"stop_changed_containers_then_restore_run_scoped_snapshot_after_owner_break_glass"},"prohibitedActions":["host_reboot","vm_power_change","firewall_change","database_content_read","secret_read","external_registry_direct_pull"]}\n' \
|
|
"$MODE" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$HOST_ID" \
|
|
"$(bool_json "$overall")" "$(bool_json "$runtime_write")" "$terminal" "$receipt_path" \
|
|
"$n8n_status" "$(bool_json "$([[ "$n8n_image" == "$N8N_IMAGE_ID" ]] && echo true || echo false)")" "$n8n_restart" "$n8n_restarts" "$n8n_local" "$n8n_public" "$(bool_json "$n8n_config_ok")" "$(bool_json "$n8n_ok")" \
|
|
"$webui_status" "$webui_health" "$(bool_json "$([[ "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" ]] && echo true || echo false)")" "$webui_restart" "$webui_restarts" "$webui_local" "$webui_public" "$(bool_json "$webui_config_ok")" "$(bool_json "$webui_ok")"
|
|
|
|
[[ "$overall" == true ]]
|
|
}
|
|
|
|
if [[ "$MODE" == "check" ]]; then
|
|
if collect_state false observed ""; then
|
|
exit 0
|
|
fi
|
|
exit 2
|
|
fi
|
|
|
|
exec 9>/tmp/agent99-host188-edge-services.lock
|
|
if ! flock -w 30 9; then
|
|
collect_state false single_flight_lock_timeout "" || true
|
|
exit 75
|
|
fi
|
|
|
|
RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
|
|
OPERATION_LOG="${RECEIPT_DIR}/operation.log"
|
|
RECEIPT_PATH="${RECEIPT_DIR}/receipt.json"
|
|
|
|
emit_apply_failure() {
|
|
local exit_code="$1"
|
|
local result
|
|
trap - ERR
|
|
set +e
|
|
result="$(collect_state true "apply_failed_exit_${exit_code}" "$RECEIPT_PATH" || true)"
|
|
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
|
chmod 600 "$RECEIPT_PATH" 2>/dev/null || true
|
|
exit "$exit_code"
|
|
}
|
|
|
|
mkdir -p "$RECEIPT_DIR" "$N8N_DIR" "$OPEN_WEBUI_DIR"
|
|
chmod 700 "$RECEIPT_DIR"
|
|
: >"$OPERATION_LOG"
|
|
chmod 600 "$OPERATION_LOG"
|
|
trap 'emit_apply_failure $?' ERR
|
|
|
|
for image in "$BACKUP_IMAGE" "$N8N_IMAGE" "$OPEN_WEBUI_IMAGE"; do
|
|
timeout 1200 docker pull "$image" >>"$OPERATION_LOG" 2>&1
|
|
done
|
|
|
|
snapshot_with_helper() {
|
|
local source_path="$1"
|
|
local archive_name="$2"
|
|
if [[ -s "${RECEIPT_DIR}/${archive_name}" ]]; then
|
|
return 0
|
|
fi
|
|
docker run --rm --pull never --userns=host \
|
|
-v "${source_path}:/data:ro" -v "${RECEIPT_DIR}:/backup" \
|
|
"$BACKUP_IMAGE" sh -c "umask 077; tar czf /backup/${archive_name} -C /data ." \
|
|
>>"$OPERATION_LOG" 2>&1
|
|
}
|
|
|
|
[[ -d "${N8N_DIR}/data" && -f "${N8N_DIR}/docker-compose.yml" ]] || {
|
|
collect_state false n8n_base_assets_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
|
|
exit 3
|
|
}
|
|
|
|
if ! docker volume inspect open-webui >/dev/null 2>&1; then
|
|
docker volume create open-webui >>"$OPERATION_LOG"
|
|
fi
|
|
|
|
snapshot_with_helper "${N8N_DIR}/data" "n8n-data-before.tgz"
|
|
docker run --rm --pull never --userns=host \
|
|
-v open-webui:/data:ro -v "${RECEIPT_DIR}:/backup" \
|
|
"$BACKUP_IMAGE" sh -c 'umask 077; tar czf /backup/open-webui-volume-before.tgz -C /data .' \
|
|
>>"$OPERATION_LOG" 2>&1
|
|
docker run --rm --pull never --userns=host -v "${RECEIPT_DIR}:/backup" "$BACKUP_IMAGE" \
|
|
chown -R "$(id -u):$(id -g)" /backup >>"$OPERATION_LOG" 2>&1
|
|
chmod 600 "${RECEIPT_DIR}/n8n-data-before.tgz" "${RECEIPT_DIR}/open-webui-volume-before.tgz"
|
|
|
|
tmp_n8n="$(mktemp "${N8N_OVERRIDE}.XXXXXX")"
|
|
cat >"$tmp_n8n" <<EOF
|
|
services:
|
|
n8n:
|
|
image: ${N8N_IMAGE}
|
|
EOF
|
|
chmod 600 "$tmp_n8n"
|
|
mv "$tmp_n8n" "$N8N_OVERRIDE"
|
|
|
|
tmp_webui="$(mktemp "${OPEN_WEBUI_COMPOSE}.XXXXXX")"
|
|
cat >"$tmp_webui" <<EOF
|
|
services:
|
|
open-webui:
|
|
image: ${OPEN_WEBUI_IMAGE}
|
|
container_name: open-webui
|
|
restart: always
|
|
ports:
|
|
- "3010:8080"
|
|
extra_hosts:
|
|
- "host.docker.internal:host-gateway"
|
|
environment:
|
|
OLLAMA_BASE_URL: http://host.docker.internal:11434
|
|
OLLAMA_BASE_URLS: http://host.docker.internal:11434
|
|
volumes:
|
|
- open-webui:/app/backend/data
|
|
|
|
volumes:
|
|
open-webui:
|
|
external: true
|
|
EOF
|
|
chmod 600 "$tmp_webui"
|
|
mv "$tmp_webui" "$OPEN_WEBUI_COMPOSE"
|
|
|
|
remap_uid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subuid 2>/dev/null || true)"
|
|
remap_gid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid 2>/dev/null || true)"
|
|
if [[ "$remap_uid" =~ ^[0-9]+$ && "$remap_gid" =~ ^[0-9]+$ ]]; then
|
|
node_uid=$((remap_uid + 1000))
|
|
node_gid=$((remap_gid + 1000))
|
|
docker run --rm --pull never --userns=host -v "${N8N_DIR}/data:/data" "$BACKUP_IMAGE" \
|
|
chown -R "${node_uid}:${node_gid}" /data >>"$OPERATION_LOG" 2>&1
|
|
else
|
|
collect_state true docker_userns_mapping_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
|
|
exit 4
|
|
fi
|
|
|
|
docker compose --project-directory "$N8N_DIR" \
|
|
-f "${N8N_DIR}/docker-compose.yml" -f "$N8N_OVERRIDE" \
|
|
up -d --pull never >>"$OPERATION_LOG" 2>&1
|
|
webui_project="$(docker inspect open-webui --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)"
|
|
if [[ -n "$(docker ps -aq --filter name='^/open-webui$')" && "$webui_project" != "open-webui" ]]; then
|
|
docker rm -f open-webui >>"$OPERATION_LOG" 2>&1
|
|
fi
|
|
docker compose --project-directory "$OPEN_WEBUI_DIR" -f "$OPEN_WEBUI_COMPOSE" \
|
|
up -d --pull never >>"$OPERATION_LOG" 2>&1
|
|
|
|
deadline=$((SECONDS + 360))
|
|
while ((SECONDS < deadline)); do
|
|
if collect_state true verifying "$RECEIPT_PATH" >/dev/null; then
|
|
break
|
|
fi
|
|
sleep 5
|
|
done
|
|
|
|
if result="$(collect_state true deployed_verified "$RECEIPT_PATH")"; then
|
|
trap - ERR
|
|
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
|
chmod 600 "$RECEIPT_PATH"
|
|
exit 0
|
|
fi
|
|
|
|
result="$(collect_state true verifier_failed_snapshot_preserved "$RECEIPT_PATH" || true)"
|
|
trap - ERR
|
|
printf '%s\n' "$result" | tee "$RECEIPT_PATH"
|
|
chmod 600 "$RECEIPT_PATH"
|
|
exit 5
|