awoooi/scripts/reboot-recovery/harbor-watchdog.sh

#!/bin/bash
# Harbor Watchdog — 運行中崩潰自動恢復
# 2026-04-05 Claude Code: 解決 Harbor Exited(128) 死鎖問題
# 首席架構師 Review 2026-04-05: 修正 I1(lockfile) I2(stderr) I3(loop防護) I4(指令退出碼) S1(驗證等待)
#
# 問題根因：
#   Docker restart:always 在 Exited(128) 情況下重試後進入 backoff 放棄
#   harbor-core/db/portal 啟動時連 syslog:1514（harbor-log 未就緒）失敗 exit
#
# 設計：此 watchdog 每 60 秒輪詢 Harbor API，
#   不健康 → 等 5s 確認 → 執行 4 Phase 修復（harbor-log first 策略）
#   使用 lockfile 防止與 startup-110.sh 並行修復（I1）
#
# 部署位置: /usr/local/bin/harbor-watchdog.sh (on 192.168.0.110)
# systemd unit: /etc/systemd/system/harbor-watchdog.service

HARBOR_DIR="/home/wooo/harbor/harbor"
LOG="/var/log/harbor-watchdog.log"
LOCKFILE="/var/lock/harbor-repair.lock"
CHECK_INTERVAL=60  # 秒
MODE="${1:---watch}"
EXPECTED_HOST_IP="${AWOOOI_HARBOR_EXPECTED_HOST_IP:-192.168.0.110}"

# 注意：不使用 set -e，watchdog 是長駐 loop，-e 會在任何指令失敗時終止腳本
# 所有關鍵指令的退出碼需明確捕捉（I4）
set -uo pipefail

log() {
    local line
    line="[$(date '+%Y-%m-%d %H:%M:%S')] [watchdog] $*"
    if { [ -e "$LOG" ] && [ -w "$LOG" ]; } || { [ ! -e "$LOG" ] && [ -w "$(dirname "$LOG")" ]; }; then
        echo "$line" | tee -a "$LOG"
    else
        echo "$line"
    fi
}

usage() {
    cat <<'EOF'
Usage: harbor-watchdog.sh [--watch|--check|--repair-once|--apply-once]

Modes:
  --watch        Default systemd watchdog loop.
  --check        Read-only Harbor readiness check; performs no Docker action.
  --repair-once  Run one bounded Harbor repair cycle on 192.168.0.110.
  --apply-once   Alias of --repair-once.

Safety:
  Repair modes fail closed unless the local host has 192.168.0.110, or
  AWOOOI_ALLOW_NON_110_HARBOR_REPAIR=1 is explicitly set.
EOF
}

probe_harbor_code() {
    local code
    # 注意：Harbor /v2/ 正常回傳 401（需認證），不能用 -f（-f 把 401 視為失敗 exit 22）
    # 使用 127.0.0.1 避免 IPv6 解析問題
    code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "000")
    if [ -z "$code" ]; then
        code="000"
    fi
    echo "$code"
}

harbor_is_healthy() {
    local code
    code="$(probe_harbor_code)"
    [ "$code" = "401" ] || [ "$code" = "200" ]
}

host_has_expected_ip() {
    command -v ip >/dev/null 2>&1 || return 1
    ip -o -4 addr show 2>/dev/null | grep -q " ${EXPECTED_HOST_IP}/"
}

require_expected_host_for_apply() {
    if [ "${AWOOOI_ALLOW_NON_110_HARBOR_REPAIR:-0}" = "1" ]; then
        log "AWOOOI_ALLOW_NON_110_HARBOR_REPAIR=1，略過 expected host guard"
        return 0
    fi
    if host_has_expected_ip; then
        return 0
    fi
    log "REFUSE harbor repair: expected_host_ip=${EXPECTED_HOST_IP} not present; run on 110 local console/root shell"
    return 1
}

check_harbor() {
    local code ready docker_cli docker_active dir_exists compose_exists lock_exists host_match
    code="$(probe_harbor_code)"
    ready=false
    if [ "$code" = "401" ] || [ "$code" = "200" ]; then
        ready=true
    fi
    docker_cli=false
    command -v docker >/dev/null 2>&1 && docker_cli=true
    docker_active="unknown"
    if command -v systemctl >/dev/null 2>&1; then
        docker_active="$(systemctl is-active docker 2>/dev/null || true)"
    fi
    dir_exists=false
    [ -d "$HARBOR_DIR" ] && dir_exists=true
    compose_exists=false
    [ -f "$HARBOR_DIR/docker-compose.yml" ] && compose_exists=true
    lock_exists=false
    [ -e "$LOCKFILE" ] && lock_exists=true
    host_match=false
    host_has_expected_ip && host_match=true

    cat <<EOF
AWOOOI_HARBOR_WATCHDOG_CHECK
mode=check
target=127.0.0.1:5000/v2/
expected_host_ip=${EXPECTED_HOST_IP}
expected_host_ip_present=${host_match}
harbor_dir=${HARBOR_DIR}
harbor_dir_exists=${dir_exists}
harbor_compose_exists=${compose_exists}
docker_cli_available=${docker_cli}
docker_service_state=${docker_active}
lockfile=${LOCKFILE}
lockfile_exists=${lock_exists}
harbor_local_v2_http_status=${code}
harbor_ready=${ready}
check_only=true
docker_compose_action_performed=false
container_remove_performed=false
service_restart_performed=false
host_reboot_performed=false
EOF
}

repair_harbor() {
    require_expected_host_for_apply || return 1

    # I1: lockfile 防止與 startup-110.sh 並行修復
    exec 9>"$LOCKFILE"
    if ! flock -n 9; then
        log "另一修復程序正在進行，跳過本次修復"
        return 0
    fi

    log "⚠️  開始修復 Harbor（4 Phase 策略）..."
    cd "$HARBOR_DIR" || { log "❌ 找不到 Harbor 目錄: $HARBOR_DIR"; return 1; }

    # Phase 1: 清除所有 Exited Harbor 容器（打破 backoff 死鎖）
    local exited
    exited=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
    if [ -n "$exited" ]; then
        log "Phase 1: 清除 Exited 容器: $(echo "$exited" | tr '\n' ' ')"
        echo "$exited" | xargs docker rm -f 2>&1 | while IFS= read -r line; do log "  $line"; done || true
    else
        log "Phase 1: 無 Exited 容器"
    fi

    # Phase 2: 只啟動 harbor-log（I2: 捕捉 stderr，I4: 確認退出碼）
    log "Phase 2: 啟動 harbor-log..."
    if ! docker compose up -d harbor-log 2>&1 | while IFS= read -r line; do log "  $line"; done; then
        log "❌ harbor-log 啟動指令失敗"
        return 1
    fi

    # Phase 3: 等 harbor-log healthy（最多 90s）
    log "Phase 3: 等待 harbor-log healthy（最多 90s）..."
    local i status
    for i in $(seq 1 18); do
        status=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing")
        if [ "$status" = "healthy" ]; then
            log "✅ harbor-log healthy (${i}x5s)"
            break
        fi
        sleep 5
    done

    status=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "unknown")
    if [ "$status" != "healthy" ]; then
        log "❌ harbor-log 未 healthy (status=$status)，放棄本次修復"
        return 1
    fi

    # Phase 4: 啟動全組件（I2: 捕捉 stderr）
    log "Phase 4: 啟動 Harbor 全組件..."
    if ! docker compose up -d 2>&1 | while IFS= read -r line; do log "  $line"; done; then
        log "❌ Harbor 全組件啟動指令失敗"
        return 1
    fi

    # S1: 等 30s 讓 harbor-core 完成初始化（原 10s 不夠）
    log "等待 30s 讓 harbor-core 完成初始化..."
    sleep 30

    if harbor_is_healthy; then
        log "✅ Harbor 修復成功"
        return 0
    else
        log "❌ Harbor 修復後仍不健康，需人工介入"
        log "   手動: cd $HARBOR_DIR && docker compose down && docker compose up -d harbor-log && sleep 60 && docker compose up -d"
        return 1
    fi
}

case "$MODE" in
    --check)
        check_harbor
        exit 0
        ;;
    --repair-once|--apply-once)
        log "=== Harbor Watchdog 單次修復開始 ==="
        if harbor_is_healthy; then
            log "Harbor 已健康，跳過修復"
            check_harbor
            exit 0
        fi
        repair_harbor
        rc=$?
        check_harbor
        exit "$rc"
        ;;
    --watch)
        ;;
    -h|--help)
        usage
        exit 0
        ;;
    *)
        usage
        exit 2
        ;;
esac

log "=== Harbor Watchdog 啟動 (interval=${CHECK_INTERVAL}s) ==="

# I3: while loop 本體用 || true 保護，子 shell 異常不終止整個 watchdog
while true; do
    (
        if ! harbor_is_healthy; then
            # 等 5 秒再確認（避免短暫不穩定誤報）
            sleep 5
            if ! harbor_is_healthy; then
                repair_harbor
            fi
        fi
    ) || log "⚠️  本次檢查週期發生未預期錯誤，繼續下一週期"

    sleep "$CHECK_INTERVAL"
done