#!/bin/bash
# 192.168.0.110 重開機自動恢復腳本
# 2026-04-05 ogt: 根據第二次重開機事故建立
# 部署位置: /usr/local/bin/awoooi-startup-110.sh (on 192.168.0.110)
# systemd unit: /etc/systemd/system/awoooi-startup-110.service
#
# 已知問題處理:
# - Docker BoltDB 損壞 (network/files/local-kv.db, volumes/metadata.db)
# - Docker container metadata JSON 損壞，dockerd 卡在 startup / loading containers
# - 舊容器使用已不存在的 Docker network (需要 docker rm -f 全部)
# - Harbor nginx 依賴 harbor-log (需要等 harbor-log healthy 後才 compose up)

set -uo pipefail
LOG="/var/log/awoooi-startup-110.log"
DOCKER_START_TIMEOUT_SECONDS="${AWOOOI_DOCKER_START_TIMEOUT_SECONDS:-45}"
DOCKER_ACTIVE_WAIT_ATTEMPTS="${AWOOOI_DOCKER_ACTIVE_WAIT_ATTEMPTS:-18}"
DOCKER_ACTIVE_WAIT_SLEEP_SECONDS="${AWOOOI_DOCKER_ACTIVE_WAIT_SLEEP_SECONDS:-5}"
exec > >(tee -a "$LOG") 2>&1

log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }

run_bounded() {
    local seconds="$1"
    shift
    if command -v timeout >/dev/null 2>&1; then
        timeout "$seconds" "$@"
    else
        "$@"
    fi
}

start_docker_bounded() {
    log "bounded start docker.service/docker.socket timeout=${DOCKER_START_TIMEOUT_SECONDS}s"
    run_bounded "$DOCKER_START_TIMEOUT_SECONDS" systemctl start docker.socket docker.service || true
}

wait_docker_active() {
    local i state
    for i in $(seq 1 "$DOCKER_ACTIVE_WAIT_ATTEMPTS"); do
        state=$(systemctl is-active docker 2>/dev/null || true)
        log "Docker wait ${i}/${DOCKER_ACTIVE_WAIT_ATTEMPTS}: ${state:-unknown}"
        [ "$state" = "active" ] && return 0
        sleep "$DOCKER_ACTIVE_WAIT_SLEEP_SECONDS"
    done
    return 1
}

stop_docker_bounded_for_metadata_recovery() {
    local state
    run_bounded 25 systemctl stop docker.service docker.socket || true
    state=$(systemctl is-active docker 2>/dev/null || true)
    if [ "$state" = "active" ] || [ "$state" = "activating" ] || [ "$state" = "deactivating" ]; then
        log "Docker stop still ${state}; TERM main process for metadata quarantine recovery"
        systemctl kill --kill-who=main --signal=TERM docker.service || true
        sleep 4
        run_bounded 20 systemctl stop docker.service docker.socket || true
    fi
}

DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT=0
quarantine_corrupt_docker_container_metadata() {
    local ids qdir id src count
    DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT=0
    ids=$(journalctl -u docker.service --since "-30 min" --no-pager 2>/dev/null \
        | awk '/failed to load container|invalid character|json: cannot unmarshal|unexpected EOF/ {
            line=$0
            while (match(line, /[0-9a-f]{64}/)) {
                print substr(line, RSTART, RLENGTH)
                line=substr(line, RSTART + RLENGTH)
            }
        }' \
        | sort -u)

    if [ -z "$ids" ]; then
        log "Docker corrupt container metadata: journal 未找到可 quarantine 的 container id"
        return 0
    fi

    qdir="/var/lib/docker/containers/.awoooi-corrupt-metadata-quarantine-$(date +%Y%m%d-%H%M%S)"
    mkdir -p "$qdir"
    count=0
    for id in $ids; do
        case "$id" in
            *[!0-9a-f]*|"")
                log "跳過不合法 container id: $id"
                continue
                ;;
        esac
        src="/var/lib/docker/containers/$id"
        if [ -d "$src" ]; then
            mv "$src" "$qdir/$id"
            count=$((count + 1))
            log "quarantine corrupt Docker container metadata: $id"
        else
            log "corrupt metadata candidate already absent: $id"
        fi
    done
    DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT="$count"
    log "Docker corrupt metadata quarantine dir=$qdir count=$count"
}

repair_ssh_control_path_metadata() {
    local script="${AWOOOI_110_SSH_REPAIR_SCRIPT:-/usr/local/bin/repair-110-ssh-publickey-auth-local.sh}"
    local target_user="${AWOOOI_110_SSH_REPAIR_USER:-wooo}"
    local reload_ssh="${AWOOOI_110_SSH_RELOAD_AFTER_REPAIR:-0}"

    log "[0/6] 修復 110 SSH control path metadata..."
    if [ ! -x "$script" ]; then
        log "⚠️  SSH control path metadata repair helper missing or not executable: $script"
        return 0
    fi

    if TARGET_USER="$target_user" RELOAD_SSH="$reload_ssh" "$script" --apply; then
        log "✅ 110 SSH control path metadata guard completed target_user=$target_user reload_ssh=$reload_ssh"
    else
        log "⚠️  110 SSH control path metadata guard failed; continuing startup"
    fi
}

log "=== 192.168.0.110 啟動序列開始 ==="
repair_ssh_control_path_metadata

# ──────────────────────────────────────────────
# STEP 1: Docker 修復（若 BoltDB 損壞）
# ──────────────────────────────────────────────
log "[1/5] 檢查 Docker..."

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 未啟動，嘗試啟動..."
    start_docker_bounded
    wait_docker_active || true
fi

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker bounded start 未 active，檢查 corrupt container metadata..."
    quarantine_corrupt_docker_container_metadata
    if [ "$DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT" -gt 0 ]; then
        stop_docker_bounded_for_metadata_recovery
        start_docker_bounded
        wait_docker_active || true
    fi
fi

if ! systemctl is-active docker >/dev/null 2>&1; then
    log "Docker 啟動失敗，修復 BoltDB..."
    # 清除所有已知 BoltDB 損壞點
    NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
    VOLUMES_DB="/var/lib/docker/volumes/metadata.db"

    if [ -f "$NETWORK_DB" ]; then
        cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$NETWORK_DB"
        log "清除損壞的 local-kv.db"
    fi
    if [ -f "$VOLUMES_DB" ]; then
        cp "$VOLUMES_DB" "${VOLUMES_DB}.bak.$(date +%Y%m%d%H%M%S)"
        rm -f "$VOLUMES_DB"
        log "清除損壞的 metadata.db"
    fi
    # 清除 buildkit 快取（也可能損壞）
    find /var/lib/docker/buildkit -name "*.db" -delete 2>/dev/null || true

    start_docker_bounded
    wait_docker_active || true
    systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
else
    log "✅ Docker 已 active"
fi

# ──────────────────────────────────────────────
# STEP 2: 清除孤兒容器（舊容器的 network 已不存在）
# ──────────────────────────────────────────────
log "[2/5] 清除孤兒容器..."
STALE=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "Exited (128)\|Exited (137)" | awk '{print $1}')
if [ -n "$STALE" ]; then
    log "發現孤兒容器: $(echo $STALE | tr '\n' ' ')"
    echo "$STALE" | xargs docker rm -f 2>/dev/null || true
    docker network prune -f 2>/dev/null || true
    log "✅ 孤兒容器清除完成"
else
    log "✅ 無孤兒容器"
fi

# ──────────────────────────────────────────────
# STEP 3: Harbor（嚴格時序：harbor-log first）
# 2026-04-05 Claude Code: 修復 Race Condition —
#   舊邏輯：compose up -d 全部 → 等 harbor-log healthy → compose up -d 全部
#   問題：第一次 up 時 harbor-core/db/portal 嘗試連 syslog:1514 失敗 exit，
#         restart:always 重試直到放棄，第二次 compose up 無效
#   新邏輯：只啟動 harbor-log → 等 healthy → 啟動其餘全部
# 首席架構師 Review I1: 使用 lockfile 防止 watchdog 並行修復
# ──────────────────────────────────────────────
log "[3/5] 啟動 Harbor..."
HARBOR_DIR="/home/wooo/harbor/harbor"
HARBOR_LOCKFILE="/var/lock/harbor-repair.lock"
if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then
    cd "$HARBOR_DIR"

    # I1: 持有 lockfile，防止 harbor-watchdog 同時修復
    exec 9>"$HARBOR_LOCKFILE"
    flock -w 30 9 || { log "⚠️  無法取得 Harbor lock，跳過（watchdog 可能正在修復）"; return 0; }

    # Phase 1: 清除任何 Exited 狀態的 Harbor 容器（避免 restart:always 干擾）
    # 使用 --filter 取代 grep/awk（更精確，不受欄位格式影響）
    HARBOR_EXITED=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
    if [ -n "$HARBOR_EXITED" ]; then
        log "Phase 1: 清除 Exited Harbor 容器: $(echo "$HARBOR_EXITED" | tr '\n' ' ')"
        echo "$HARBOR_EXITED" | xargs docker rm -f 2>&1 | tail -3 || true
    else
        log "Phase 1: 無 Exited 容器"
    fi

    # Phase 2: 只啟動 harbor-log（syslog 接收端，其他容器的前置依賴）
    log "Phase 2: 啟動 harbor-log..."
    docker compose up -d harbor-log 2>&1 | tail -3

    # Phase 3: 等待 harbor-log healthy（最多 90 秒）
    log "Phase 3: 等待 harbor-log healthy（最多 90s）..."
    for i in $(seq 1 18); do
        STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing")
        if [ "$STATUS" = "healthy" ]; then
            log "✅ harbor-log healthy (${i}x5s)"
            break
        fi
        sleep 5
    done

    STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "unknown")
    if [ "$STATUS" = "healthy" ]; then
        # Phase 4: harbor-log 就緒後才啟動其餘組件
        log "Phase 4: 啟動 Harbor 全組件..."
        docker compose up -d 2>&1 | tail -5

        # S1: 等 30s 讓 harbor-core 完成初始化（之前 5s 不夠）
        sleep 30
        # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗）
        HTTP=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0")
        if [ "$HTTP" = "401" ] || [ "$HTTP" = "200" ]; then
            log "✅ Harbor 就緒 (HTTP $HTTP)"
        else
            log "⚠️  Harbor API 未就緒 (HTTP $HTTP)，harbor-watchdog 會在 60s 後自動修復"
        fi
    else
        log "❌ harbor-log 未 healthy (status=$STATUS)，跳過其餘 Harbor 容器"
        log "   harbor-watchdog 會在 60s 後自動偵測並修復"
    fi
else
    log "⚠️  找不到 Harbor compose 檔案"
fi

# ──────────────────────────────────────────────
# STEP 4: 其他服務（Gitea, Langfuse, Monitoring）
# ──────────────────────────────────────────────
log "[4/6] 啟動其他服務（Gitea, Langfuse, Monitoring）..."

GITEA_DIR="/home/wooo/gitea"
if [ -f "$GITEA_DIR/docker-compose.yml" ]; then
    cd "$GITEA_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Gitea 啟動指令已發送"
fi

LANGFUSE_DIR="/home/wooo/langfuse"
if [ -f "$LANGFUSE_DIR/docker-compose.yml" ]; then
    cd "$LANGFUSE_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Langfuse 啟動指令已發送"
fi

MONITORING_DIR="/home/wooo/monitoring"
if [ -f "$MONITORING_DIR/docker-compose.yml" ]; then
    cd "$MONITORING_DIR"
    docker compose up -d 2>&1 | tail -3
    log "✅ Monitoring (Prometheus/Grafana/Alertmanager) 啟動指令已發送"
    sleep 10
    # 驗證 Alertmanager 是否啟動
    if curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1; then
        log "✅ Alertmanager healthy"
    else
        log "⚠️  Alertmanager 尚未就緒，等待 20 秒..."
        sleep 20
        curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1 && log "✅ Alertmanager 就緒" || log "❌ Alertmanager 未就緒，需手動檢查"
    fi
fi

# ──────────────────────────────────────────────
# STEP 5: SignOz
# ──────────────────────────────────────────────
log "[5/6] 啟動 SignOz..."
SIGNOZ_DIR="/home/wooo/signoz/deploy/docker"
if [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
    cd "$SIGNOZ_DIR"
    docker compose up -d 2>&1 | tail -5
    log "✅ SignOz 啟動指令已發送"
fi

# ──────────────────────────────────────────────
# STEP 6: Gitea Act Runner（CI/CD 核心）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Gitea runner 離線、CD 失效
# 2026-06-27 Codex: 110 runner labels 收斂，避免接泛用 shared CI。
# 2026-06-27 Codex: 110 是 production / registry / observability 主機；
# legacy runner 預設維持停用降壓；controlled drain lane 可在受控授權下啟動。
# ──────────────────────────────────────────────
log "[6/6] 檢查 Gitea Act Runner（legacy fail-closed；controlled drain 守門自動啟動）..."
RUNNER_DIR="/home/wooo/act-runner"
RUNNER_SERVICE="gitea-act-runner-host.service"
RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled"
CD_LANE_DIR="/home/wooo/awoooi-cd-lane"
CD_LANE_SERVICE="awoooi-cd-lane.service"
CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane"
CD_LANE_CONFIG="$CD_LANE_DIR/config.yaml"
CD_LANE_DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain"
CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service"
CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml"
CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled"
START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-1}"
START_GITEA_RUNNER_ALLOWED=0
START_CD_LANE_ALLOWED=0
RUNNER_FAIL_CLOSED_SERVICES=(
    "awoooi-cd-lane.service"
    "awoooi-direct-runner-open.service"
    "awoooi-direct-runner.service"
    "gitea-act-runner-host.service"
    "gitea-act-runner-awoooi-controlled.service"
    "gitea-awoooi-controlled-runner.service"
    "gitea-act-runner-awoooi-open.service"
)
RUNNER_FAIL_CLOSED_BINARY_PATHS=(
    "/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
    "/home/wooo/act-runner/act_runner"
    "/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
    "/home/wooo/act-runner-controlled/act_runner"
    "/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
)
# Host runner still needs both keys. The direct cd-lane stays fail-closed until
# it is migrated or hard-limited outside this production host pressure lane.
# The drain lane is a narrow post-reboot recovery path; it auto-starts only when
# config/binary/label/restore-source guardrails pass, and can be explicitly
# disabled with AWOOOI_START_CONTROLLED_CD_LANE=0.
if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then
    START_GITEA_RUNNER_ALLOWED=1
fi

mask_runner_unit_file() {
    local unit="$1"
    local unit_dir="$2"
    local owner_user="${3:-}"
    local unit_file="$unit_dir/$unit"
    local quarantine_stamp
    quarantine_stamp="$(date +%Y%m%d%H%M%S)"

    mkdir -p "$unit_dir" >/dev/null 2>&1 || true
    if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
        return 0
    fi
    if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
        chattr -i "$unit_file" >/dev/null 2>&1 || true
        mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
    fi
    ln -s /dev/null "$unit_file" >/dev/null 2>&1 || true
    if [ -n "$owner_user" ]; then
        chown -h "$owner_user:$owner_user" "$unit_file" >/dev/null 2>&1 || true
    fi
}

guard_runner_binary_fail_closed() {
    local path="$1"
    local tmp
    local quarantine_stamp
    quarantine_stamp="$(date +%Y%m%d%H%M%S)"

    if [ -e "$path" ]; then
        chattr -i "$path" >/dev/null 2>&1 || true
        if file "$path" 2>/dev/null | grep -qi "ELF"; then
            mv "$path" "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
            chmod 0400 "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
            chattr +i "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
        fi
    fi

    tmp="$(mktemp)"
    cat >"$tmp" <<'EOF'
#!/usr/bin/env bash
set -eu
echo "AWOOOI host runner is fail-closed on 110 after 2026-06-28 pressure incident; migrate or rate-limit before enabling." >&2
exit 75
EOF
    install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
    rm -f "$tmp"
    chattr +i "$path" >/dev/null 2>&1 || true
}

install_cd_lane_fail_closed_unit() {
    local unit_file="/etc/systemd/system/awoooi-cd-lane.service"
    local quarantine_stamp
    quarantine_stamp="$(date +%Y%m%d%H%M%S)"

    systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true
    if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
        chattr -i "$unit_file" >/dev/null 2>&1 || true
        if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then
            mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
        fi
    fi
    ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true
}

install_controlled_cd_lane_unit() {
    local unit_file="/etc/systemd/system/$CD_LANE_SERVICE"
    local tmp
    chattr -i "$unit_file" "$CD_LANE_BINARY" >/dev/null 2>&1 || true
    tmp="$(mktemp)"
    cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service

[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_BINARY} daemon --config ${CD_LANE_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100

[Install]
WantedBy=multi-user.target
EOF
    install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
    rm -f "$tmp"
}

install_controlled_cd_lane_drain_unit() {
    local unit_file="/etc/systemd/system/$CD_LANE_DRAIN_SERVICE"
    local tmp
    chattr -i "$unit_file" "$CD_LANE_DRAIN_BINARY" >/dev/null 2>&1 || true
    if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
        rm -f "$unit_file" >/dev/null 2>&1 || true
    fi
    tmp="$(mktemp)"
    cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane drain bypass for old queued guards
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
ConditionPathExists=${CD_LANE_DRAIN_DIR}/data/.runner

[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DRAIN_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_DRAIN_BINARY} daemon --config ${CD_LANE_DRAIN_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100

[Install]
WantedBy=multi-user.target
EOF
    install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
    rm -f "$tmp"
}

cd_lane_config_path_is_controlled() {
    local config_path="$1"
    [ -f "$config_path" ] || return 1
    grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
    grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
    grep -q 'awoooi-host:host' "$config_path" || return 1
    if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
        return 1
    fi
    return 0
}

cd_lane_config_is_controlled() {
    cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
}

cd_lane_drain_config_is_controlled() {
    cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
}

cd_lane_drain_is_controlled_open() {
    local active
    active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
    [ "$active" = "active" ] || return 1
    cd_lane_drain_config_is_controlled || return 1
    file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
    return 0
}

cd_lane_drain_is_controlled_available() {
    cd_lane_drain_config_is_controlled || return 1
    file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
    return 0
}

cd_lane_root_restore_sources_left() {
    find /root -maxdepth 1 -type d \( \
        -name 'awoooi-cd-lane-disabled-*' -o \
        -name 'awoooi-cd-lane-drain-disabled-*' \
    \) -print 2>/dev/null | wc -l | tr -d " "
}

quarantine_cd_lane_registration_fail_closed() {
    local quarantine_dir
    local lane_dir
    local path
    local target

    rm -f /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open >/dev/null 2>&1 || true

    for lane_dir in "$CD_LANE_DIR" "$CD_LANE_DRAIN_DIR"; do
        [ -d "$lane_dir" ] || continue
        quarantine_dir="$lane_dir/quarantine-startup-$(date +%Y%m%d%H%M%S)"
        chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
        mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true
        while IFS= read -r -d '' path; do
            [ -e "$path" ] || continue
            chattr -i "$path" >/dev/null 2>&1 || true
            target="$quarantine_dir/$(basename "$path")"
            mv "$path" "$target" >/dev/null 2>&1 || true
            chmod 0400 "$target" >/dev/null 2>&1 || true
            chattr +i "$target" >/dev/null 2>&1 || true
        done < <(
            {
                find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
                find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
            } || true
        )
        chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
    done
}

quarantine_cd_lane_root_restore_sources_fail_closed() {
    local final_root
    local path
    local target_dir

    final_root="/root/awoooi-runner-restore-sources-disabled-final-$(date +%Y%m%dT%H%M%S%z)"
    target_dir="$final_root/cd-lane-restore-sources"
    mkdir -p "$target_dir" >/dev/null 2>&1 || true
    while IFS= read -r -d '' path; do
        [ -d "$path" ] || continue
        chattr -R -i "$path" >/dev/null 2>&1 || true
        mv "$path" "$target_dir/" >/dev/null 2>&1 || true
    done < <(
        {
            find /root -maxdepth 1 -type d -name 'awoooi-cd-lane-disabled-*' -print0 2>/dev/null
            find /root -maxdepth 1 -type d -name 'awoooi-cd-lane-drain-disabled-*' -print0 2>/dev/null
        } || true
    )
}

apply_cd_lane_fail_closed_guard() {
    local unit
    for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do
        systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
        systemctl stop "$unit" >/dev/null 2>&1 || true
        systemctl reset-failed "$unit" >/dev/null 2>&1 || true
        systemctl disable "$unit" >/dev/null 2>&1 || true
        if [ "$unit" = "awoooi-cd-lane.service" ]; then
            install_cd_lane_fail_closed_unit
        else
            systemctl mask "$unit" >/dev/null 2>&1 || mask_runner_unit_file "$unit" "/etc/systemd/system"
            mask_runner_unit_file "$unit" "/etc/systemd/system"
        fi
    done
    install_cd_lane_fail_closed_unit
    pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true
    pkill -KILL -f "^${CD_LANE_DRAIN_DIR}/awoooi_cd_lane_controlled daemon" >/dev/null 2>&1 || true
    quarantine_cd_lane_registration_fail_closed
    quarantine_cd_lane_root_restore_sources_fail_closed
    guard_runner_binary_fail_closed "$CD_LANE_DIR/awoooi_cd_lane"
    guard_runner_binary_fail_closed "$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
    systemctl daemon-reload >/dev/null 2>&1 || true
    systemctl reset-failed awoooi-cd-lane.service awoooi-cd-lane-drain.service >/dev/null 2>&1 || true
}

ensure_cd_lane_fail_closed() {
    apply_cd_lane_fail_closed_guard
}

ensure_controlled_cd_lane_open() {
    mkdir -p /run >/dev/null 2>&1 || true
    touch /run/awoooi-cd-lane-controlled-open /run/awoooi-cd-lane-drain-ok >/dev/null 2>&1 || true
    log "✅ controlled cd-lane startup override active; drain lane remains open"
}

ensure_host_runner_fail_closed() {
    local unit
    local binary
    local wooo_uid

    for unit in "${RUNNER_FAIL_CLOSED_SERVICES[@]}"; do
        systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
        systemctl reset-failed "$unit" >/dev/null 2>&1 || true
        systemctl disable "$unit" >/dev/null 2>&1 || true
        if [ "$unit" = "awoooi-cd-lane.service" ]; then
            install_cd_lane_fail_closed_unit
        else
            systemctl mask "$unit" >/dev/null 2>&1 || mask_runner_unit_file "$unit" "/etc/systemd/system"
            mask_runner_unit_file "$unit" "/etc/systemd/system"
        fi
    done
    systemctl daemon-reload >/dev/null 2>&1 || true

    if wooo_uid="$(id -u wooo 2>/dev/null)"; then
        mkdir -p /home/wooo/.config/systemd/user >/dev/null 2>&1 || true
        for unit in "${RUNNER_FAIL_CLOSED_SERVICES[@]}"; do
            if [ -d "/run/user/$wooo_uid" ] && command -v runuser >/dev/null 2>&1; then
                runuser -u wooo -- env XDG_RUNTIME_DIR="/run/user/$wooo_uid" systemctl --user kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
            fi
            mask_runner_unit_file "$unit" "/home/wooo/.config/systemd/user" "wooo"
        done
    fi

    pkill -KILL -f "^${RUNNER_DIR}/act_runner(\\.real-[^ ]*)? daemon" >/dev/null 2>&1 || true
    quarantine_cd_lane_root_restore_sources_fail_closed
    for binary in "${RUNNER_FAIL_CLOSED_BINARY_PATHS[@]}"; do
        if [ "$START_CD_LANE_ALLOWED" = "1" ] && [ "$binary" = "$CD_LANE_DRAIN_BINARY" ]; then
            continue
        fi
        guard_runner_binary_fail_closed "$binary"
    done
}

CD_LANE_ROOT_RESTORE_LEFT="$(cd_lane_root_restore_sources_left)"
if [ "$START_CONTROLLED_CD_LANE" = "1" ]; then
    if ! cd_lane_drain_is_controlled_available; then
        log "⛔ AWOOOI_START_CONTROLLED_CD_LANE=1 但 controlled drain lane config/binary 未通過；維持 fail-closed"
    elif [ "$CD_LANE_ROOT_RESTORE_LEFT" != "0" ]; then
        log "⛔ AWOOOI_START_CONTROLLED_CD_LANE=1 但 root restore-source left=${CD_LANE_ROOT_RESTORE_LEFT}；維持 fail-closed"
    else
        START_CD_LANE_ALLOWED=1
        log "✅ controlled cd-lane drain preflight passed; legacy runner 仍維持 fail-closed"
    fi
fi

if [ -x "$RUNNER_DIR/act_runner" ] && [ -f "$RUNNER_DIR/config.yaml" ]; then
    # 若舊的 .runner 配置指向過期 hostname，只有在明確允許啟動 runner
    # 時才清除重新註冊；預設降壓模式不得碰 registration 狀態。
    RUNNER_FILE="$RUNNER_DIR/data/.runner"
    if [ "$START_GITEA_RUNNER_ALLOWED" = "1" ] && [ -f "$RUNNER_FILE" ]; then
        OLD_URL=$(python3 -c "import json; d=json.load(open('$RUNNER_FILE')); print(d.get('address',''))" 2>/dev/null || echo "")
        if [ "$OLD_URL" != "http://192.168.0.110:3001" ]; then
            log "⚠️  runner 配置過期 ($OLD_URL)，清除重新註冊..."
            rm -f "$RUNNER_FILE" || true
        fi
    fi

    # act_runner 預設 shutdown_timeout=0s，會在 daemon 重啟時立刻取消
    # 正在執行的 CD job。這會造成部署實際完成但 Gitea status 失真。
    python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true
import sys
from pathlib import Path

path = Path(sys.argv[1])
lines = path.read_text().splitlines()
if any(line.strip().startswith("shutdown_timeout:") for line in lines):
    lines = [
        "  shutdown_timeout: 1h" if line.strip().startswith("shutdown_timeout:") else line
        for line in lines
    ]
else:
    for idx, line in enumerate(lines):
        if line.strip().startswith("timeout:") and idx > 0:
            lines.insert(idx + 1, "  shutdown_timeout: 1h")
            break
path.write_text("\n".join(lines) + "\n")
PY

    # 110 是 production / registry / observability 主機，不再接泛用
    # ubuntu-latest 類 job。泛用 label 會讓 StockPlatform 等其他 repo 的
    # build 跑到 110，造成 CPU 長時間尖峰。AWOOOI 非 host job 改用
    # awoooi-ubuntu，CD / post-deploy 保留 awoooi-host。
    python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true
import sys
from pathlib import Path

path = Path(sys.argv[1])
lines = path.read_text().splitlines()
output = []
idx = 0
while idx < len(lines):
    line = lines[idx]
    if line.strip() == "labels:":
        output.append(line)
        output.append('    - "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04"')
        output.append('    - "awoooi-host:host"')
        idx += 1
        while idx < len(lines) and (lines[idx].startswith("    - ") or not lines[idx].strip()):
            idx += 1
        continue
    output.append(line)
    idx += 1
path.write_text("\n".join(output) + "\n")
PY

    if [ "$START_GITEA_RUNNER_ALLOWED" = "1" ]; then
        if systemctl list-unit-files "$RUNNER_SERVICE" >/dev/null 2>&1; then
            systemctl enable --now "$RUNNER_SERVICE" >/dev/null 2>&1 || true
        elif ! pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
            nohup "$RUNNER_DIR/run-host-runner.sh" >> "$RUNNER_DIR/host-runner.log" 2>&1 &
        fi
    else
        if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ]; then
            log "⛔ AWOOOI_START_GITEA_RUNNER_ON_BOOT=1 但缺少 $RUNNER_ENABLE_SENTINEL；runner fail-closed"
        else
            log "⏸️  Gitea host runner 維持停用；需同時設定 AWOOOI_START_GITEA_RUNNER_ON_BOOT=1 與建立 $RUNNER_ENABLE_SENTINEL 才允許 startup 啟動"
        fi
        ensure_host_runner_fail_closed
    fi

    # 已停用 Docker-wrapped runner；避免它搶走 host label job。
    # 若手動執行此 recovery script 時仍有 task container，送 SIGINT
    # 讓 act_runner drain，不再接新 job，並等手上的 job 收尾。
    docker update --restart=no gitea-runner >/dev/null 2>&1 || true
    if docker ps --format '{{.Names}}' | grep -Eq '^(GITEA-ACTIONS-|[A-Za-z0-9][A-Za-z0-9_.-]*-(cd|code-review)-[0-9]+-)'; then
        log "⚠️  Gitea Actions task container still running; draining docker-wrapped gitea-runner"
        docker kill --signal=SIGINT gitea-runner >/dev/null 2>&1 || true
    else
        docker stop -t 3700 gitea-runner >/dev/null 2>&1 || true
    fi

    sleep 15

    # 驗證 runner 已連線 Gitea
    if pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
        log "⚠️  Gitea host act_runner 目前正在執行；請確認是否為受控限流 / 搬遷後狀態"
    elif [ "$START_GITEA_RUNNER_ALLOWED" = "1" ]; then
        log "⚠️  Gitea host act_runner 可能尚未啟動，查看: $RUNNER_DIR/host-runner.log"
    else
        log "✅ Gitea host act_runner 維持 inactive 降壓狀態"
    fi
else
    log "⚠️  找不到 act-runner binary/config: $RUNNER_DIR"
fi

if [ "$START_CD_LANE_ALLOWED" = "1" ]; then
    install_controlled_cd_lane_drain_unit
    systemctl daemon-reload >/dev/null 2>&1 || true
    systemctl unmask "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
    systemctl enable --now "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
    ensure_controlled_cd_lane_open
else
    log "✅ controlled cd-lane remains closed because guardrails failed or AWOOOI_START_CONTROLLED_CD_LANE=0"
fi

# ──────────────────────────────────────────────
# STEP 7: Sentry（Error Tracking）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Sentry 未自動啟動
# 安裝位置: /opt/sentry (2026-03-24 已安裝)
# DSN: awoooi-web :2, awoooi-api :3 (見 memory/project_sentry_full_integration.md)
#
# 已知重開機後損壞問題 (2026-04-05 事故記錄):
#   - sentry-postgres: WAL 損壞 → pg_resetwal -f
#   - sentry-redis: dump.rdb 損壞 → 刪除重建 (redis 是 session cache，可丟失)
#   - sentry-clickhouse: system table parts 損壞 → 刪除對應 store/ 目錄
# ──────────────────────────────────────────────
log "[7/7] 啟動 Sentry..."
SENTRY_DIR="/opt/sentry"
if [ -d "$SENTRY_DIR" ]; then
    cd "$SENTRY_DIR"

    # 修復 Sentry PostgreSQL WAL (若損壞)
    SENTRY_PG_VOL="sentry-postgres"
    PG_RUNNING=$(docker ps --filter name=sentry-self-hosted-postgres-1 --format "{{.Status}}" 2>/dev/null | head -1)
    if echo "$PG_RUNNING" | grep -q "Restarting"; then
        log "⚠️  sentry-postgres 損壞，嘗試 pg_resetwal 修復..."
        docker run --rm -u 999 -v ${SENTRY_PG_VOL}:/var/lib/postgresql/data \
            postgres:14 pg_resetwal -f /var/lib/postgresql/data 2>&1 | head -3 || true
        log "✅ sentry-postgres WAL 修復完成"
    fi

    # 修復 Sentry Redis (若 dump.rdb 損壞)
    REDIS_STATUS=$(docker inspect sentry-self-hosted-redis-1 --format='{{.State.Status}}' 2>/dev/null || echo "missing")
    if [ "$REDIS_STATUS" = "restarting" ]; then
        log "⚠️  sentry-redis 損壞，清除 dump.rdb..."
        docker run --rm --user root -v sentry-redis:/data alpine \
            sh -c 'rm -f /data/dump.rdb && echo cleared' 2>/dev/null || true
        log "✅ sentry-redis dump.rdb 已清除"
    fi

    docker compose up -d 2>&1 | tail -5
    log "✅ Sentry 啟動指令已發送 (啟動約需 2-3 分鐘)"
    sleep 20
    # 非阻塞驗證：Sentry 啟動慢，只做快速健康檢查
    HTTP_CODE=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" http://localhost:9000/ 2>/dev/null || echo "0")
    if echo "$HTTP_CODE" | grep -qE "^(200|302|400)$"; then
        log "✅ Sentry 已回應 (HTTP $HTTP_CODE)"
    else
        log "⚠️  Sentry 尚未就緒（正常現象，通常需 2-3 分鐘，HTTP=$HTTP_CODE）"
    fi
else
    log "⚠️  找不到 Sentry 目錄: $SENTRY_DIR"
fi

# ──────────────────────────────────────────────
# 完成
# ──────────────────────────────────────────────
log "=== 192.168.0.110 啟動序列完成 ==="
log "Harbor:       http://192.168.0.110:5000"
log "Gitea:        http://192.168.0.110:3001"
log "Grafana:      http://192.168.0.110:3002"
log "Alertmanager: http://192.168.0.110:9093"
log "Gitea Runner: systemctl --user status gitea-act-runner-host.service"
log "Sentry:       http://192.168.0.110:9000"

exit 0