fix(infra): 首席架構師 Review 修正 — C1/I1/I2/I3/I4/S1

C1: 移除 deploy-to-110.sh 密碼明文，改用 SSH key + sudoers NOPASSWD I1: 加入 /var/lock/harbor-repair.lock 防止 watchdog 與 startup 並行修復 I2: docker compose 的 stderr 不再靜默（改用 tee -a log | while read 輸出） I3: watchdog while loop 包在子 shell + || true，子 shell 異常不終止 watchdog I4: repair_harbor 關鍵指令（harbor-log 啟動）加入退出碼捕捉 S1: 修復後驗證等待從 5s/10s 改為 30s（harbor-core 初始化需要足夠時間） S2: docker ps 改用 --filter status=exited 取代 grep/awk Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
2026-04-05 12:18:41 +08:00
parent 66b12bf9eb
commit 3f7a742683
3 changed files with 104 additions and 68 deletions
--- a/scripts/reboot-recovery/awoooi-startup-110.sh
+++ b/scripts/reboot-recovery/awoooi-startup-110.sh
@@ -75,17 +75,26 @@ fi
 #   問題：第一次 up 時 harbor-core/db/portal 嘗試連 syslog:1514 失敗 exit，
 #         restart:always 重試直到放棄，第二次 compose up 無效
 #   新邏輯：只啟動 harbor-log → 等 healthy → 啟動其餘全部
+# 首席架構師 Review I1: 使用 lockfile 防止 watchdog 並行修復
 # ──────────────────────────────────────────────
 log "[3/5] 啟動 Harbor..."
 HARBOR_DIR="/home/wooo/harbor/harbor"
+HARBOR_LOCKFILE="/var/lock/harbor-repair.lock"
 if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then
    cd "$HARBOR_DIR"

+    # I1: 持有 lockfile，防止 harbor-watchdog 同時修復
+    exec 9>"$HARBOR_LOCKFILE"
+    flock -w 30 9 || { log "⚠️  無法取得 Harbor lock，跳過（watchdog 可能正在修復）"; return 0; }
+
    # Phase 1: 清除任何 Exited 狀態的 Harbor 容器（避免 restart:always 干擾）
-    HARBOR_EXITED=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "^harbor-" | grep "Exited" | awk '{print $1}')
+    # 使用 --filter 取代 grep/awk（更精確，不受欄位格式影響）
+    HARBOR_EXITED=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
    if [ -n "$HARBOR_EXITED" ]; then
-        log "清除 Exited Harbor 容器: $(echo $HARBOR_EXITED | tr '\n' ' ')"
-        echo "$HARBOR_EXITED" | xargs docker rm -f 2>/dev/null || true
+        log "Phase 1: 清除 Exited Harbor 容器: $(echo "$HARBOR_EXITED" | tr '\n' ' ')"
+        echo "$HARBOR_EXITED" | xargs docker rm -f 2>&1 | tail -3 || true
+    else
+        log "Phase 1: 無 Exited 容器"
    fi

    # Phase 2: 只啟動 harbor-log（syslog 接收端，其他容器的前置依賴）
@@ -109,18 +118,18 @@ if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then
        log "Phase 4: 啟動 Harbor 全組件..."
        docker compose up -d 2>&1 | tail -5

-        # 驗證
-        sleep 5
+        # S1: 等 30s 讓 harbor-core 完成初始化（之前 5s 不夠）
+        sleep 30
        # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗）
        HTTP=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0")
        if [ "$HTTP" = "401" ] || [ "$HTTP" = "200" ]; then
            log "✅ Harbor 就緒 (HTTP $HTTP)"
        else
-            log "⚠️  Harbor API 未就緒 (HTTP $HTTP)，通常需再等 10-20s"
+            log "⚠️  Harbor API 未就緒 (HTTP $HTTP)，harbor-watchdog 會在 60s 後自動修復"
        fi
    else
        log "❌ harbor-log 未 healthy (status=$STATUS)，跳過其餘 Harbor 容器"
-        log "   手動修復: cd $HARBOR_DIR && docker compose up -d"
+        log "   harbor-watchdog 會在 60s 後自動偵測並修復"
    fi
 else
    log "⚠️  找不到 Harbor compose 檔案"
--- a/scripts/reboot-recovery/deploy-to-110.sh
+++ b/scripts/reboot-recovery/deploy-to-110.sh
@@ -1,51 +1,50 @@
 #!/bin/bash
 # 將 awoooi-startup-110 + harbor-watchdog 部署到 192.168.0.110
 # 2026-04-05 Claude Code: 加入 harbor-watchdog（運行中崩潰自動恢復）
-# 執行方式: bash deploy-to-110.sh
+# 執行方式: bash scripts/reboot-recovery/deploy-to-110.sh
+#
+# 前提: 執行端已 ssh-copy-id wooo@192.168.0.110
+#       且 wooo 有 sudoers NOPASSWD 權限（feedback_cd_security_nopasswd.md）

 set -euo pipefail
 HOST="wooo@192.168.0.110"
-PASS="0936223270"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"

 echo "=== 部署 awoooi-startup-110 + harbor-watchdog 到 192.168.0.110 ==="

 # 1. 上傳腳本
 echo "[1/5] 上傳啟動腳本..."
-scp awoooi-startup-110.sh "$HOST:/tmp/awoooi-startup-110.sh"
-scp awoooi-startup-110.service "$HOST:/tmp/awoooi-startup-110.service"
+scp "$SCRIPT_DIR/awoooi-startup-110.sh" "$HOST:/tmp/awoooi-startup-110.sh"
+scp "$SCRIPT_DIR/awoooi-startup-110.service" "$HOST:/tmp/awoooi-startup-110.service"

 # 2. 上傳 watchdog
 echo "[2/5] 上傳 harbor-watchdog..."
-scp harbor-watchdog.sh "$HOST:/tmp/harbor-watchdog.sh"
-scp harbor-watchdog.service "$HOST:/tmp/harbor-watchdog.service"
+scp "$SCRIPT_DIR/harbor-watchdog.sh" "$HOST:/tmp/harbor-watchdog.sh"
+scp "$SCRIPT_DIR/harbor-watchdog.service" "$HOST:/tmp/harbor-watchdog.service"

 # 3. 安裝 startup service
 echo "[3/5] 安裝 startup service..."
-ssh "$HOST" "echo '$PASS' | sudo -S bash -c '
-    cp /tmp/awoooi-startup-110.sh /usr/local/bin/awoooi-startup-110.sh
-    chmod +x /usr/local/bin/awoooi-startup-110.sh
-    cp /tmp/awoooi-startup-110.service /etc/systemd/system/awoooi-startup-110.service
-    systemctl daemon-reload
-    systemctl enable awoooi-startup-110.service
-    echo startup OK
-'"
+ssh "$HOST" "sudo cp /tmp/awoooi-startup-110.sh /usr/local/bin/awoooi-startup-110.sh && \
+    sudo chmod +x /usr/local/bin/awoooi-startup-110.sh && \
+    sudo cp /tmp/awoooi-startup-110.service /etc/systemd/system/awoooi-startup-110.service && \
+    sudo systemctl daemon-reload && \
+    sudo systemctl enable awoooi-startup-110.service && \
+    echo 'startup OK'"

 # 4. 安裝 watchdog service（立即啟動）
 echo "[4/5] 安裝並啟動 harbor-watchdog..."
-ssh "$HOST" "echo '$PASS' | sudo -S bash -c '
-    cp /tmp/harbor-watchdog.sh /usr/local/bin/harbor-watchdog.sh
-    chmod +x /usr/local/bin/harbor-watchdog.sh
-    cp /tmp/harbor-watchdog.service /etc/systemd/system/harbor-watchdog.service
-    systemctl daemon-reload
-    systemctl enable harbor-watchdog.service
-    systemctl restart harbor-watchdog.service
-    echo watchdog OK
-'"
+ssh "$HOST" "sudo cp /tmp/harbor-watchdog.sh /usr/local/bin/harbor-watchdog.sh && \
+    sudo chmod +x /usr/local/bin/harbor-watchdog.sh && \
+    sudo cp /tmp/harbor-watchdog.service /etc/systemd/system/harbor-watchdog.service && \
+    sudo systemctl daemon-reload && \
+    sudo systemctl enable harbor-watchdog.service && \
+    sudo systemctl restart harbor-watchdog.service && \
+    echo 'watchdog OK'"

 # 5. 驗證
 echo "[5/5] 驗證..."
-ssh "$HOST" "echo '$PASS' | sudo -S systemctl is-active harbor-watchdog.service && echo '✅ harbor-watchdog active' || echo '❌ harbor-watchdog 失敗'"
-ssh "$HOST" "echo '$PASS' | sudo -S systemctl is-enabled awoooi-startup-110.service && echo '✅ startup-110 enabled' || echo '❌ startup-110 未啟用'"
+ssh "$HOST" "sudo systemctl is-active harbor-watchdog.service && echo '✅ harbor-watchdog active' || echo '❌ harbor-watchdog 失敗'"
+ssh "$HOST" "sudo systemctl is-enabled awoooi-startup-110.service && echo '✅ startup-110 enabled' || echo '❌ startup-110 未啟用'"

 echo ""
 echo "✅ 部署完成！"
@@ -54,4 +53,4 @@ echo "harbor-watchdog: 每 60 秒檢查 Harbor 健康，不健康自動修復"
 echo "startup-110:     下次重開機後自動恢復所有服務"
 echo ""
 echo "查看 watchdog 狀態:"
-echo "  journalctl -u harbor-watchdog.service -f"
+echo "  ssh $HOST 'journalctl -u harbor-watchdog.service -f'"
--- a/scripts/reboot-recovery/harbor-watchdog.sh
+++ b/scripts/reboot-recovery/harbor-watchdog.sh
@@ -1,52 +1,75 @@
 #!/bin/bash
 # Harbor Watchdog — 運行中崩潰自動恢復
-# 2026-04-05 Claude Code: 解決 Harbor 在非重開機情境下崩潰後無法自動恢復
+# 2026-04-05 Claude Code: 解決 Harbor Exited(128) 死鎖問題
+# 首席架構師 Review 2026-04-05: 修正 I1(lockfile) I2(stderr) I3(loop防護) I4(指令退出碼) S1(驗證等待)
 #
 # 問題根因：
-#   Docker restart:always 在 Exited (128) 情況下會重試直到 backoff 放棄
-#   因為 harbor-core/db 在啟動時嘗試連 syslog:1514 失敗就 exit，
-#   形成死鎖：harbor-log healthy → 但其他容器已進入 backoff 不再重試
+#   Docker restart:always 在 Exited(128) 情況下重試後進入 backoff 放棄
+#   harbor-core/db/portal 啟動時連 syslog:1514（harbor-log 未就緒）失敗 exit
 #
-# 解法：此 watchdog 每 60 秒檢查 Harbor API 健康，
-#   若不健康則執行完整的時序修復（harbor-log first 策略）
+# 設計：此 watchdog 每 60 秒輪詢 Harbor API，
+#   不健康 → 等 5s 確認 → 執行 4 Phase 修復（harbor-log first 策略）
+#   使用 lockfile 防止與 startup-110.sh 並行修復（I1）
 #
 # 部署位置: /usr/local/bin/harbor-watchdog.sh (on 192.168.0.110)
-# systemd unit: /etc/systemd/system/harbor-watchdog.service (Type=simple, loop)
+# systemd unit: /etc/systemd/system/harbor-watchdog.service

 HARBOR_DIR="/home/wooo/harbor/harbor"
 LOG="/var/log/harbor-watchdog.log"
+LOCKFILE="/var/lock/harbor-repair.lock"
 CHECK_INTERVAL=60  # 秒

+# 注意：不使用 set -e，watchdog 是長駐 loop，-e 會在任何指令失敗時終止腳本
+# 所有關鍵指令的退出碼需明確捕捉（I4）
+set -uo pipefail
+
 log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [watchdog] $*" | tee -a "$LOG"; }

 harbor_is_healthy() {
    local code
-    # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗）
-    code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0")
+    # 注意：Harbor /v2/ 正常回傳 401（需認證），不能用 -f（-f 把 401 視為失敗 exit 22）
+    # 使用 127.0.0.1 避免 IPv6 解析問題
+    code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "000")
    [ "$code" = "401" ] || [ "$code" = "200" ]
 }

 repair_harbor() {
-    log "⚠️  Harbor 不健康，開始修復程序..."
-    cd "$HARBOR_DIR" || { log "❌ 找不到 Harbor 目錄"; return 1; }
-
-    # Step 1: 清除所有 Exited Harbor 容器（打破 backoff 死鎖）
-    local exited
-    exited=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "^harbor-" | grep "Exited" | awk '{print $1}')
-    if [ -n "$exited" ]; then
-        log "清除 Exited 容器: $(echo "$exited" | tr '\n' ' ')"
-        echo "$exited" | xargs docker rm -f 2>/dev/null || true
+    # I1: lockfile 防止與 startup-110.sh 並行修復
+    exec 9>"$LOCKFILE"
+    if ! flock -n 9; then
+        log "另一修復程序正在進行，跳過本次修復"
+        return 0
    fi

-    # Step 2: 只啟動 harbor-log
-    log "啟動 harbor-log..."
-    docker compose up -d harbor-log 2>/dev/null
+    log "⚠️  開始修復 Harbor（4 Phase 策略）..."
+    cd "$HARBOR_DIR" || { log "❌ 找不到 Harbor 目錄: $HARBOR_DIR"; return 1; }

-    # Step 3: 等 harbor-log healthy（最多 90s）
+    # Phase 1: 清除所有 Exited Harbor 容器（打破 backoff 死鎖）
+    local exited
+    exited=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
+    if [ -n "$exited" ]; then
+        log "Phase 1: 清除 Exited 容器: $(echo "$exited" | tr '\n' ' ')"
+        echo "$exited" | xargs docker rm -f 2>&1 | while IFS= read -r line; do log "  $line"; done || true
+    else
+        log "Phase 1: 無 Exited 容器"
+    fi
+
+    # Phase 2: 只啟動 harbor-log（I2: 捕捉 stderr，I4: 確認退出碼）
+    log "Phase 2: 啟動 harbor-log..."
+    if ! docker compose up -d harbor-log 2>&1 | while IFS= read -r line; do log "  $line"; done; then
+        log "❌ harbor-log 啟動指令失敗"
+        return 1
+    fi
+
+    # Phase 3: 等 harbor-log healthy（最多 90s）
+    log "Phase 3: 等待 harbor-log healthy（最多 90s）..."
    local i status
    for i in $(seq 1 18); do
        status=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing")
-        [ "$status" = "healthy" ] && break
+        if [ "$status" = "healthy" ]; then
+            log "✅ harbor-log healthy (${i}x5s)"
+            break
+        fi
        sleep 5
    done

@@ -56,30 +79,35 @@ repair_harbor() {
        return 1
    fi

-    # Step 4: 啟動全部
-    log "harbor-log healthy，啟動全組件..."
-    docker compose up -d 2>/dev/null | tail -5
+    # Phase 4: 啟動全組件（I2: 捕捉 stderr）
+    log "Phase 4: 啟動 Harbor 全組件..."
+    docker compose up -d 2>&1 | while IFS= read -r line; do log "  $line"; done || true
+
+    # S1: 等 30s 讓 harbor-core 完成初始化（原 10s 不夠）
+    log "等待 30s 讓 harbor-core 完成初始化..."
+    sleep 30

-    sleep 10
    if harbor_is_healthy; then
        log "✅ Harbor 修復成功"
    else
        log "❌ Harbor 修復後仍不健康，需人工介入"
-        log "   手動: cd $HARBOR_DIR && docker compose down && docker compose up -d harbor-log && sleep 30 && docker compose up -d"
+        log "   手動: cd $HARBOR_DIR && docker compose down && docker compose up -d harbor-log && sleep 60 && docker compose up -d"
    fi
 }

 log "=== Harbor Watchdog 啟動 (interval=${CHECK_INTERVAL}s) ==="

+# I3: while loop 本體用 || true 保護，子 shell 異常不終止整個 watchdog
 while true; do
-    if harbor_is_healthy; then
-        : # 健康，靜默
-    else
-        # 等 5 秒再確認（避免短暫不穩定誤報）
-        sleep 5
+    (
        if ! harbor_is_healthy; then
-            repair_harbor
+            # 等 5 秒再確認（避免短暫不穩定誤報）
+            sleep 5
+            if ! harbor_is_healthy; then
+                repair_harbor
+            fi
        fi
-    fi
+    ) || log "⚠️  本次檢查週期發生未預期錯誤，繼續下一週期"
+
    sleep "$CHECK_INTERVAL"
 done