From 3f7a7426836168a95634e865ca81d512439825f3 Mon Sep 17 00:00:00 2001 From: OG T Date: Sun, 5 Apr 2026 12:18:41 +0800 Subject: [PATCH] =?UTF-8?q?fix(infra):=20=E9=A6=96=E5=B8=AD=E6=9E=B6?= =?UTF-8?q?=E6=A7=8B=E5=B8=AB=20Review=20=E4=BF=AE=E6=AD=A3=20=E2=80=94=20?= =?UTF-8?q?C1/I1/I2/I3/I4/S1?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit C1: 移除 deploy-to-110.sh 密碼明文，改用 SSH key + sudoers NOPASSWD I1: 加入 /var/lock/harbor-repair.lock 防止 watchdog 與 startup 並行修復 I2: docker compose 的 stderr 不再靜默（改用 tee -a log | while read 輸出） I3: watchdog while loop 包在子 shell + || true，子 shell 異常不終止 watchdog I4: repair_harbor 關鍵指令（harbor-log 啟動）加入退出碼捕捉 S1: 修復後驗證等待從 5s/10s 改為 30s（harbor-core 初始化需要足夠時間） S2: docker ps 改用 --filter status=exited 取代 grep/awk Co-Authored-By: Claude Sonnet 4.6 --- scripts/reboot-recovery/awoooi-startup-110.sh | 23 +++-- scripts/reboot-recovery/deploy-to-110.sh | 51 +++++----- scripts/reboot-recovery/harbor-watchdog.sh | 98 ++++++++++++------- 3 files changed, 104 insertions(+), 68 deletions(-) diff --git a/scripts/reboot-recovery/awoooi-startup-110.sh b/scripts/reboot-recovery/awoooi-startup-110.sh index d8c936ba0..4dbd8878e 100644 --- a/scripts/reboot-recovery/awoooi-startup-110.sh +++ b/scripts/reboot-recovery/awoooi-startup-110.sh @@ -75,17 +75,26 @@ fi # 問題：第一次 up 時 harbor-core/db/portal 嘗試連 syslog:1514 失敗 exit， # restart:always 重試直到放棄，第二次 compose up 無效 # 新邏輯：只啟動 harbor-log → 等 healthy → 啟動其餘全部 +# 首席架構師 Review I1: 使用 lockfile 防止 watchdog 並行修復 # ────────────────────────────────────────────── log "[3/5] 啟動 Harbor..." HARBOR_DIR="/home/wooo/harbor/harbor" +HARBOR_LOCKFILE="/var/lock/harbor-repair.lock" if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then cd "$HARBOR_DIR" + # I1: 持有 lockfile，防止 harbor-watchdog 同時修復 + exec 9>"$HARBOR_LOCKFILE" + flock -w 30 9 || { log "⚠️ 無法取得 Harbor lock，跳過（watchdog 可能正在修復）"; return 0; } + # Phase 1: 清除任何 Exited 狀態的 Harbor 容器（避免 restart:always 干擾） - HARBOR_EXITED=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "^harbor-" | grep "Exited" | awk '{print $1}') + # 使用 --filter 取代 grep/awk（更精確，不受欄位格式影響） + HARBOR_EXITED=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true) if [ -n "$HARBOR_EXITED" ]; then - log "清除 Exited Harbor 容器: $(echo $HARBOR_EXITED | tr '\n' ' ')" - echo "$HARBOR_EXITED" | xargs docker rm -f 2>/dev/null || true + log "Phase 1: 清除 Exited Harbor 容器: $(echo "$HARBOR_EXITED" | tr '\n' ' ')" + echo "$HARBOR_EXITED" | xargs docker rm -f 2>&1 | tail -3 || true + else + log "Phase 1: 無 Exited 容器" fi # Phase 2: 只啟動 harbor-log（syslog 接收端，其他容器的前置依賴） @@ -109,18 +118,18 @@ if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then log "Phase 4: 啟動 Harbor 全組件..." docker compose up -d 2>&1 | tail -5 - # 驗證 - sleep 5 + # S1: 等 30s 讓 harbor-core 完成初始化（之前 5s 不夠） + sleep 30 # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗） HTTP=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0") if [ "$HTTP" = "401" ] || [ "$HTTP" = "200" ]; then log "✅ Harbor 就緒 (HTTP $HTTP)" else - log "⚠️ Harbor API 未就緒 (HTTP $HTTP)，通常需再等 10-20s" + log "⚠️ Harbor API 未就緒 (HTTP $HTTP)，harbor-watchdog 會在 60s 後自動修復" fi else log "❌ harbor-log 未 healthy (status=$STATUS)，跳過其餘 Harbor 容器" - log " 手動修復: cd $HARBOR_DIR && docker compose up -d" + log " harbor-watchdog 會在 60s 後自動偵測並修復" fi else log "⚠️ 找不到 Harbor compose 檔案" diff --git a/scripts/reboot-recovery/deploy-to-110.sh b/scripts/reboot-recovery/deploy-to-110.sh index c2cdd38ed..0d23949f6 100644 --- a/scripts/reboot-recovery/deploy-to-110.sh +++ b/scripts/reboot-recovery/deploy-to-110.sh @@ -1,51 +1,50 @@ #!/bin/bash # 將 awoooi-startup-110 + harbor-watchdog 部署到 192.168.0.110 # 2026-04-05 Claude Code: 加入 harbor-watchdog（運行中崩潰自動恢復） -# 執行方式: bash deploy-to-110.sh +# 執行方式: bash scripts/reboot-recovery/deploy-to-110.sh +# +# 前提: 執行端已 ssh-copy-id wooo@192.168.0.110 +# 且 wooo 有 sudoers NOPASSWD 權限（feedback_cd_security_nopasswd.md） set -euo pipefail HOST="wooo@192.168.0.110" -PASS="0936223270" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" echo "=== 部署 awoooi-startup-110 + harbor-watchdog 到 192.168.0.110 ===" # 1. 上傳腳本 echo "[1/5] 上傳啟動腳本..." -scp awoooi-startup-110.sh "$HOST:/tmp/awoooi-startup-110.sh" -scp awoooi-startup-110.service "$HOST:/tmp/awoooi-startup-110.service" +scp "$SCRIPT_DIR/awoooi-startup-110.sh" "$HOST:/tmp/awoooi-startup-110.sh" +scp "$SCRIPT_DIR/awoooi-startup-110.service" "$HOST:/tmp/awoooi-startup-110.service" # 2. 上傳 watchdog echo "[2/5] 上傳 harbor-watchdog..." -scp harbor-watchdog.sh "$HOST:/tmp/harbor-watchdog.sh" -scp harbor-watchdog.service "$HOST:/tmp/harbor-watchdog.service" +scp "$SCRIPT_DIR/harbor-watchdog.sh" "$HOST:/tmp/harbor-watchdog.sh" +scp "$SCRIPT_DIR/harbor-watchdog.service" "$HOST:/tmp/harbor-watchdog.service" # 3. 安裝 startup service echo "[3/5] 安裝 startup service..." -ssh "$HOST" "echo '$PASS' | sudo -S bash -c ' - cp /tmp/awoooi-startup-110.sh /usr/local/bin/awoooi-startup-110.sh - chmod +x /usr/local/bin/awoooi-startup-110.sh - cp /tmp/awoooi-startup-110.service /etc/systemd/system/awoooi-startup-110.service - systemctl daemon-reload - systemctl enable awoooi-startup-110.service - echo startup OK -'" +ssh "$HOST" "sudo cp /tmp/awoooi-startup-110.sh /usr/local/bin/awoooi-startup-110.sh && \ + sudo chmod +x /usr/local/bin/awoooi-startup-110.sh && \ + sudo cp /tmp/awoooi-startup-110.service /etc/systemd/system/awoooi-startup-110.service && \ + sudo systemctl daemon-reload && \ + sudo systemctl enable awoooi-startup-110.service && \ + echo 'startup OK'" # 4. 安裝 watchdog service（立即啟動） echo "[4/5] 安裝並啟動 harbor-watchdog..." -ssh "$HOST" "echo '$PASS' | sudo -S bash -c ' - cp /tmp/harbor-watchdog.sh /usr/local/bin/harbor-watchdog.sh - chmod +x /usr/local/bin/harbor-watchdog.sh - cp /tmp/harbor-watchdog.service /etc/systemd/system/harbor-watchdog.service - systemctl daemon-reload - systemctl enable harbor-watchdog.service - systemctl restart harbor-watchdog.service - echo watchdog OK -'" +ssh "$HOST" "sudo cp /tmp/harbor-watchdog.sh /usr/local/bin/harbor-watchdog.sh && \ + sudo chmod +x /usr/local/bin/harbor-watchdog.sh && \ + sudo cp /tmp/harbor-watchdog.service /etc/systemd/system/harbor-watchdog.service && \ + sudo systemctl daemon-reload && \ + sudo systemctl enable harbor-watchdog.service && \ + sudo systemctl restart harbor-watchdog.service && \ + echo 'watchdog OK'" # 5. 驗證 echo "[5/5] 驗證..." -ssh "$HOST" "echo '$PASS' | sudo -S systemctl is-active harbor-watchdog.service && echo '✅ harbor-watchdog active' || echo '❌ harbor-watchdog 失敗'" -ssh "$HOST" "echo '$PASS' | sudo -S systemctl is-enabled awoooi-startup-110.service && echo '✅ startup-110 enabled' || echo '❌ startup-110 未啟用'" +ssh "$HOST" "sudo systemctl is-active harbor-watchdog.service && echo '✅ harbor-watchdog active' || echo '❌ harbor-watchdog 失敗'" +ssh "$HOST" "sudo systemctl is-enabled awoooi-startup-110.service && echo '✅ startup-110 enabled' || echo '❌ startup-110 未啟用'" echo "" echo "✅ 部署完成！" @@ -54,4 +53,4 @@ echo "harbor-watchdog: 每 60 秒檢查 Harbor 健康，不健康自動修復" echo "startup-110: 下次重開機後自動恢復所有服務" echo "" echo "查看 watchdog 狀態:" -echo " journalctl -u harbor-watchdog.service -f" +echo " ssh $HOST 'journalctl -u harbor-watchdog.service -f'" diff --git a/scripts/reboot-recovery/harbor-watchdog.sh b/scripts/reboot-recovery/harbor-watchdog.sh index b002fa53f..32565fdbb 100644 --- a/scripts/reboot-recovery/harbor-watchdog.sh +++ b/scripts/reboot-recovery/harbor-watchdog.sh @@ -1,52 +1,75 @@ #!/bin/bash # Harbor Watchdog — 運行中崩潰自動恢復 -# 2026-04-05 Claude Code: 解決 Harbor 在非重開機情境下崩潰後無法自動恢復 +# 2026-04-05 Claude Code: 解決 Harbor Exited(128) 死鎖問題 +# 首席架構師 Review 2026-04-05: 修正 I1(lockfile) I2(stderr) I3(loop防護) I4(指令退出碼) S1(驗證等待) # # 問題根因： -# Docker restart:always 在 Exited (128) 情況下會重試直到 backoff 放棄 -# 因為 harbor-core/db 在啟動時嘗試連 syslog:1514 失敗就 exit， -# 形成死鎖：harbor-log healthy → 但其他容器已進入 backoff 不再重試 +# Docker restart:always 在 Exited(128) 情況下重試後進入 backoff 放棄 +# harbor-core/db/portal 啟動時連 syslog:1514（harbor-log 未就緒）失敗 exit # -# 解法：此 watchdog 每 60 秒檢查 Harbor API 健康， -# 若不健康則執行完整的時序修復（harbor-log first 策略） +# 設計：此 watchdog 每 60 秒輪詢 Harbor API， +# 不健康 → 等 5s 確認 → 執行 4 Phase 修復（harbor-log first 策略） +# 使用 lockfile 防止與 startup-110.sh 並行修復（I1） # # 部署位置: /usr/local/bin/harbor-watchdog.sh (on 192.168.0.110) -# systemd unit: /etc/systemd/system/harbor-watchdog.service (Type=simple, loop) +# systemd unit: /etc/systemd/system/harbor-watchdog.service HARBOR_DIR="/home/wooo/harbor/harbor" LOG="/var/log/harbor-watchdog.log" +LOCKFILE="/var/lock/harbor-repair.lock" CHECK_INTERVAL=60 # 秒 +# 注意：不使用 set -e，watchdog 是長駐 loop，-e 會在任何指令失敗時終止腳本 +# 所有關鍵指令的退出碼需明確捕捉（I4） +set -uo pipefail + log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] [watchdog] $*" | tee -a "$LOG"; } harbor_is_healthy() { local code - # 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗） - code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0") + # 注意：Harbor /v2/ 正常回傳 401（需認證），不能用 -f（-f 把 401 視為失敗 exit 22） + # 使用 127.0.0.1 避免 IPv6 解析問題 + code=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "000") [ "$code" = "401" ] || [ "$code" = "200" ] } repair_harbor() { - log "⚠️ Harbor 不健康，開始修復程序..." - cd "$HARBOR_DIR" || { log "❌ 找不到 Harbor 目錄"; return 1; } - - # Step 1: 清除所有 Exited Harbor 容器（打破 backoff 死鎖） - local exited - exited=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "^harbor-" | grep "Exited" | awk '{print $1}') - if [ -n "$exited" ]; then - log "清除 Exited 容器: $(echo "$exited" | tr '\n' ' ')" - echo "$exited" | xargs docker rm -f 2>/dev/null || true + # I1: lockfile 防止與 startup-110.sh 並行修復 + exec 9>"$LOCKFILE" + if ! flock -n 9; then + log "另一修復程序正在進行，跳過本次修復" + return 0 fi - # Step 2: 只啟動 harbor-log - log "啟動 harbor-log..." - docker compose up -d harbor-log 2>/dev/null + log "⚠️ 開始修復 Harbor（4 Phase 策略）..." + cd "$HARBOR_DIR" || { log "❌ 找不到 Harbor 目錄: $HARBOR_DIR"; return 1; } - # Step 3: 等 harbor-log healthy（最多 90s） + # Phase 1: 清除所有 Exited Harbor 容器（打破 backoff 死鎖） + local exited + exited=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true) + if [ -n "$exited" ]; then + log "Phase 1: 清除 Exited 容器: $(echo "$exited" | tr '\n' ' ')" + echo "$exited" | xargs docker rm -f 2>&1 | while IFS= read -r line; do log " $line"; done || true + else + log "Phase 1: 無 Exited 容器" + fi + + # Phase 2: 只啟動 harbor-log（I2: 捕捉 stderr，I4: 確認退出碼） + log "Phase 2: 啟動 harbor-log..." + if ! docker compose up -d harbor-log 2>&1 | while IFS= read -r line; do log " $line"; done; then + log "❌ harbor-log 啟動指令失敗" + return 1 + fi + + # Phase 3: 等 harbor-log healthy（最多 90s） + log "Phase 3: 等待 harbor-log healthy（最多 90s）..." local i status for i in $(seq 1 18); do status=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing") - [ "$status" = "healthy" ] && break + if [ "$status" = "healthy" ]; then + log "✅ harbor-log healthy (${i}x5s)" + break + fi sleep 5 done @@ -56,30 +79,35 @@ repair_harbor() { return 1 fi - # Step 4: 啟動全部 - log "harbor-log healthy，啟動全組件..." - docker compose up -d 2>/dev/null | tail -5 + # Phase 4: 啟動全組件（I2: 捕捉 stderr） + log "Phase 4: 啟動 Harbor 全組件..." + docker compose up -d 2>&1 | while IFS= read -r line; do log " $line"; done || true + + # S1: 等 30s 讓 harbor-core 完成初始化（原 10s 不夠） + log "等待 30s 讓 harbor-core 完成初始化..." + sleep 30 - sleep 10 if harbor_is_healthy; then log "✅ Harbor 修復成功" else log "❌ Harbor 修復後仍不健康，需人工介入" - log " 手動: cd $HARBOR_DIR && docker compose down && docker compose up -d harbor-log && sleep 30 && docker compose up -d" + log " 手動: cd $HARBOR_DIR && docker compose down && docker compose up -d harbor-log && sleep 60 && docker compose up -d" fi } log "=== Harbor Watchdog 啟動 (interval=${CHECK_INTERVAL}s) ===" +# I3: while loop 本體用 || true 保護，子 shell 異常不終止整個 watchdog while true; do - if harbor_is_healthy; then - : # 健康，靜默 - else - # 等 5 秒再確認（避免短暫不穩定誤報） - sleep 5 + ( if ! harbor_is_healthy; then - repair_harbor + # 等 5 秒再確認（避免短暫不穩定誤報） + sleep 5 + if ! harbor_is_healthy; then + repair_harbor + fi fi - fi + ) || log "⚠️ 本次檢查週期發生未預期錯誤，繼續下一週期" + sleep "$CHECK_INTERVAL" done