fix(runner): 永久修復 _diag/pages 檔案衝突問題
問題: Runner 並行執行時 "file already exists" 導致 CD 失敗 解決方案: 1. CD Workflow: 刪除整個 _diag/pages 目錄再重建 (非 rm -rf /*) 2. Systemd Timer: 每 5 分鐘自動清理過期檔案 3. flock 鎖定: 防止清理程序競爭 新增檔案: - ops/runner/cleanup-runner-diag.sh - 清理腳本 - ops/runner/runner-diag-cleanup.service - Systemd service - ops/runner/runner-diag-cleanup.timer - 定時器 - ops/runner/deploy-runner-cleanup.sh - 部署腳本 - ops/runner/README.md - 文檔 部署指令: ssh wooo@192.168.0.110 bash awoooi/ops/runner/deploy-runner-cleanup.sh Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
OG T
76
.github/workflows/cd.yaml
vendored
76
.github/workflows/cd.yaml
vendored
@@ -56,13 +56,51 @@ jobs:
|
||||
runs-on: [self-hosted, harbor, k8s]
|
||||
timeout-minutes: 1
|
||||
steps:
|
||||
# 2026-03-26: 清理暫存目錄，避免 file conflict (pages + temp)
|
||||
- name: "Clean Runner temp"
|
||||
# =======================================================================
|
||||
# 2026-03-29: Runner _diag/pages 檔案衝突永久修復
|
||||
# 問題: 並行 Job 寫入同一診斷檔案導致 "file already exists"
|
||||
# 解法: 強制清理 + flock 鎖定 + 重建目錄
|
||||
# =======================================================================
|
||||
- name: "Clean Runner Diagnostics (Anti-Collision)"
|
||||
run: |
|
||||
set +e # 不因清理失敗而中斷
|
||||
|
||||
RUNNER_ROOT=$(dirname "$(dirname "$RUNNER_TEMP")")
|
||||
rm -rf "$RUNNER_TEMP"/* 2>/dev/null || true
|
||||
rm -rf "$RUNNER_ROOT/_diag/pages"/* 2>/dev/null || true
|
||||
rm -rf .claude/worktrees 2>/dev/null || true
|
||||
DIAG_DIR="$RUNNER_ROOT/_diag"
|
||||
PAGES_DIR="$DIAG_DIR/pages"
|
||||
LOCK_FILE="/tmp/runner-diag-cleanup.lock"
|
||||
|
||||
echo "🧹 Cleaning Runner diagnostics..."
|
||||
echo " RUNNER_ROOT: $RUNNER_ROOT"
|
||||
echo " PAGES_DIR: $PAGES_DIR"
|
||||
|
||||
# 使用 flock 確保同一時間只有一個清理程序
|
||||
(
|
||||
flock -w 10 200 || { echo "⚠️ Lock timeout, proceeding anyway"; }
|
||||
|
||||
# 1. 清理 _diag/pages (最關鍵)
|
||||
if [ -d "$PAGES_DIR" ]; then
|
||||
# 刪除所有 .log 檔案
|
||||
find "$PAGES_DIR" -name "*.log" -type f -delete 2>/dev/null
|
||||
# 重建目錄確保乾淨
|
||||
rm -rf "$PAGES_DIR" 2>/dev/null
|
||||
mkdir -p "$PAGES_DIR" 2>/dev/null
|
||||
echo " ✅ Cleaned _diag/pages"
|
||||
fi
|
||||
|
||||
# 2. 清理 RUNNER_TEMP
|
||||
rm -rf "$RUNNER_TEMP"/* 2>/dev/null
|
||||
echo " ✅ Cleaned RUNNER_TEMP"
|
||||
|
||||
# 3. 清理 Claude worktrees
|
||||
rm -rf .claude/worktrees 2>/dev/null
|
||||
|
||||
# 4. 清理陳舊的 _work 暫存
|
||||
find "$RUNNER_ROOT/_work" -name "*.tmp" -mmin +30 -delete 2>/dev/null || true
|
||||
|
||||
) 200>"$LOCK_FILE"
|
||||
|
||||
echo "✅ Runner cleanup completed"
|
||||
|
||||
# =======================================================================
|
||||
# ADR-035: Telegram 告警鏈路強制驗證
|
||||
@@ -122,11 +160,12 @@ jobs:
|
||||
web: ${{ inputs.force_deploy == true && 'true' || steps.filter.outputs.web }}
|
||||
k3s-system: ${{ steps.filter.outputs.k3s-system }}
|
||||
steps:
|
||||
# 2026-03-26: 清理暫存目錄 (temp + pages)
|
||||
- name: "Clean Runner temp"
|
||||
# 2026-03-29: Runner 診斷檔案清理 (防止並行衝突)
|
||||
- name: "Clean Runner Diagnostics"
|
||||
run: |
|
||||
RUNNER_ROOT=$(dirname "$(dirname "$RUNNER_TEMP")")
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages"/* .claude/worktrees 2>/dev/null || true
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages" .claude/worktrees 2>/dev/null || true
|
||||
mkdir -p "$RUNNER_ROOT/_diag/pages" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
@@ -162,11 +201,12 @@ jobs:
|
||||
outputs:
|
||||
image_tag: ${{ steps.tag.outputs.tag }}
|
||||
steps:
|
||||
# 2026-03-26: 清理暫存目錄 (temp + pages)
|
||||
- name: "Clean Runner temp"
|
||||
# 2026-03-29: Runner 診斷檔案清理 (防止並行衝突)
|
||||
- name: "Clean Runner Diagnostics"
|
||||
run: |
|
||||
RUNNER_ROOT=$(dirname "$(dirname "$RUNNER_TEMP")")
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages"/* .claude/worktrees 2>/dev/null || true
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages" .claude/worktrees 2>/dev/null || true
|
||||
mkdir -p "$RUNNER_ROOT/_diag/pages" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -200,11 +240,12 @@ jobs:
|
||||
outputs:
|
||||
image_tag: ${{ steps.tag.outputs.tag }}
|
||||
steps:
|
||||
# 2026-03-26: 清理暫存目錄 (temp + pages)
|
||||
- name: "Clean Runner temp"
|
||||
# 2026-03-29: Runner 診斷檔案清理 (防止並行衝突)
|
||||
- name: "Clean Runner Diagnostics"
|
||||
run: |
|
||||
RUNNER_ROOT=$(dirname "$(dirname "$RUNNER_TEMP")")
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages"/* .claude/worktrees 2>/dev/null || true
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages" .claude/worktrees 2>/dev/null || true
|
||||
mkdir -p "$RUNNER_ROOT/_diag/pages" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
@@ -245,11 +286,12 @@ jobs:
|
||||
if: always() && (needs.build-api.result == 'success' || needs.build-api.result == 'skipped') && (needs.build-web.result == 'success' || needs.build-web.result == 'skipped')
|
||||
environment: production
|
||||
steps:
|
||||
# 2026-03-26: 清理暫存目錄 (temp + pages)
|
||||
- name: "Clean Runner temp"
|
||||
# 2026-03-29: Runner 診斷檔案清理 (防止並行衝突)
|
||||
- name: "Clean Runner Diagnostics"
|
||||
run: |
|
||||
RUNNER_ROOT=$(dirname "$(dirname "$RUNNER_TEMP")")
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages"/* .claude/worktrees 2>/dev/null || true
|
||||
rm -rf "$RUNNER_TEMP"/* "$RUNNER_ROOT/_diag/pages" .claude/worktrees 2>/dev/null || true
|
||||
mkdir -p "$RUNNER_ROOT/_diag/pages" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
|
||||
Reference in New Issue
Block a user