wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity
Files
c9bb4bbfc6af05c41df7c3af546ba44cbcf45708
awoooi/scripts/reboot-recovery/awoooi-startup-110.sh
Your Name d84be37e26
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Successful in 48s
Details
CD Pipeline / post-deploy-checks (push) Has been cancelled
Details
CD Pipeline / build-and-deploy (push) Has been cancelled
Details
fix(reboot): repair 110 ssh control path on startup
2026-07-02 10:53:16 +08:00

848 lines
35 KiB
Bash
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/bin/bash
# 192.168.0.110 重開機自動恢復腳本
# 2026-04-05 ogt: 根據第二次重開機事故建立
# 部署位置: /usr/local/bin/awoooi-startup-110.sh (on 192.168.0.110)
# systemd unit: /etc/systemd/system/awoooi-startup-110.service
#
# 已知問題處理:
# - Docker BoltDB 損壞 (network/files/local-kv.db, volumes/metadata.db)
# - Docker container metadata JSON 損壞，dockerd 卡在 startup / loading containers
# - 舊容器使用已不存在的 Docker network (需要 docker rm -f 全部)
# - Harbor nginx 依賴 harbor-log (需要等 harbor-log healthy 後才 compose up)
set -uo pipefail
LOG="/var/log/awoooi-startup-110.log"
DOCKER_START_TIMEOUT_SECONDS="${AWOOOI_DOCKER_START_TIMEOUT_SECONDS:-45}"
DOCKER_ACTIVE_WAIT_ATTEMPTS="${AWOOOI_DOCKER_ACTIVE_WAIT_ATTEMPTS:-18}"
DOCKER_ACTIVE_WAIT_SLEEP_SECONDS="${AWOOOI_DOCKER_ACTIVE_WAIT_SLEEP_SECONDS:-5}"
exec > >(tee -a "$LOG") 2>&1
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; }
run_bounded() {
local seconds="$1"
shift
if command -v timeout >/dev/null 2>&1; then
timeout "$seconds" "$@"
else
"$@"
fi
}
start_docker_bounded() {
log "bounded start docker.service/docker.socket timeout=${DOCKER_START_TIMEOUT_SECONDS}s"
run_bounded "$DOCKER_START_TIMEOUT_SECONDS" systemctl start docker.socket docker.service || true
}
wait_docker_active() {
local i state
for i in $(seq 1 "$DOCKER_ACTIVE_WAIT_ATTEMPTS"); do
state=$(systemctl is-active docker 2>/dev/null || true)
log "Docker wait ${i}/${DOCKER_ACTIVE_WAIT_ATTEMPTS}: ${state:-unknown}"
[ "$state" = "active" ] && return 0
sleep "$DOCKER_ACTIVE_WAIT_SLEEP_SECONDS"
done
return 1
}
stop_docker_bounded_for_metadata_recovery() {
local state
run_bounded 25 systemctl stop docker.service docker.socket || true
state=$(systemctl is-active docker 2>/dev/null || true)
if [ "$state" = "active" ] || [ "$state" = "activating" ] || [ "$state" = "deactivating" ]; then
log "Docker stop still ${state}; TERM main process for metadata quarantine recovery"
systemctl kill --kill-who=main --signal=TERM docker.service || true
sleep 4
run_bounded 20 systemctl stop docker.service docker.socket || true
fi
}
DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT=0
quarantine_corrupt_docker_container_metadata() {
local ids qdir id src count
DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT=0
ids=$(journalctl -u docker.service --since "-30 min" --no-pager 2>/dev/null \
| awk '/failed to load container|invalid character|json: cannot unmarshal|unexpected EOF/ {
line=$0
while (match(line, /[0-9a-f]{64}/)) {
print substr(line, RSTART, RLENGTH)
line=substr(line, RSTART + RLENGTH)
}
}' \
| sort -u)
if [ -z "$ids" ]; then
log "Docker corrupt container metadata: journal 未找到可 quarantine 的 container id"
return 0
fi
qdir="/var/lib/docker/containers/.awoooi-corrupt-metadata-quarantine-$(date +%Y%m%d-%H%M%S)"
mkdir -p "$qdir"
count=0
for id in $ids; do
case "$id" in
*[!0-9a-f]*|"")
log "跳過不合法 container id: $id"
continue
;;
esac
src="/var/lib/docker/containers/$id"
if [ -d "$src" ]; then
mv "$src" "$qdir/$id"
count=$((count + 1))
log "quarantine corrupt Docker container metadata: $id"
else
log "corrupt metadata candidate already absent: $id"
fi
done
DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT="$count"
log "Docker corrupt metadata quarantine dir=$qdir count=$count"
}
repair_ssh_control_path_metadata() {
local script="${AWOOOI_110_SSH_REPAIR_SCRIPT:-/usr/local/bin/repair-110-ssh-publickey-auth-local.sh}"
local target_user="${AWOOOI_110_SSH_REPAIR_USER:-wooo}"
local reload_ssh="${AWOOOI_110_SSH_RELOAD_AFTER_REPAIR:-0}"
log "[0/6] 修復 110 SSH control path metadata..."
if [ ! -x "$script" ]; then
log "⚠️ SSH control path metadata repair helper missing or not executable: $script"
return 0
fi
if TARGET_USER="$target_user" RELOAD_SSH="$reload_ssh" "$script" --apply; then
log "✅ 110 SSH control path metadata guard completed target_user=$target_user reload_ssh=$reload_ssh"
else
log "⚠️ 110 SSH control path metadata guard failed; continuing startup"
fi
}
log "=== 192.168.0.110 啟動序列開始 ==="
repair_ssh_control_path_metadata
# ──────────────────────────────────────────────
# STEP 1: Docker 修復（若 BoltDB 損壞）
# ──────────────────────────────────────────────
log "[1/5] 檢查 Docker..."
if ! systemctl is-active docker >/dev/null 2>&1; then
log "Docker 未啟動，嘗試啟動..."
start_docker_bounded
wait_docker_active || true
fi
if ! systemctl is-active docker >/dev/null 2>&1; then
log "Docker bounded start 未 active，檢查 corrupt container metadata..."
quarantine_corrupt_docker_container_metadata
if [ "$DOCKER_CORRUPT_METADATA_QUARANTINED_COUNT" -gt 0 ]; then
stop_docker_bounded_for_metadata_recovery
start_docker_bounded
wait_docker_active || true
fi
fi
if ! systemctl is-active docker >/dev/null 2>&1; then
log "Docker 啟動失敗，修復 BoltDB..."
# 清除所有已知 BoltDB 損壞點
NETWORK_DB="/var/lib/docker/network/files/local-kv.db"
VOLUMES_DB="/var/lib/docker/volumes/metadata.db"
if [ -f "$NETWORK_DB" ]; then
cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)"
rm -f "$NETWORK_DB"
log "清除損壞的 local-kv.db"
fi
if [ -f "$VOLUMES_DB" ]; then
cp "$VOLUMES_DB" "${VOLUMES_DB}.bak.$(date +%Y%m%d%H%M%S)"
rm -f "$VOLUMES_DB"
log "清除損壞的 metadata.db"
fi
# 清除 buildkit 快取（也可能損壞）
find /var/lib/docker/buildkit -name "*.db" -delete 2>/dev/null || true
start_docker_bounded
wait_docker_active || true
systemctl is-active docker && log "✅ Docker 修復成功" || { log "❌ Docker 修復失敗"; exit 1; }
else
log "✅ Docker 已 active"
fi
# ──────────────────────────────────────────────
# STEP 2: 清除孤兒容器（舊容器的 network 已不存在）
# ──────────────────────────────────────────────
log "[2/5] 清除孤兒容器..."
STALE=$(docker ps -a --format "{{.Names}}\t{{.Status}}" | grep "Exited (128)\|Exited (137)" | awk '{print $1}')
if [ -n "$STALE" ]; then
log "發現孤兒容器: $(echo $STALE | tr '\n' ' ')"
echo "$STALE" | xargs docker rm -f 2>/dev/null || true
docker network prune -f 2>/dev/null || true
log "✅ 孤兒容器清除完成"
else
log "✅ 無孤兒容器"
fi
# ──────────────────────────────────────────────
# STEP 3: Harbor（嚴格時序：harbor-log first）
# 2026-04-05 Claude Code: 修復 Race Condition —
# 舊邏輯：compose up -d 全部 → 等 harbor-log healthy → compose up -d 全部
# 問題：第一次 up 時 harbor-core/db/portal 嘗試連 syslog:1514 失敗 exit，
# restart:always 重試直到放棄，第二次 compose up 無效
# 新邏輯：只啟動 harbor-log → 等 healthy → 啟動其餘全部
# 首席架構師 Review I1: 使用 lockfile 防止 watchdog 並行修復
# ──────────────────────────────────────────────
log "[3/5] 啟動 Harbor..."
HARBOR_DIR="/home/wooo/harbor/harbor"
HARBOR_LOCKFILE="/var/lock/harbor-repair.lock"
if [ -f "$HARBOR_DIR/docker-compose.yml" ]; then
cd "$HARBOR_DIR"
# I1: 持有 lockfile，防止 harbor-watchdog 同時修復
exec 9>"$HARBOR_LOCKFILE"
flock -w 30 9 || { log "⚠️ 無法取得 Harbor lock，跳過（watchdog 可能正在修復）"; return 0; }
# Phase 1: 清除任何 Exited 狀態的 Harbor 容器（避免 restart:always 干擾）
# 使用 --filter 取代 grep/awk（更精確，不受欄位格式影響）
HARBOR_EXITED=$(docker ps -a --filter name="harbor-" --filter status=exited --format "{{.Names}}" 2>/dev/null || true)
if [ -n "$HARBOR_EXITED" ]; then
log "Phase 1: 清除 Exited Harbor 容器: $(echo "$HARBOR_EXITED" | tr '\n' ' ')"
echo "$HARBOR_EXITED" | xargs docker rm -f 2>&1 | tail -3 || true
else
log "Phase 1: 無 Exited 容器"
fi
# Phase 2: 只啟動 harbor-log（syslog 接收端，其他容器的前置依賴）
log "Phase 2: 啟動 harbor-log..."
docker compose up -d harbor-log 2>&1 | tail -3
# Phase 3: 等待 harbor-log healthy（最多 90 秒）
log "Phase 3: 等待 harbor-log healthy（最多 90s）..."
for i in $(seq 1 18); do
STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "missing")
if [ "$STATUS" = "healthy" ]; then
log "✅ harbor-log healthy (${i}x5s)"
break
fi
sleep 5
done
STATUS=$(docker inspect --format='{{.State.Health.Status}}' harbor-log 2>/dev/null || echo "unknown")
if [ "$STATUS" = "healthy" ]; then
# Phase 4: harbor-log 就緒後才啟動其餘組件
log "Phase 4: 啟動 Harbor 全組件..."
docker compose up -d 2>&1 | tail -5
# S1: 等 30s 讓 harbor-core 完成初始化（之前 5s 不夠）
sleep 30
# 注意：Harbor /v2/ 回 401（需認證），不能用 -f flag（會把 401 當失敗）
HTTP=$(curl -s --max-time 5 -o /dev/null -w "%{http_code}" http://127.0.0.1:5000/v2/ 2>/dev/null || echo "0")
if [ "$HTTP" = "401" ] || [ "$HTTP" = "200" ]; then
log "✅ Harbor 就緒 (HTTP $HTTP)"
else
log "⚠️ Harbor API 未就緒 (HTTP $HTTP)，harbor-watchdog 會在 60s 後自動修復"
fi
else
log "❌ harbor-log 未 healthy (status=$STATUS)，跳過其餘 Harbor 容器"
log " harbor-watchdog 會在 60s 後自動偵測並修復"
fi
else
log "⚠️ 找不到 Harbor compose 檔案"
fi
# ──────────────────────────────────────────────
# STEP 4: 其他服務（Gitea, Langfuse, Monitoring）
# ──────────────────────────────────────────────
log "[4/6] 啟動其他服務（Gitea, Langfuse, Monitoring）..."
GITEA_DIR="/home/wooo/gitea"
if [ -f "$GITEA_DIR/docker-compose.yml" ]; then
cd "$GITEA_DIR"
docker compose up -d 2>&1 | tail -3
log "✅ Gitea 啟動指令已發送"
fi
LANGFUSE_DIR="/home/wooo/langfuse"
if [ -f "$LANGFUSE_DIR/docker-compose.yml" ]; then
cd "$LANGFUSE_DIR"
docker compose up -d 2>&1 | tail -3
log "✅ Langfuse 啟動指令已發送"
fi
MONITORING_DIR="/home/wooo/monitoring"
if [ -f "$MONITORING_DIR/docker-compose.yml" ]; then
cd "$MONITORING_DIR"
docker compose up -d 2>&1 | tail -3
log "✅ Monitoring (Prometheus/Grafana/Alertmanager) 啟動指令已發送"
sleep 10
# 驗證 Alertmanager 是否啟動
if curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1; then
log "✅ Alertmanager healthy"
else
log "⚠️ Alertmanager 尚未就緒，等待 20 秒..."
sleep 20
curl -sf --max-time 5 http://localhost:9093/-/healthy >/dev/null 2>&1 && log "✅ Alertmanager 就緒" || log "❌ Alertmanager 未就緒，需手動檢查"
fi
fi
# ──────────────────────────────────────────────
# STEP 5: SignOz
# ──────────────────────────────────────────────
log "[5/6] 啟動 SignOz..."
SIGNOZ_DIR="/home/wooo/signoz/deploy/docker"
if [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then
cd "$SIGNOZ_DIR"
docker compose up -d 2>&1 | tail -5
log "✅ SignOz 啟動指令已發送"
fi
# ──────────────────────────────────────────────
# STEP 6: Gitea Act Runner（CI/CD 核心）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Gitea runner 離線、CD 失效
# 2026-06-27 Codex: 110 runner labels 收斂，避免接泛用 shared CI。
# 2026-06-27 Codex: 110 是 production / registry / observability 主機；
# legacy runner 預設維持停用降壓；controlled drain lane 可在受控授權下啟動。
# ──────────────────────────────────────────────
log "[6/6] 檢查 Gitea Act Runner（legacy fail-closed；controlled drain 守門自動啟動）..."
RUNNER_DIR="/home/wooo/act-runner"
RUNNER_SERVICE="gitea-act-runner-host.service"
RUNNER_ENABLE_SENTINEL="/run/awoooi-runner-host-enabled"
CD_LANE_DIR="/home/wooo/awoooi-cd-lane"
CD_LANE_SERVICE="awoooi-cd-lane.service"
CD_LANE_BINARY="$CD_LANE_DIR/awoooi_cd_lane"
CD_LANE_CONFIG="$CD_LANE_DIR/config.yaml"
CD_LANE_DRAIN_DIR="/home/wooo/awoooi-cd-lane-drain"
CD_LANE_DRAIN_SERVICE="awoooi-cd-lane-drain.service"
CD_LANE_DRAIN_BINARY="$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
CD_LANE_DRAIN_CONFIG="$CD_LANE_DRAIN_DIR/config.yaml"
CD_LANE_ENABLE_SENTINEL="/run/awoooi-cd-lane-enabled"
START_GITEA_RUNNER_ON_BOOT="${AWOOOI_START_GITEA_RUNNER_ON_BOOT:-0}"
START_CONTROLLED_CD_LANE="${AWOOOI_START_CONTROLLED_CD_LANE:-1}"
START_GITEA_RUNNER_ALLOWED=0
START_CD_LANE_ALLOWED=0
RUNNER_FAIL_CLOSED_SERVICES=(
"awoooi-cd-lane.service"
"awoooi-direct-runner-open.service"
"awoooi-direct-runner.service"
"gitea-act-runner-host.service"
"gitea-act-runner-awoooi-controlled.service"
"gitea-awoooi-controlled-runner.service"
"gitea-act-runner-awoooi-open.service"
)
RUNNER_FAIL_CLOSED_BINARY_PATHS=(
"/home/wooo/awoooi-cd-lane/awoooi_cd_lane"
"/home/wooo/act-runner/act_runner"
"/home/wooo/act-runner/act_runner.real-20260628-runner-pressure-guard"
"/home/wooo/act-runner-controlled/act_runner"
"/home/wooo/awoooi-controlled-runner/awoooi_controlled_runner"
)
# Host runner still needs both keys. The direct cd-lane stays fail-closed until
# it is migrated or hard-limited outside this production host pressure lane.
# The drain lane is a narrow post-reboot recovery path; it auto-starts only when
# config/binary/label/restore-source guardrails pass, and can be explicitly
# disabled with AWOOOI_START_CONTROLLED_CD_LANE=0.
if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ] && [ -e "$RUNNER_ENABLE_SENTINEL" ]; then
START_GITEA_RUNNER_ALLOWED=1
fi
mask_runner_unit_file() {
local unit="$1"
local unit_dir="$2"
local owner_user="${3:-}"
local unit_file="$unit_dir/$unit"
local quarantine_stamp
quarantine_stamp="$(date +%Y%m%d%H%M%S)"
mkdir -p "$unit_dir" >/dev/null 2>&1 || true
if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
return 0
fi
if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
chattr -i "$unit_file" >/dev/null 2>&1 || true
mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
fi
ln -s /dev/null "$unit_file" >/dev/null 2>&1 || true
if [ -n "$owner_user" ]; then
chown -h "$owner_user:$owner_user" "$unit_file" >/dev/null 2>&1 || true
fi
}
guard_runner_binary_fail_closed() {
local path="$1"
local tmp
local quarantine_stamp
quarantine_stamp="$(date +%Y%m%d%H%M%S)"
if [ -e "$path" ]; then
chattr -i "$path" >/dev/null 2>&1 || true
if file "$path" 2>/dev/null | grep -qi "ELF"; then
mv "$path" "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
chmod 0400 "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
chattr +i "${path}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
fi
fi
tmp="$(mktemp)"
cat >"$tmp" <<'EOF'
#!/usr/bin/env bash
set -eu
echo "AWOOOI host runner is fail-closed on 110 after 2026-06-28 pressure incident; migrate or rate-limit before enabling." >&2
exit 75
EOF
install -o root -g root -m 0755 "$tmp" "$path" >/dev/null 2>&1 || true
rm -f "$tmp"
chattr +i "$path" >/dev/null 2>&1 || true
}
install_cd_lane_fail_closed_unit() {
local unit_file="/etc/systemd/system/awoooi-cd-lane.service"
local quarantine_stamp
quarantine_stamp="$(date +%Y%m%d%H%M%S)"
systemctl mask awoooi-cd-lane.service >/dev/null 2>&1 || true
if [ -e "$unit_file" ] || [ -L "$unit_file" ]; then
chattr -i "$unit_file" >/dev/null 2>&1 || true
if ! { [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; }; then
mv "$unit_file" "${unit_file}.quarantined-runner-incident-${quarantine_stamp}" >/dev/null 2>&1 || true
fi
fi
ln -sfn /dev/null "$unit_file" >/dev/null 2>&1 || true
}
install_controlled_cd_lane_unit() {
local unit_file="/etc/systemd/system/$CD_LANE_SERVICE"
local tmp
chattr -i "$unit_file" "$CD_LANE_BINARY" >/dev/null 2>&1 || true
tmp="$(mktemp)"
cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_BINARY} daemon --config ${CD_LANE_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100
[Install]
WantedBy=multi-user.target
EOF
install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
rm -f "$tmp"
}
install_controlled_cd_lane_drain_unit() {
local unit_file="/etc/systemd/system/$CD_LANE_DRAIN_SERVICE"
local tmp
chattr -i "$unit_file" "$CD_LANE_DRAIN_BINARY" >/dev/null 2>&1 || true
if [ -L "$unit_file" ] && [ "$(readlink "$unit_file" 2>/dev/null || true)" = "/dev/null" ]; then
rm -f "$unit_file" >/dev/null 2>&1 || true
fi
tmp="$(mktemp)"
cat >"$tmp" <<EOF
[Unit]
Description=AWOOOI controlled CD lane drain bypass for old queued guards
After=network-online.target docker.service
Wants=network-online.target
Requires=docker.service
ConditionPathExists=${CD_LANE_DRAIN_DIR}/data/.runner
[Service]
Type=simple
User=wooo
WorkingDirectory=${CD_LANE_DRAIN_DIR}/data
Environment=HOME=/home/wooo
Environment=AWOOOI_CONTROLLED_RUNNER_OPEN=1
Environment=HOST_WEB_BUILD_PRESSURE_ATTEMPTS=1
Environment=HOST_WEB_BUILD_PRESSURE_SLEEP_SECONDS=1
ExecStart=${CD_LANE_DRAIN_BINARY} daemon --config ${CD_LANE_DRAIN_CONFIG}
Restart=always
RestartSec=10
KillSignal=SIGINT
TimeoutStopSec=3700
SuccessExitStatus=0 130 143
CPUAccounting=true
CPUQuota=250%
MemoryAccounting=true
MemoryHigh=8G
MemoryMax=12G
TasksAccounting=true
TasksMax=512
IOAccounting=true
IOWeight=100
[Install]
WantedBy=multi-user.target
EOF
install -o root -g root -m 0644 "$tmp" "$unit_file" >/dev/null 2>&1 || true
rm -f "$tmp"
}
cd_lane_config_path_is_controlled() {
local config_path="$1"
[ -f "$config_path" ] || return 1
grep -Eq '^[[:space:]]+capacity:[[:space:]]*1[[:space:]]*$' "$config_path" || return 1
grep -q 'awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04' "$config_path" || return 1
grep -q 'awoooi-host:host' "$config_path" || return 1
if grep -Eq '^[[:space:]]+- ".*(ubuntu-latest|stockplatform|headless|playwright)' "$config_path"; then
return 1
fi
return 0
}
cd_lane_config_is_controlled() {
cd_lane_config_path_is_controlled "$CD_LANE_CONFIG"
}
cd_lane_drain_config_is_controlled() {
cd_lane_config_path_is_controlled "$CD_LANE_DRAIN_CONFIG"
}
cd_lane_drain_is_controlled_open() {
local active
active="$(systemctl show "$CD_LANE_DRAIN_SERVICE" -p ActiveState --value 2>/dev/null || true)"
[ "$active" = "active" ] || return 1
cd_lane_drain_config_is_controlled || return 1
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
return 0
}
cd_lane_drain_is_controlled_available() {
cd_lane_drain_config_is_controlled || return 1
file "$CD_LANE_DRAIN_BINARY" 2>/dev/null | grep -qi "ELF" || return 1
return 0
}
cd_lane_root_restore_sources_left() {
find /root -maxdepth 1 -type d \( \
-name 'awoooi-cd-lane-disabled-*' -o \
-name 'awoooi-cd-lane-drain-disabled-*' \
\) -print 2>/dev/null | wc -l | tr -d " "
}
quarantine_cd_lane_registration_fail_closed() {
local quarantine_dir
local lane_dir
local path
local target
rm -f /run/awoooi-cd-lane-enabled /run/awoooi-cd-lane-controlled-open >/dev/null 2>&1 || true
for lane_dir in "$CD_LANE_DIR" "$CD_LANE_DRAIN_DIR"; do
[ -d "$lane_dir" ] || continue
quarantine_dir="$lane_dir/quarantine-startup-$(date +%Y%m%d%H%M%S)"
chattr -i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
mkdir -p "$quarantine_dir" >/dev/null 2>&1 || true
while IFS= read -r -d '' path; do
[ -e "$path" ] || continue
chattr -i "$path" >/dev/null 2>&1 || true
target="$quarantine_dir/$(basename "$path")"
mv "$path" "$target" >/dev/null 2>&1 || true
chmod 0400 "$target" >/dev/null 2>&1 || true
chattr +i "$target" >/dev/null 2>&1 || true
done < <(
{
find "$lane_dir" -maxdepth 1 \( -name 'config.yaml' -o -name 'config.yaml.*' -o -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
find "$lane_dir/data" -maxdepth 1 \( -name '.runner' -o -name '.runner.*' \) -print0 2>/dev/null
} || true
)
chattr +i "$lane_dir" "$lane_dir/data" >/dev/null 2>&1 || true
done
}
quarantine_cd_lane_root_restore_sources_fail_closed() {
local final_root
local path
local target_dir
final_root="/root/awoooi-runner-restore-sources-disabled-final-$(date +%Y%m%dT%H%M%S%z)"
target_dir="$final_root/cd-lane-restore-sources"
mkdir -p "$target_dir" >/dev/null 2>&1 || true
while IFS= read -r -d '' path; do
[ -d "$path" ] || continue
chattr -R -i "$path" >/dev/null 2>&1 || true
mv "$path" "$target_dir/" >/dev/null 2>&1 || true
done < <(
{
find /root -maxdepth 1 -type d -name 'awoooi-cd-lane-disabled-*' -print0 2>/dev/null
find /root -maxdepth 1 -type d -name 'awoooi-cd-lane-drain-disabled-*' -print0 2>/dev/null
} || true
)
}
apply_cd_lane_fail_closed_guard() {
local unit
for unit in awoooi-cd-lane.service awoooi-cd-lane-drain.service; do
systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
systemctl stop "$unit" >/dev/null 2>&1 || true
systemctl reset-failed "$unit" >/dev/null 2>&1 || true
systemctl disable "$unit" >/dev/null 2>&1 || true
if [ "$unit" = "awoooi-cd-lane.service" ]; then
install_cd_lane_fail_closed_unit
else
systemctl mask "$unit" >/dev/null 2>&1 || mask_runner_unit_file "$unit" "/etc/systemd/system"
mask_runner_unit_file "$unit" "/etc/systemd/system"
fi
done
install_cd_lane_fail_closed_unit
pkill -KILL -f "^${CD_LANE_DIR}/awoooi_cd_lane daemon" >/dev/null 2>&1 || true
pkill -KILL -f "^${CD_LANE_DRAIN_DIR}/awoooi_cd_lane_controlled daemon" >/dev/null 2>&1 || true
quarantine_cd_lane_registration_fail_closed
quarantine_cd_lane_root_restore_sources_fail_closed
guard_runner_binary_fail_closed "$CD_LANE_DIR/awoooi_cd_lane"
guard_runner_binary_fail_closed "$CD_LANE_DRAIN_DIR/awoooi_cd_lane_controlled"
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl reset-failed awoooi-cd-lane.service awoooi-cd-lane-drain.service >/dev/null 2>&1 || true
}
ensure_cd_lane_fail_closed() {
apply_cd_lane_fail_closed_guard
}
ensure_controlled_cd_lane_open() {
mkdir -p /run >/dev/null 2>&1 || true
touch /run/awoooi-cd-lane-controlled-open /run/awoooi-cd-lane-drain-ok >/dev/null 2>&1 || true
log "✅ controlled cd-lane startup override active; drain lane remains open"
}
ensure_host_runner_fail_closed() {
local unit
local binary
local wooo_uid
for unit in "${RUNNER_FAIL_CLOSED_SERVICES[@]}"; do
systemctl kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
systemctl reset-failed "$unit" >/dev/null 2>&1 || true
systemctl disable "$unit" >/dev/null 2>&1 || true
if [ "$unit" = "awoooi-cd-lane.service" ]; then
install_cd_lane_fail_closed_unit
else
systemctl mask "$unit" >/dev/null 2>&1 || mask_runner_unit_file "$unit" "/etc/systemd/system"
mask_runner_unit_file "$unit" "/etc/systemd/system"
fi
done
systemctl daemon-reload >/dev/null 2>&1 || true
if wooo_uid="$(id -u wooo 2>/dev/null)"; then
mkdir -p /home/wooo/.config/systemd/user >/dev/null 2>&1 || true
for unit in "${RUNNER_FAIL_CLOSED_SERVICES[@]}"; do
if [ -d "/run/user/$wooo_uid" ] && command -v runuser >/dev/null 2>&1; then
runuser -u wooo -- env XDG_RUNTIME_DIR="/run/user/$wooo_uid" systemctl --user kill --signal=SIGKILL "$unit" >/dev/null 2>&1 || true
fi
mask_runner_unit_file "$unit" "/home/wooo/.config/systemd/user" "wooo"
done
fi
pkill -KILL -f "^${RUNNER_DIR}/act_runner(\\.real-[^ ]*)? daemon" >/dev/null 2>&1 || true
quarantine_cd_lane_root_restore_sources_fail_closed
for binary in "${RUNNER_FAIL_CLOSED_BINARY_PATHS[@]}"; do
if [ "$START_CD_LANE_ALLOWED" = "1" ] && [ "$binary" = "$CD_LANE_DRAIN_BINARY" ]; then
continue
fi
guard_runner_binary_fail_closed "$binary"
done
}
CD_LANE_ROOT_RESTORE_LEFT="$(cd_lane_root_restore_sources_left)"
if [ "$START_CONTROLLED_CD_LANE" = "1" ]; then
if ! cd_lane_drain_is_controlled_available; then
log "⛔ AWOOOI_START_CONTROLLED_CD_LANE=1 但 controlled drain lane config/binary 未通過；維持 fail-closed"
elif [ "$CD_LANE_ROOT_RESTORE_LEFT" != "0" ]; then
log "⛔ AWOOOI_START_CONTROLLED_CD_LANE=1 但 root restore-source left=${CD_LANE_ROOT_RESTORE_LEFT}；維持 fail-closed"
else
START_CD_LANE_ALLOWED=1
log "✅ controlled cd-lane drain preflight passed; legacy runner 仍維持 fail-closed"
fi
fi
if [ -x "$RUNNER_DIR/act_runner" ] && [ -f "$RUNNER_DIR/config.yaml" ]; then
# 若舊的 .runner 配置指向過期 hostname，只有在明確允許啟動 runner
# 時才清除重新註冊；預設降壓模式不得碰 registration 狀態。
RUNNER_FILE="$RUNNER_DIR/data/.runner"
if [ "$START_GITEA_RUNNER_ALLOWED" = "1" ] && [ -f "$RUNNER_FILE" ]; then
OLD_URL=$(python3 -c "import json; d=json.load(open('$RUNNER_FILE')); print(d.get('address',''))" 2>/dev/null || echo "")
if [ "$OLD_URL" != "http://192.168.0.110:3001" ]; then
log "⚠️ runner 配置過期 ($OLD_URL)，清除重新註冊..."
rm -f "$RUNNER_FILE" || true
fi
fi
# act_runner 預設 shutdown_timeout=0s，會在 daemon 重啟時立刻取消
# 正在執行的 CD job。這會造成部署實際完成但 Gitea status 失真。
python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true
import sys
from pathlib import Path
path = Path(sys.argv[1])
lines = path.read_text().splitlines()
if any(line.strip().startswith("shutdown_timeout:") for line in lines):
lines = [
" shutdown_timeout: 1h" if line.strip().startswith("shutdown_timeout:") else line
for line in lines
]
else:
for idx, line in enumerate(lines):
if line.strip().startswith("timeout:") and idx > 0:
lines.insert(idx + 1, " shutdown_timeout: 1h")
break
path.write_text("\n".join(lines) + "\n")
PY
# 110 是 production / registry / observability 主機，不再接泛用
# ubuntu-latest 類 job。泛用 label 會讓 StockPlatform 等其他 repo 的
# build 跑到 110，造成 CPU 長時間尖峰。AWOOOI 非 host job 改用
# awoooi-ubuntu，CD / post-deploy 保留 awoooi-host。
python3 - "$RUNNER_DIR/config.yaml" <<'PY' || true
import sys
from pathlib import Path
path = Path(sys.argv[1])
lines = path.read_text().splitlines()
output = []
idx = 0
while idx < len(lines):
line = lines[idx]
if line.strip() == "labels:":
output.append(line)
output.append(' - "awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04"')
output.append(' - "awoooi-host:host"')
idx += 1
while idx < len(lines) and (lines[idx].startswith(" - ") or not lines[idx].strip()):
idx += 1
continue
output.append(line)
idx += 1
path.write_text("\n".join(output) + "\n")
PY
if [ "$START_GITEA_RUNNER_ALLOWED" = "1" ]; then
if systemctl list-unit-files "$RUNNER_SERVICE" >/dev/null 2>&1; then
systemctl enable --now "$RUNNER_SERVICE" >/dev/null 2>&1 || true
elif ! pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
nohup "$RUNNER_DIR/run-host-runner.sh" >> "$RUNNER_DIR/host-runner.log" 2>&1 &
fi
else
if [ "$START_GITEA_RUNNER_ON_BOOT" = "1" ]; then
log "⛔ AWOOOI_START_GITEA_RUNNER_ON_BOOT=1 但缺少 $RUNNER_ENABLE_SENTINEL；runner fail-closed"
else
log "⏸️ Gitea host runner 維持停用；需同時設定 AWOOOI_START_GITEA_RUNNER_ON_BOOT=1 與建立 $RUNNER_ENABLE_SENTINEL 才允許 startup 啟動"
fi
ensure_host_runner_fail_closed
fi
# 已停用 Docker-wrapped runner；避免它搶走 host label job。
# 若手動執行此 recovery script 時仍有 task container，送 SIGINT
# 讓 act_runner drain，不再接新 job，並等手上的 job 收尾。
docker update --restart=no gitea-runner >/dev/null 2>&1 || true
if docker ps --format '{{.Names}}' | grep -Eq '^(GITEA-ACTIONS-|[A-Za-z0-9][A-Za-z0-9_.-]*-(cd|code-review)-[0-9]+-)'; then
log "⚠️ Gitea Actions task container still running; draining docker-wrapped gitea-runner"
docker kill --signal=SIGINT gitea-runner >/dev/null 2>&1 || true
else
docker stop -t 3700 gitea-runner >/dev/null 2>&1 || true
fi
sleep 15
# 驗證 runner 已連線 Gitea
if pgrep -f "$RUNNER_DIR/act_runner daemon" >/dev/null; then
log "⚠️ Gitea host act_runner 目前正在執行；請確認是否為受控限流 / 搬遷後狀態"
elif [ "$START_GITEA_RUNNER_ALLOWED" = "1" ]; then
log "⚠️ Gitea host act_runner 可能尚未啟動，查看: $RUNNER_DIR/host-runner.log"
else
log "✅ Gitea host act_runner 維持 inactive 降壓狀態"
fi
else
log "⚠️ 找不到 act-runner binary/config: $RUNNER_DIR"
fi
if [ "$START_CD_LANE_ALLOWED" = "1" ]; then
install_controlled_cd_lane_drain_unit
systemctl daemon-reload >/dev/null 2>&1 || true
systemctl unmask "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
systemctl enable --now "$CD_LANE_DRAIN_SERVICE" >/dev/null 2>&1 || true
ensure_controlled_cd_lane_open
else
log "✅ controlled cd-lane remains closed because guardrails failed or AWOOOI_START_CONTROLLED_CD_LANE=0"
fi
# ──────────────────────────────────────────────
# STEP 7: Sentry（Error Tracking）
# 2026-04-05 Claude Code: 加入 — 解決重開機後 Sentry 未自動啟動
# 安裝位置: /opt/sentry (2026-03-24 已安裝)
# DSN: awoooi-web :2, awoooi-api :3 (見 memory/project_sentry_full_integration.md)
#
# 已知重開機後損壞問題 (2026-04-05 事故記錄):
# - sentry-postgres: WAL 損壞 → pg_resetwal -f
# - sentry-redis: dump.rdb 損壞 → 刪除重建 (redis 是 session cache，可丟失)
# - sentry-clickhouse: system table parts 損壞 → 刪除對應 store/ 目錄
# ──────────────────────────────────────────────
log "[7/7] 啟動 Sentry..."
SENTRY_DIR="/opt/sentry"
if [ -d "$SENTRY_DIR" ]; then
cd "$SENTRY_DIR"
# 修復 Sentry PostgreSQL WAL (若損壞)
SENTRY_PG_VOL="sentry-postgres"
PG_RUNNING=$(docker ps --filter name=sentry-self-hosted-postgres-1 --format "{{.Status}}" 2>/dev/null | head -1)
if echo "$PG_RUNNING" | grep -q "Restarting"; then
log "⚠️ sentry-postgres 損壞，嘗試 pg_resetwal 修復..."
docker run --rm -u 999 -v ${SENTRY_PG_VOL}:/var/lib/postgresql/data \
postgres:14 pg_resetwal -f /var/lib/postgresql/data 2>&1 | head -3 || true
log "✅ sentry-postgres WAL 修復完成"
fi
# 修復 Sentry Redis (若 dump.rdb 損壞)
REDIS_STATUS=$(docker inspect sentry-self-hosted-redis-1 --format='{{.State.Status}}' 2>/dev/null || echo "missing")
if [ "$REDIS_STATUS" = "restarting" ]; then
log "⚠️ sentry-redis 損壞，清除 dump.rdb..."
docker run --rm --user root -v sentry-redis:/data alpine \
sh -c 'rm -f /data/dump.rdb && echo cleared' 2>/dev/null || true
log "✅ sentry-redis dump.rdb 已清除"
fi
docker compose up -d 2>&1 | tail -5
log "✅ Sentry 啟動指令已發送 (啟動約需 2-3 分鐘)"
sleep 20
# 非阻塞驗證：Sentry 啟動慢，只做快速健康檢查
HTTP_CODE=$(curl -sf --max-time 10 -o /dev/null -w "%{http_code}" http://localhost:9000/ 2>/dev/null || echo "0")
if echo "$HTTP_CODE" | grep -qE "^(200|302|400)$"; then
log "✅ Sentry 已回應 (HTTP $HTTP_CODE)"
else
log "⚠️ Sentry 尚未就緒（正常現象，通常需 2-3 分鐘，HTTP=$HTTP_CODE）"
fi
else
log "⚠️ 找不到 Sentry 目錄: $SENTRY_DIR"
fi
# ──────────────────────────────────────────────
# 完成
# ──────────────────────────────────────────────
log "=== 192.168.0.110 啟動序列完成 ==="
log "Harbor: http://192.168.0.110:5000"
log "Gitea: http://192.168.0.110:3001"
log "Grafana: http://192.168.0.110:3002"
log "Alertmanager: http://192.168.0.110:9093"
log "Gitea Runner: systemctl --user status gitea-act-runner-host.service"
log "Sentry: http://192.168.0.110:9000"
exit 0
Reference in New Issue View Git Blame Copy Permalink