322 lines
11 KiB
Bash
322 lines
11 KiB
Bash
#!/bin/bash
|
|
# =============================================================================
|
|
|
|
# Production backup entrypoints share this stable gate. Agent99 takes the
|
|
# exclusive side while promoting a complete, digest-bound runtime bundle.
|
|
resolve_backup_runtime_source_path() {
|
|
local raw_path="$1"
|
|
local candidate=""
|
|
local resolved=""
|
|
local resolved_directory=""
|
|
[ -n "${raw_path}" ] || return 1
|
|
case "${raw_path}" in
|
|
/*) candidate="${raw_path}" ;;
|
|
*) candidate="${PWD}/${raw_path}" ;;
|
|
esac
|
|
if command -v realpath >/dev/null 2>&1; then
|
|
resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved=""
|
|
if [ -n "${resolved}" ]; then
|
|
printf '%s\n' "${resolved}"
|
|
return 0
|
|
fi
|
|
fi
|
|
if command -v readlink >/dev/null 2>&1; then
|
|
resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved=""
|
|
if [ -n "${resolved}" ]; then
|
|
printf '%s\n' "${resolved}"
|
|
return 0
|
|
fi
|
|
fi
|
|
[ ! -L "${candidate}" ] || return 1
|
|
resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1
|
|
[ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1
|
|
printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")"
|
|
}
|
|
|
|
backup_runtime_inherited_fd_status() {
|
|
local expected_path="$1"
|
|
local fd_path=""
|
|
local fd_target=""
|
|
if [ -d "/proc/$$/fd" ]; then
|
|
fd_path="/proc/$$/fd/197"
|
|
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
|
fd_target="$(resolve_backup_runtime_source_path "${fd_path}")" || return 2
|
|
[ "${fd_target}" = "${expected_path}" ] || return 2
|
|
return 0
|
|
fi
|
|
fd_path="/dev/fd/197"
|
|
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
|
command -v python3 >/dev/null 2>&1 || return 2
|
|
fd_target="$(python3 - <<'PY'
|
|
import fcntl
|
|
import os
|
|
|
|
try:
|
|
target = os.readlink("/dev/fd/197")
|
|
except OSError:
|
|
value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
|
target = os.fsdecode(value.split(b"\0", 1)[0])
|
|
print(os.path.realpath(target))
|
|
PY
|
|
)" || return 2
|
|
[ "${fd_target}" = "${expected_path}" ] || return 2
|
|
}
|
|
|
|
acquire_backup_runtime_shared_lock() {
|
|
local caller_path=""
|
|
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
|
|
local lock_canonical=""
|
|
local inherited_fd_status=0
|
|
caller_path="$(resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}")" || {
|
|
echo "backup runtime gate unavailable: caller identity unresolved" >&2
|
|
return 69
|
|
}
|
|
case "${caller_path}" in
|
|
/backup/scripts/*) ;;
|
|
*) return 0 ;;
|
|
esac
|
|
command -v flock >/dev/null 2>&1 || {
|
|
echo "backup runtime gate unavailable: flock missing" >&2
|
|
return 69
|
|
}
|
|
[ ! -L "${lock_path}" ] || {
|
|
echo "backup runtime gate unsafe: lock is a symlink" >&2
|
|
return 69
|
|
}
|
|
(umask 077; : >> "${lock_path}") || return 69
|
|
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
|
|
lock_canonical="$(resolve_backup_runtime_source_path "${lock_path}")" || {
|
|
echo "backup runtime gate unavailable: lock identity unresolved" >&2
|
|
return 69
|
|
}
|
|
if backup_runtime_inherited_fd_status "${lock_canonical}"; then
|
|
:
|
|
else
|
|
inherited_fd_status=$?
|
|
if [ "${inherited_fd_status}" -eq 1 ]; then
|
|
exec 197>>"${lock_path}"
|
|
else
|
|
echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2
|
|
return 69
|
|
fi
|
|
fi
|
|
flock -s -w 60 197 || {
|
|
echo "backup runtime deployment is active; retry later" >&2
|
|
return 75
|
|
}
|
|
}
|
|
|
|
acquire_backup_runtime_shared_lock || {
|
|
runtime_lock_status=$?
|
|
return "${runtime_lock_status}" 2>/dev/null || exit "${runtime_lock_status}"
|
|
}
|
|
# WOOO AIOps - 備份共用函式庫
|
|
# 版本: 1.0.0
|
|
# 建立日期: 2026-03-12
|
|
# =============================================================================
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 配置區 (待 CEO 提供 B2 帳號後更新)
|
|
# -----------------------------------------------------------------------------
|
|
export BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
|
export BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}"
|
|
export RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}"
|
|
|
|
# Backblaze B2 配置 (待填入)
|
|
export B2_ACCOUNT_ID="" # 待 CEO 提供
|
|
export B2_APPLICATION_KEY="" # 待 CEO 提供
|
|
export B2_BUCKET="wooo-aiops-backup"
|
|
|
|
# ClawBot 通知 Webhook
|
|
export CLAWBOT_WEBHOOK="http://192.168.0.188:8088/api/v1/webhook/custom"
|
|
|
|
# 保留策略 (GFS 祖父子)
|
|
export KEEP_DAILY=30 # 2026-04-05 Claude Code: 延長保留 (原7→30)
|
|
export KEEP_WEEKLY=12 # 2026-04-05 Claude Code: 延長保留 (原4→12)
|
|
export KEEP_MONTHLY=24 # 2026-04-05 Claude Code: 延長保留 (原6→24)
|
|
|
|
# Ephemeral backup containers must remain bounded even when cron has no env file.
|
|
export BACKUP_DOCKER_CPUS="${BACKUP_DOCKER_CPUS:-1.0}"
|
|
export BACKUP_DOCKER_MEMORY="${BACKUP_DOCKER_MEMORY:-1g}"
|
|
export BACKUP_DOCKER_MEMORY_SWAP="${BACKUP_DOCKER_MEMORY_SWAP:-1g}"
|
|
|
|
# Cron and service managers do not always preserve HOME. Bind rclone to the
|
|
# runtime account's config path when the operator did not provide an explicit
|
|
# RCLONE_CONFIG, without reading or printing the credential-bearing file.
|
|
bind_rclone_config_to_runtime_user() {
|
|
[ -z "${RCLONE_CONFIG:-}" ] || return 0
|
|
command -v getent >/dev/null 2>&1 || return 0
|
|
|
|
local runtime_home
|
|
local candidate
|
|
runtime_home="$(getent passwd "$(id -u)" 2>/dev/null | awk -F: 'NR == 1 {print $6}')"
|
|
candidate="${runtime_home}/.config/rclone/rclone.conf"
|
|
if [ -n "${runtime_home}" ] && [ -r "${candidate}" ]; then
|
|
export RCLONE_CONFIG="${candidate}"
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 日誌函式
|
|
# -----------------------------------------------------------------------------
|
|
log() {
|
|
local level="$1"
|
|
local message="$2"
|
|
local timestamp=$(date "+%Y-%m-%d %H:%M:%S")
|
|
echo "[${timestamp}] [${level}] ${message}" | tee -a "${BACKUP_LOG_DIR}/backup.log"
|
|
}
|
|
|
|
log_info() { log "INFO" "$1"; }
|
|
log_warn() { log "WARN" "$1"; }
|
|
log_error() { log "ERROR" "$1"; }
|
|
log_success() { log "SUCCESS" "$1"; }
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 通知函式
|
|
# -----------------------------------------------------------------------------
|
|
notify_clawbot() {
|
|
local status="$1"
|
|
local service="$2"
|
|
local message="$3"
|
|
local duration="${4:-0}"
|
|
|
|
# 2026-04-05 Claude Code: 正確的 /webhook/custom payload + severity 依狀態
|
|
local severity="info"
|
|
[ "$status" = "warning" ] && severity="warning"
|
|
[ "$status" = "failed" ] && severity="critical"
|
|
|
|
if command -v curl &> /dev/null; then
|
|
curl -s -X POST "${CLAWBOT_WEBHOOK}" \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"name\":\"Backup.${service}\",\"severity\":\"${severity}\",\"service\":\"${service}\",\"description\":\"[${status}] ${message} (${duration}s)\"}" \
|
|
--connect-timeout 5 2>/dev/null || true
|
|
fi
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# Restic 標籤函式
|
|
# -----------------------------------------------------------------------------
|
|
get_app_version() {
|
|
local service="$1"
|
|
case "$service" in
|
|
gitea)
|
|
docker exec gitea gitea --version 2>/dev/null | grep -oP "\\d+\\.\\d+\\.\\d+" | head -1 || echo "unknown"
|
|
;;
|
|
harbor)
|
|
cat /opt/harbor/harbor.yml 2>/dev/null | grep -oP "version: \\K.*" || echo "unknown"
|
|
;;
|
|
momo)
|
|
echo "1.0.0" # MOMO 版本固定或從配置讀取
|
|
;;
|
|
*)
|
|
echo "unknown"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
get_git_hash() {
|
|
local service="$1"
|
|
case "$service" in
|
|
gitea)
|
|
cd /var/lib/gitea 2>/dev/null && git rev-parse --short HEAD 2>/dev/null || echo "none"
|
|
;;
|
|
*)
|
|
echo "none"
|
|
;;
|
|
esac
|
|
}
|
|
|
|
build_tags() {
|
|
local service="$1"
|
|
local version=$(get_app_version "$service")
|
|
local git_hash=$(get_git_hash "$service")
|
|
local timestamp=$(date "+%Y%m%d_%H%M%S")
|
|
|
|
echo "--tag service:${service} --tag version:${version} --tag git:${git_hash} --tag timestamp:${timestamp}"
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 備份驗證函式
|
|
# -----------------------------------------------------------------------------
|
|
verify_backup() {
|
|
local repo="$1"
|
|
local snapshot_id="$2"
|
|
local snapshot_rows snapshot_files
|
|
|
|
if ! [[ "${snapshot_id}" =~ ^[0-9a-fA-F]{8,64}$ ]]; then
|
|
log_error "無效 snapshot id"
|
|
return 64
|
|
fi
|
|
log_info "驗證備份快照: ${snapshot_id}"
|
|
snapshot_rows="$(restic -r "${repo}" snapshots "${snapshot_id}" --json \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null)" || return 1
|
|
if ! printf '%s' "${snapshot_rows}" | python3 -c \
|
|
'import json,sys; rows=json.load(sys.stdin); raise SystemExit(0 if len(rows) == 1 else 1)'; then
|
|
log_error "指定 snapshot 無法唯一讀回: ${snapshot_id}"
|
|
return 1
|
|
fi
|
|
snapshot_files="$(restic -r "${repo}" ls "${snapshot_id}" --json \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
|
python3 -c 'import json,sys
|
|
count=0
|
|
for line in sys.stdin:
|
|
try: row=json.loads(line)
|
|
except json.JSONDecodeError: continue
|
|
if row.get("type") == "file": count += 1
|
|
print(count)')" || return 1
|
|
if [ "${snapshot_files}" -le 0 ]; then
|
|
log_error "指定 snapshot 不含可還原檔案: ${snapshot_id}"
|
|
return 1
|
|
fi
|
|
restic -r "${repo}" check --read-data-subset=1% \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 清理函式 (GFS 策略)
|
|
# -----------------------------------------------------------------------------
|
|
cleanup_old_backups() {
|
|
local repo="$1"
|
|
local mode="${BACKUP_RETENTION_MODE:-gfs}"
|
|
local keep_last="${BACKUP_KEEP_LAST:-1}"
|
|
|
|
if [ "${mode}" = "latest" ]; then
|
|
case "${keep_last}" in
|
|
''|*[!0-9]*|0) log_error "BACKUP_KEEP_LAST 必須是正整數"; return 64 ;;
|
|
esac
|
|
log_info "執行 latest-only 清理策略 keep-last=${keep_last}"
|
|
restic -r "${repo}" forget \
|
|
--keep-last "${keep_last}" \
|
|
--prune \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1
|
|
return $?
|
|
fi
|
|
if [ "${mode}" != "gfs" ]; then
|
|
log_error "未知 BACKUP_RETENTION_MODE: ${mode}"
|
|
return 64
|
|
fi
|
|
log_info "執行 GFS 清理策略"
|
|
restic -r "${repo}" forget \
|
|
--keep-daily "${KEEP_DAILY}" \
|
|
--keep-weekly "${KEEP_WEEKLY}" \
|
|
--keep-monthly "${KEEP_MONTHLY}" \
|
|
--prune \
|
|
--password-file "${RESTIC_PASSWORD_FILE}" 2>&1
|
|
}
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# 檢查配置
|
|
# -----------------------------------------------------------------------------
|
|
check_b2_config() {
|
|
if [ -z "${B2_ACCOUNT_ID}" ] || [ -z "${B2_APPLICATION_KEY}" ]; then
|
|
log_warn "B2 配置未設定，僅執行本地備份"
|
|
return 1
|
|
fi
|
|
return 0
|
|
}
|
|
|
|
# 初始化日誌目錄
|
|
mkdir -p "${BACKUP_LOG_DIR}"
|
|
|
|
log_info "共用函式庫載入完成 (v1.0.0)"
|