fix(ops): close Gitea backup restore verification
This commit is contained in:
ogt
@@ -221,8 +221,10 @@ configured_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_conf
|
||||
configured_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_configured" 0)"
|
||||
script_missing_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_script_present" 0)"
|
||||
script_missing_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_script_present" 0)"
|
||||
fresh_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")"
|
||||
fresh_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")"
|
||||
monitored_total_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh")"
|
||||
monitored_total_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh")"
|
||||
fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)"
|
||||
fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)"
|
||||
stale_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 0)"
|
||||
stale_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 0)"
|
||||
failed_total_110="$(metric_sum "${TEXTFILE_110}" "awoooi_backup_last_run_failed_count")"
|
||||
@@ -236,7 +238,12 @@ offsite_rclone_configured="$(awk '/^awoooi_backup_offsite_configured\{.*provider
|
||||
offsite_rclone_fresh="$(awk '/^awoooi_backup_offsite_fresh\{.*provider="rclone"/ { print $2; found=1; exit } END { if (!found) print 0 }' "${TEXTFILE_110}" 2>/dev/null || echo 0)"
|
||||
escrow_missing="$(metric_first "${TEXTFILE_110}" "awoooi_backup_dr_credential_escrow_missing_count")"
|
||||
|
||||
core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110))
|
||||
freshness_metrics_missing_110=0
|
||||
freshness_metrics_missing_188=0
|
||||
[ "${monitored_total_110}" -gt 0 ] || freshness_metrics_missing_110=1
|
||||
[ "${monitored_total_188}" -gt 0 ] || freshness_metrics_missing_188=1
|
||||
|
||||
core_blockers=$((host_110_missing + host_188_missing + configured_missing_110 + configured_missing_188 + script_missing_110 + script_missing_188 + freshness_metrics_missing_110 + freshness_metrics_missing_188 + stale_110 + stale_188 + component_failed_110 + component_failed_188 + integrity_stale_110))
|
||||
dr_warnings=0
|
||||
if [ "${offsite_configured%.*}" -lt 1 ] 2>/dev/null; then
|
||||
dr_warnings=$((dr_warnings + 1))
|
||||
@@ -265,7 +272,7 @@ missing_scripts_188="$(label_list_for_zero "${TEXTFILE_188_TMP}" "awoooi_backup_
|
||||
backup_all_ts="$(metric_value_for_label "${TEXTFILE_110}" "awoooi_backup_job_last_success_timestamp" "job" "backup_all")"
|
||||
last_backup_all="$(human_timestamp "${backup_all_ts}")"
|
||||
|
||||
message="${headline}; 110備份=${fresh_total_110}/13 fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_total_188}/2 fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
|
||||
message="${headline}; 110備份=${fresh_ok_110}/${monitored_total_110} fresh component_failed=${component_failed_110} aggregate_failed=${failed_total_110}; 188備份=${fresh_ok_188}/${monitored_total_188} fresh component_failed=${component_failed_188} aggregate_failed=${failed_total_188}; freshness_metrics_missing=${freshness_metrics_missing_110}/${freshness_metrics_missing_188}; integrity_stale=${integrity_stale_110}; offsite_configured=${offsite_configured}; offsite_fresh=${offsite_fresh}; rclone_gdrive_configured=${offsite_rclone_configured}; rclone_gdrive_fresh=${offsite_rclone_fresh}; escrow_missing=${escrow_missing}; last_backup_all=${last_backup_all}"
|
||||
|
||||
if [ "${core_blockers}" -gt 0 ]; then
|
||||
message="${message}; stale110=${stale_jobs_110:-none}; stale188=${stale_jobs_188:-none}; missing_script110=${missing_scripts_110:-none}; missing_script188=${missing_scripts_188:-none}"
|
||||
@@ -273,7 +280,7 @@ fi
|
||||
|
||||
{
|
||||
log_line "${message}"
|
||||
log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188}"
|
||||
log_line "DETAIL core_blockers=${core_blockers} dr_warnings=${dr_warnings} configured_missing_110=${configured_missing_110} configured_missing_188=${configured_missing_188} script_missing_110=${script_missing_110} script_missing_188=${script_missing_188} freshness_metrics_missing_110=${freshness_metrics_missing_110} freshness_metrics_missing_188=${freshness_metrics_missing_188}"
|
||||
} | tee -a "${LOG_DIR}/backup-status.log"
|
||||
|
||||
if [ "${NOTIFY}" -eq 1 ]; then
|
||||
@@ -285,7 +292,7 @@ if [ "${NOTIFY}" -eq 1 ]; then
|
||||
failed_interval_hours="${BACKUP_STATUS_FAILED_INTERVAL_HOURS:-6}"
|
||||
now_ts="$(date +%s)"
|
||||
notify_fingerprint="$(
|
||||
printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \
|
||||
printf '%s' "status=${status};core=${core_blockers};dr=${dr_warnings};cm110=${configured_missing_110};cm188=${configured_missing_188};sm110=${script_missing_110};sm188=${script_missing_188};fm110=${freshness_metrics_missing_110};fm188=${freshness_metrics_missing_188};fresh110=${fresh_ok_110}/${monitored_total_110};fresh188=${fresh_ok_188}/${monitored_total_188};stale110=${stale_jobs_110:-none};stale188=${stale_jobs_188:-none};offsite=${offsite_configured}:${offsite_fresh};escrow=${escrow_missing}" \
|
||||
| cksum \
|
||||
| awk '{print $1}'
|
||||
)"
|
||||
|
||||
@@ -190,6 +190,18 @@ main() {
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
else
|
||||
if [ "${name}" = "gitea" ] && [ -x "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" ]; then
|
||||
log_info "Gitea specialized restore drill: archive CRC, required assets, and live inventory match"
|
||||
if "${BACKUP_BASE}/scripts/verify-gitea-backup.sh" --no-notify >> "${LOG_FILE}" 2>&1; then
|
||||
log_success "repo ${name} specialized restore drill OK"
|
||||
echo "repo=${name} status=restore_drill_specialized_ok snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
|
||||
else
|
||||
log_error "repo ${name} specialized restore drill failed"
|
||||
echo "repo=${name} status=restore_drill_specialized_failed snapshots=${count} latest=${latest_ts}" >> "${LOG_FILE}"
|
||||
failed=$((failed + 1))
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
local sample
|
||||
local sample_out
|
||||
sample=$(MAX_SAMPLE_BYTES="${MAX_SAMPLE_BYTES}" sample_path_for_repo "${repo}")
|
||||
|
||||
172
scripts/backup/gitea-bundle-backup-sync-188.sh
Executable file
172
scripts/backup/gitea-bundle-backup-sync-188.sh
Executable file
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Generate Gitea Git bundles on host 110 from the local Gitea container, sync
|
||||
# the complete set to 188 without deletion, then run an isolated sample restore.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
LOCAL_OUTPUT_ROOT="${GITEA_BUNDLE_LOCAL_ROOT:-/backup/gitea-bundles}"
|
||||
LOCAL_CONTAINER="${GITEA_BUNDLE_LOCAL_CONTAINER:-gitea}"
|
||||
REMOTE_HOST="${GITEA_BUNDLE_REMOTE_HOST:-ollama@192.168.0.188}"
|
||||
REMOTE_OUTPUT_ROOT="${GITEA_BUNDLE_REMOTE_ROOT:-/home/ollama/backup/110/gitea/git-bundles}"
|
||||
REMOTE_VERIFY_SCRIPT="${GITEA_BUNDLE_REMOTE_VERIFY_SCRIPT:-/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh}"
|
||||
REMOTE_EXPORTER_SCRIPT="${GITEA_BUNDLE_REMOTE_EXPORTER_SCRIPT:-/home/ollama/scripts/backup-health-textfile-exporter.py}"
|
||||
BUNDLE_SCRIPT="${GITEA_BUNDLE_SCRIPT:-${SCRIPT_DIR}/gitea-repo-bundle-backup.sh}"
|
||||
STATUS_FILE="${GITEA_BUNDLE_SYNC_STATUS_FILE:-/backup/status/gitea-bundle-sync-188.status}"
|
||||
LOCK_FILE="${GITEA_BUNDLE_SYNC_LOCK_FILE:-${LOCAL_OUTPUT_ROOT}/.sync-188.lock}"
|
||||
START_TIME="$(date +%s)"
|
||||
SSH_OPTS=(-o BatchMode=yes -o ConnectTimeout=8 -o StrictHostKeyChecking=accept-new)
|
||||
phase="preflight"
|
||||
promoted=0
|
||||
previous_complete=""
|
||||
previous_latest=""
|
||||
expected_count=0
|
||||
stamp="unknown"
|
||||
|
||||
mkdir -p "${LOCAL_OUTPUT_ROOT}" "$(dirname "${STATUS_FILE}")"
|
||||
exec 9>"${LOCK_FILE}"
|
||||
if ! flock -n 9; then
|
||||
printf 'SKIP Gitea bundle sync already running\n'
|
||||
exit 0
|
||||
fi
|
||||
|
||||
write_status() {
|
||||
local success="$1"
|
||||
local reason="$2"
|
||||
local temp_status="${STATUS_FILE}.tmp.$$"
|
||||
{
|
||||
printf 'timestamp=%s\n' "$(date +%s)"
|
||||
printf 'success=%s\n' "${success}"
|
||||
printf 'reason=%s\n' "${reason}"
|
||||
printf 'phase=%s\n' "${phase}"
|
||||
printf 'snapshot_stamp=%s\n' "${stamp}"
|
||||
printf 'expected_repo_count=%s\n' "${expected_count}"
|
||||
printf 'duration_seconds=%s\n' "$(($(date +%s) - START_TIME))"
|
||||
} >"${temp_status}"
|
||||
chmod 600 "${temp_status}"
|
||||
mv "${temp_status}" "${STATUS_FILE}"
|
||||
}
|
||||
|
||||
refresh_remote_exporter() {
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'" \
|
||||
>/dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
rollback_remote_links() {
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
python3 - "${REMOTE_OUTPUT_ROOT}" "${previous_complete}" "${previous_latest}" <<'PY'
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
root = Path(sys.argv[1])
|
||||
previous = {
|
||||
root / "latest-private-complete": sys.argv[2],
|
||||
root / "latest": sys.argv[3],
|
||||
}
|
||||
for link, target in previous.items():
|
||||
if target:
|
||||
temporary = link.with_name(f".{link.name}.rollback.{os.getpid()}")
|
||||
temporary.unlink(missing_ok=True)
|
||||
temporary.symlink_to(target)
|
||||
os.replace(temporary, link)
|
||||
elif link.is_symlink():
|
||||
link.unlink()
|
||||
PY
|
||||
}
|
||||
|
||||
finish() {
|
||||
local rc=$?
|
||||
trap - EXIT
|
||||
if [ "${rc}" -eq 0 ] && [ "${phase}" = "complete" ]; then
|
||||
write_status 1 verified
|
||||
else
|
||||
if [ "${promoted}" -eq 1 ]; then
|
||||
rollback_remote_links >/dev/null 2>&1 || true
|
||||
fi
|
||||
refresh_remote_exporter
|
||||
write_status 0 "${phase}_failed"
|
||||
fi
|
||||
exit "${rc}"
|
||||
}
|
||||
trap finish EXIT
|
||||
|
||||
[ -x "${BUNDLE_SCRIPT}" ] || { printf 'missing bundle script: %s\n' "${BUNDLE_SCRIPT}" >&2; exit 1; }
|
||||
command -v docker >/dev/null
|
||||
command -v rsync >/dev/null
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"test -x '${REMOTE_VERIFY_SCRIPT}' && test -x '${REMOTE_EXPORTER_SCRIPT}'"
|
||||
|
||||
phase="bundle_local_container"
|
||||
"${BUNDLE_SCRIPT}" \
|
||||
--local-container "${LOCAL_CONTAINER}" \
|
||||
--output-root "${LOCAL_OUTPUT_ROOT}"
|
||||
|
||||
local_snapshot="$(readlink -f "${LOCAL_OUTPUT_ROOT}/latest-private-complete")"
|
||||
stamp="$(basename "${local_snapshot}")"
|
||||
manifest="${local_snapshot}/manifest.tsv"
|
||||
[ -f "${manifest}" ]
|
||||
expected_count="$(awk -F '\t' 'NR > 1 {total += 1; if ($2 != "ok") failed += 1} END {if (failed > 0 || total == 0) exit 1; print total}' "${manifest}")"
|
||||
|
||||
phase="sync_complete_set_to_188"
|
||||
previous_complete="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest-private-complete' 2>/dev/null || true")"
|
||||
previous_latest="$(ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "readlink -f '${REMOTE_OUTPUT_ROOT}/latest' 2>/dev/null || true")"
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" "mkdir -p '${REMOTE_OUTPUT_ROOT}/${stamp}'"
|
||||
rsync -a --checksum \
|
||||
-e "ssh ${SSH_OPTS[*]}" \
|
||||
"${local_snapshot}/" \
|
||||
"${REMOTE_HOST}:${REMOTE_OUTPUT_ROOT}/${stamp}/"
|
||||
|
||||
phase="remote_manifest_verification"
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
python3 - "${REMOTE_OUTPUT_ROOT}/${stamp}" "${expected_count}" <<'PY'
|
||||
import csv
|
||||
import hashlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
snapshot = Path(sys.argv[1])
|
||||
expected = int(sys.argv[2])
|
||||
with (snapshot / "manifest.tsv").open(encoding="utf-8", newline="") as handle:
|
||||
rows = list(csv.DictReader(handle, delimiter="\t"))
|
||||
ok = len(rows) == expected and expected > 0
|
||||
verified_checksums = 0
|
||||
for row in rows:
|
||||
bundle = snapshot / Path(row.get("bundle", "")).name
|
||||
checksum = snapshot / Path(row.get("checksum", "")).name
|
||||
row_ok = row.get("status") == "ok" and bundle.is_file() and checksum.is_file()
|
||||
if row_ok:
|
||||
expected_digest = checksum.read_text(encoding="utf-8", errors="replace").split()[0].lower()
|
||||
digest = hashlib.sha256()
|
||||
with bundle.open("rb") as handle:
|
||||
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
||||
digest.update(chunk)
|
||||
row_ok = len(expected_digest) == 64 and digest.hexdigest() == expected_digest
|
||||
if row_ok:
|
||||
verified_checksums += 1
|
||||
ok = ok and row_ok
|
||||
print(
|
||||
f"remote_manifest_expected={expected} remote_manifest_rows={len(rows)} "
|
||||
f"remote_manifest_checksum_ok={verified_checksums} remote_manifest_ok={int(ok)}"
|
||||
)
|
||||
raise SystemExit(0 if ok else 1)
|
||||
PY
|
||||
|
||||
phase="remote_promote_complete"
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"for link in '${REMOTE_OUTPUT_ROOT}/latest-private-complete' '${REMOTE_OUTPUT_ROOT}/latest'; do test ! -e \"\$link\" || test -L \"\$link\" || exit 1; done"
|
||||
promoted=1
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest-private-complete' && ln -sfn '${REMOTE_OUTPUT_ROOT}/${stamp}' '${REMOTE_OUTPUT_ROOT}/latest'"
|
||||
|
||||
phase="remote_sample_restore_fsck"
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"AIOPS_GITEA_BUNDLE_ROOT='${REMOTE_OUTPUT_ROOT}/latest-private-complete' AIOPS_HOST_LABEL=188 '${REMOTE_VERIFY_SCRIPT}' --write-textfile"
|
||||
|
||||
phase="refresh_remote_backup_health"
|
||||
ssh "${SSH_OPTS[@]}" "${REMOTE_HOST}" \
|
||||
"AIOPS_HOST_LABEL=188 NODE_EXPORTER_TEXTFILE_DIR=/home/ollama/node_exporter_textfiles '${REMOTE_EXPORTER_SCRIPT}'"
|
||||
|
||||
phase="complete"
|
||||
printf 'GITEA_BUNDLE_SYNC_COMPLETE=1 expected=%s evidence=%s\n' "${expected_count}" "${STATUS_FILE}"
|
||||
@@ -7,6 +7,8 @@ set -euo pipefail
|
||||
|
||||
GITEA_BASE_URL="${GITEA_BASE_URL:-http://192.168.0.110:3001}"
|
||||
GIT_REMOTE_BASE="${GIT_REMOTE_BASE:-}"
|
||||
LOCAL_GITEA_CONTAINER="${LOCAL_GITEA_CONTAINER:-}"
|
||||
LOCAL_REPO_ROOT="${LOCAL_REPO_ROOT:-/data/git/repositories}"
|
||||
OUTPUT_ROOT="${OUTPUT_ROOT:-/home/ollama/backup/110/gitea/git-bundles}"
|
||||
TIMEOUT_LS_REMOTE_SECONDS="${TIMEOUT_LS_REMOTE_SECONDS:-60}"
|
||||
TIMEOUT_CLONE_SECONDS="${TIMEOUT_CLONE_SECONDS:-180}"
|
||||
@@ -27,6 +29,11 @@ Options:
|
||||
--git-remote-base URL
|
||||
Git remote base, such as ssh://git@192.168.0.110:2222.
|
||||
When set, it is used instead of --gitea-base.
|
||||
--local-container NAME
|
||||
Read bare repos directly from a local Gitea container.
|
||||
This avoids tokens and SSH keys on the Gitea host.
|
||||
--local-repo-root PATH
|
||||
Repo root inside the local container.
|
||||
--stamp STAMP Output directory stamp. Default: current local timestamp.
|
||||
--dry-run Print target URLs without cloning or writing bundles.
|
||||
-h, --help Show this help.
|
||||
@@ -56,6 +63,14 @@ while [ "$#" -gt 0 ]; do
|
||||
shift
|
||||
GIT_REMOTE_BASE="${1:?--git-remote-base requires URL}"
|
||||
;;
|
||||
--local-container)
|
||||
shift
|
||||
LOCAL_GITEA_CONTAINER="${1:?--local-container requires NAME}"
|
||||
;;
|
||||
--local-repo-root)
|
||||
shift
|
||||
LOCAL_REPO_ROOT="${1:?--local-repo-root requires PATH}"
|
||||
;;
|
||||
--stamp)
|
||||
shift
|
||||
STAMP="${1:?--stamp requires value}"
|
||||
@@ -104,7 +119,11 @@ repo_remote_url() {
|
||||
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
for repo in "${REPOS[@]}"; do
|
||||
printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")"
|
||||
if [ -n "$LOCAL_GITEA_CONTAINER" ]; then
|
||||
printf 'DRY_RUN repo=%s transport=container container=%s\n' "$repo" "$LOCAL_GITEA_CONTAINER"
|
||||
else
|
||||
printf 'DRY_RUN repo=%s url=%s\n' "$repo" "$(repo_remote_url "$repo")"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
@@ -116,19 +135,80 @@ mkdir -p "$backup_root"
|
||||
trap 'rm -rf "$tmp_root"' EXIT
|
||||
|
||||
printf 'repo\tstatus\thead_count\tbundle\tchecksum\n' > "$manifest"
|
||||
failed_count=0
|
||||
success_count=0
|
||||
|
||||
for repo in "${REPOS[@]}"; do
|
||||
slug="${repo//\//__}"
|
||||
url="$(repo_remote_url "$repo")"
|
||||
refs_file="${backup_root}/${slug}.refs"
|
||||
err_file="${backup_root}/${slug}.err"
|
||||
bundle_file="${backup_root}/${slug}.bundle"
|
||||
checksum_file="${bundle_file}.sha256"
|
||||
repo_tmp="${tmp_root}/${slug}.git"
|
||||
|
||||
if [ -n "$LOCAL_GITEA_CONTAINER" ]; then
|
||||
container_repo="${LOCAL_REPO_ROOT%/}/${repo}.git"
|
||||
container_bundle="/tmp/awoooi-${slug}-${STAMP}.bundle"
|
||||
if ! docker exec -u git "$LOCAL_GITEA_CONTAINER" test -d "$container_repo" >/dev/null 2>&1; then
|
||||
owner="${repo%%/*}"
|
||||
name="${repo#*/}"
|
||||
resolved_paths="$(docker exec -u git "$LOCAL_GITEA_CONTAINER" sh -c \
|
||||
'find "$1/$2" -mindepth 1 -maxdepth 1 -type d -iname "$3.git" -print' \
|
||||
sh "${LOCAL_REPO_ROOT%/}" "$owner" "$name" 2>>"$err_file" || true)"
|
||||
resolved_count="$(printf '%s\n' "$resolved_paths" | awk 'NF {count += 1} END {print count + 0}')"
|
||||
if [ "$resolved_count" -ne 1 ]; then
|
||||
printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
container_repo="$resolved_paths"
|
||||
fi
|
||||
if ! timeout "$TIMEOUT_LS_REMOTE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
|
||||
git -C "$container_repo" for-each-ref --format='%(objectname)%09%(refname)' \
|
||||
>"$refs_file" 2>"$err_file"; then
|
||||
printf '%s\tlocal_repo_read_failed\t0\t\t\n' "$repo" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
head_count="$(wc -l < "$refs_file" | tr -d ' ')"
|
||||
if ! timeout "$TIMEOUT_BUNDLE_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
|
||||
git -C "$container_repo" bundle create "$container_bundle" --all \
|
||||
>/dev/null 2>>"$err_file"; then
|
||||
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
|
||||
printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! timeout "$TIMEOUT_VERIFY_SECONDS" docker exec -u git "$LOCAL_GITEA_CONTAINER" \
|
||||
git -C "$container_repo" bundle verify "$container_bundle" \
|
||||
>/dev/null 2>>"$err_file"; then
|
||||
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
|
||||
printf '%s\tverify_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! docker cp "${LOCAL_GITEA_CONTAINER}:${container_bundle}" "$bundle_file" 2>>"$err_file"; then
|
||||
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
|
||||
printf '%s\tcopy_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
docker exec -u git "$LOCAL_GITEA_CONTAINER" rm -f "$container_bundle" >/dev/null 2>&1 || true
|
||||
(sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file"
|
||||
printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest"
|
||||
success_count=$((success_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
url="$(repo_remote_url "$repo")"
|
||||
|
||||
if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_LS_REMOTE_SECONDS" \
|
||||
git ls-remote --heads "$url" >"$refs_file" 2>"$err_file"; then
|
||||
printf '%s\tls_remote_failed\t0\t\t\n' "$repo" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
@@ -137,30 +217,52 @@ for repo in "${REPOS[@]}"; do
|
||||
if ! GIT_TERMINAL_PROMPT=0 timeout "$TIMEOUT_CLONE_SECONDS" \
|
||||
git clone --mirror --quiet "$url" "$repo_tmp" 2>>"$err_file"; then
|
||||
printf '%s\tclone_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! timeout "$TIMEOUT_BUNDLE_SECONDS" \
|
||||
git -C "$repo_tmp" bundle create "$bundle_file" --all >/dev/null 2>>"$err_file"; then
|
||||
printf '%s\tbundle_failed\t%s\t\t\n' "$repo" "$head_count" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
if ! timeout "$TIMEOUT_VERIFY_SECONDS" \
|
||||
git -C "$repo_tmp" bundle verify "$bundle_file" >/dev/null 2>>"$err_file"; then
|
||||
printf '%s\tverify_failed\t%s\t%s\t\n' "$repo" "$head_count" "$bundle_file" >> "$manifest"
|
||||
failed_count=$((failed_count + 1))
|
||||
continue
|
||||
fi
|
||||
|
||||
(sha256sum "$bundle_file" 2>/dev/null || shasum -a 256 "$bundle_file") > "$checksum_file"
|
||||
rm -rf "$repo_tmp"
|
||||
printf '%s\tok\t%s\t%s\t%s\n' "$repo" "$head_count" "$bundle_file" "$checksum_file" >> "$manifest"
|
||||
success_count=$((success_count + 1))
|
||||
done
|
||||
|
||||
latest_link="${OUTPUT_ROOT%/}/latest"
|
||||
if [ ! -e "$latest_link" ] || [ -L "$latest_link" ]; then
|
||||
ln -sfn "$backup_root" "$latest_link"
|
||||
else
|
||||
echo "WARN latest path exists and is not a symlink: $latest_link" >&2
|
||||
fi
|
||||
expected_count="${#REPOS[@]}"
|
||||
cat "$manifest"
|
||||
|
||||
if [ "$failed_count" -ne 0 ] || [ "$success_count" -ne "$expected_count" ]; then
|
||||
printf 'ERROR bundle backup incomplete success=%s expected=%s failed=%s; complete links unchanged\n' \
|
||||
"$success_count" "$expected_count" "$failed_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
promotion_path_ready() {
|
||||
link="$1"
|
||||
if [ ! -e "$link" ] || [ -L "$link" ]; then
|
||||
return 0
|
||||
fi
|
||||
printf 'ERROR promotion path exists and is not a symlink: %s\n' "$link" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
latest_link="${OUTPUT_ROOT%/}/latest"
|
||||
complete_link="${OUTPUT_ROOT%/}/latest-private-complete"
|
||||
promotion_path_ready "$latest_link"
|
||||
promotion_path_ready "$complete_link"
|
||||
ln -sfn "$backup_root" "$latest_link"
|
||||
ln -sfn "$backup_root" "$complete_link"
|
||||
printf 'BUNDLE_BACKUP_COMPLETE=1 success=%s expected=%s root=%s\n' "$success_count" "$expected_count" "$backup_root"
|
||||
|
||||
@@ -19,3 +19,17 @@ def test_backup_status_keeps_aggregate_failure_out_of_core_blockers() -> None:
|
||||
assert "component_failed_188" in core_line
|
||||
assert "failed_total_110" not in core_line
|
||||
assert "failed_total_188" not in core_line
|
||||
|
||||
|
||||
def test_backup_status_uses_dynamic_healthy_over_monitored_ratio() -> None:
|
||||
text = BACKUP_STATUS.read_text(encoding="utf-8")
|
||||
core_line = next(line for line in text.splitlines() if line.startswith("core_blockers="))
|
||||
|
||||
assert 'fresh_ok_110="$(metric_count "${TEXTFILE_110}" "awoooi_backup_job_fresh" 1)"' in text
|
||||
assert 'fresh_ok_188="$(metric_count "${TEXTFILE_188_TMP}" "awoooi_backup_job_fresh" 1)"' in text
|
||||
assert "110備份=${fresh_ok_110}/${monitored_total_110}" in text
|
||||
assert "188備份=${fresh_ok_188}/${monitored_total_188}" in text
|
||||
assert "freshness_metrics_missing_110" in core_line
|
||||
assert "freshness_metrics_missing_188" in core_line
|
||||
assert "/13 fresh" not in text
|
||||
assert "/2 fresh" not in text
|
||||
|
||||
309
scripts/backup/tests/test_gitea_backup_automation.py
Normal file
309
scripts/backup/tests/test_gitea_backup_automation.py
Normal file
@@ -0,0 +1,309 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
BUNDLE_SCRIPT = ROOT / "scripts" / "backup" / "gitea-repo-bundle-backup.sh"
|
||||
RESTORE_DRILL_SCRIPT = ROOT / "scripts" / "backup" / "verify-gitea-backup.sh"
|
||||
|
||||
|
||||
def _fake_git(tmp_path: Path) -> Path:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
git = fake_bin / "git"
|
||||
git.write_text(
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
if [ -n "${FAKE_GIT_FAIL_PATTERN:-}" ] && [[ "$*" == *"${FAKE_GIT_FAIL_PATTERN}"* ]]; then
|
||||
exit 42
|
||||
fi
|
||||
if [ "$1" = "ls-remote" ]; then
|
||||
printf '0000000000000000000000000000000000000001\trefs/heads/main\n'
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "clone" ]; then
|
||||
mkdir -p "${!#}"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "create" ]; then
|
||||
printf 'synthetic bundle\n' > "$5"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "-C" ] && [ "$3" = "bundle" ] && [ "$4" = "verify" ]; then
|
||||
exit 0
|
||||
fi
|
||||
printf 'unexpected fake git invocation: %s\n' "$*" >&2
|
||||
exit 64
|
||||
"""
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
git.chmod(0o755)
|
||||
return fake_bin
|
||||
|
||||
|
||||
def _fake_docker(fake_bin: Path, tmp_path: Path) -> None:
|
||||
docker = fake_bin / "docker"
|
||||
docker.write_text(
|
||||
textwrap.dedent(
|
||||
"""\
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
root="${FAKE_DOCKER_ROOT:?}"
|
||||
if [ "$1" = "exec" ]; then
|
||||
shift 4
|
||||
if [ "$1" = "test" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "git" ] && [ "$4" = "for-each-ref" ]; then
|
||||
printf '0000000000000000000000000000000000000001\trefs/heads/main\n'
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "create" ]; then
|
||||
mkdir -p "${root}$(dirname "$6")"
|
||||
printf 'synthetic container bundle\n' > "${root}$6"
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "git" ] && [ "$4" = "bundle" ] && [ "$5" = "verify" ]; then
|
||||
exit 0
|
||||
fi
|
||||
if [ "$1" = "rm" ]; then
|
||||
rm -f "${root}${3}"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
if [ "$1" = "cp" ]; then
|
||||
source_path="${2#*:}"
|
||||
cp "${root}${source_path}" "$3"
|
||||
exit 0
|
||||
fi
|
||||
printf 'unexpected fake docker invocation: %s\n' "$*" >&2
|
||||
exit 64
|
||||
"""
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
docker.chmod(0o755)
|
||||
|
||||
|
||||
def test_bundle_backup_promotes_only_a_complete_repo_set(tmp_path: Path) -> None:
|
||||
fake_bin = _fake_git(tmp_path)
|
||||
output_root = tmp_path / "bundles"
|
||||
env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"}
|
||||
|
||||
complete = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(BUNDLE_SCRIPT),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--git-remote-base",
|
||||
"ssh://git@example.invalid:2222",
|
||||
"--stamp",
|
||||
"complete",
|
||||
"--repo",
|
||||
"owner/one",
|
||||
"--repo",
|
||||
"owner/two",
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert complete.returncode == 0, complete.stderr
|
||||
assert (output_root / "latest-private-complete").resolve() == output_root / "complete"
|
||||
assert (output_root / "latest").resolve() == output_root / "complete"
|
||||
assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in complete.stdout
|
||||
|
||||
failed = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(BUNDLE_SCRIPT),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--git-remote-base",
|
||||
"ssh://git@example.invalid:2222",
|
||||
"--stamp",
|
||||
"partial",
|
||||
"--repo",
|
||||
"owner/one",
|
||||
"--repo",
|
||||
"owner/fail",
|
||||
],
|
||||
env=env | {"FAKE_GIT_FAIL_PATTERN": "owner/fail"},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert failed.returncode != 0
|
||||
assert "complete links unchanged" in failed.stderr
|
||||
assert (output_root / "latest-private-complete").resolve() == output_root / "complete"
|
||||
assert "owner/fail\tls_remote_failed" in (output_root / "partial" / "manifest.tsv").read_text(
|
||||
encoding="utf-8"
|
||||
)
|
||||
|
||||
|
||||
def test_bundle_backup_supports_local_gitea_container_without_credentials(tmp_path: Path) -> None:
|
||||
fake_bin = _fake_git(tmp_path)
|
||||
_fake_docker(fake_bin, tmp_path)
|
||||
output_root = tmp_path / "local-bundles"
|
||||
docker_root = tmp_path / "fake-docker"
|
||||
docker_root.mkdir()
|
||||
env = os.environ | {
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"FAKE_DOCKER_ROOT": str(docker_root),
|
||||
}
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(BUNDLE_SCRIPT),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--local-container",
|
||||
"gitea",
|
||||
"--stamp",
|
||||
"local-complete",
|
||||
"--repo",
|
||||
"owner/one",
|
||||
"--repo",
|
||||
"owner/two",
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert (output_root / "latest-private-complete").resolve() == output_root / "local-complete"
|
||||
assert "BUNDLE_BACKUP_COMPLETE=1 success=2 expected=2" in result.stdout
|
||||
|
||||
|
||||
def test_bundle_backup_preflights_both_promotion_links(tmp_path: Path) -> None:
|
||||
fake_bin = _fake_git(tmp_path)
|
||||
output_root = tmp_path / "blocked-promotion"
|
||||
output_root.mkdir()
|
||||
(output_root / "latest-private-complete").mkdir()
|
||||
env = os.environ | {"PATH": f"{fake_bin}:{os.environ['PATH']}"}
|
||||
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(BUNDLE_SCRIPT),
|
||||
"--output-root",
|
||||
str(output_root),
|
||||
"--git-remote-base",
|
||||
"ssh://git@example.invalid:2222",
|
||||
"--stamp",
|
||||
"candidate",
|
||||
"--repo",
|
||||
"owner/one",
|
||||
],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
assert "promotion path exists and is not a symlink" in result.stderr
|
||||
assert not (output_root / "latest").exists()
|
||||
assert (output_root / "latest-private-complete").is_dir()
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier")
|
||||
def test_gitea_restore_drill_blocks_same_count_inventory_mismatch(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "gitea-dump.zip"
|
||||
with zipfile.ZipFile(archive, "w") as payload:
|
||||
payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n")
|
||||
payload.writestr("gitea-repo/owner/two.git/HEAD", "ref: refs/heads/main\n")
|
||||
payload.writestr("gitea-db.sql", "synthetic database placeholder\n")
|
||||
payload.writestr("config/app.ini", "synthetic config placeholder\n")
|
||||
|
||||
state_dir = tmp_path / "state"
|
||||
work_dir = tmp_path / "work"
|
||||
state_dir.mkdir()
|
||||
work_dir.mkdir()
|
||||
matching_inventory = tmp_path / "matching.txt"
|
||||
matching_inventory.write_text("owner/one.git\nowner/two.git\n", encoding="utf-8")
|
||||
mismatch_inventory = tmp_path / "mismatch.txt"
|
||||
mismatch_inventory.write_text("owner/one.git\nowner/different.git\n", encoding="utf-8")
|
||||
base_env = os.environ | {
|
||||
"GITEA_RESTORE_DRILL_TEST_MODE": "1",
|
||||
"GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive),
|
||||
"GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir),
|
||||
"GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir),
|
||||
"BACKUP_BASE": str(tmp_path),
|
||||
}
|
||||
|
||||
success = subprocess.run(
|
||||
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
|
||||
env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(matching_inventory)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert success.returncode == 0, success.stderr
|
||||
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
|
||||
assert "success=1" in status
|
||||
assert "inventory_digest_match=1" in status
|
||||
|
||||
mismatch = subprocess.run(
|
||||
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
|
||||
env=base_env | {"GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE": str(mismatch_inventory)},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert mismatch.returncode != 0
|
||||
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
|
||||
assert "success=0" in status
|
||||
assert "reason=repository_inventory_mismatch" in status
|
||||
|
||||
|
||||
@pytest.mark.skipif(shutil.which("unzip") is None, reason="unzip is required by the production verifier")
|
||||
def test_gitea_restore_drill_records_live_inventory_read_failure(tmp_path: Path) -> None:
|
||||
archive = tmp_path / "gitea-dump.zip"
|
||||
with zipfile.ZipFile(archive, "w") as payload:
|
||||
payload.writestr("gitea-repo/owner/one.git/HEAD", "ref: refs/heads/main\n")
|
||||
payload.writestr("gitea-db.sql", "synthetic database placeholder\n")
|
||||
payload.writestr("config/app.ini", "synthetic config placeholder\n")
|
||||
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
docker = fake_bin / "docker"
|
||||
docker.write_text("#!/usr/bin/env bash\nexit 42\n", encoding="utf-8")
|
||||
docker.chmod(0o755)
|
||||
state_dir = tmp_path / "state"
|
||||
work_dir = tmp_path / "work"
|
||||
state_dir.mkdir()
|
||||
work_dir.mkdir()
|
||||
result = subprocess.run(
|
||||
["bash", str(RESTORE_DRILL_SCRIPT), "--no-notify"],
|
||||
env=os.environ
|
||||
| {
|
||||
"PATH": f"{fake_bin}:{os.environ['PATH']}",
|
||||
"GITEA_RESTORE_DRILL_TEST_MODE": "1",
|
||||
"GITEA_RESTORE_DRILL_SOURCE_ZIP": str(archive),
|
||||
"GITEA_RESTORE_DRILL_STATE_DIR": str(state_dir),
|
||||
"GITEA_RESTORE_DRILL_WORK_ROOT": str(work_dir),
|
||||
"BACKUP_BASE": str(tmp_path),
|
||||
},
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
assert result.returncode != 0
|
||||
status = (state_dir / "gitea-restore-drill.status").read_text(encoding="utf-8")
|
||||
assert "success=0" in status
|
||||
assert "reason=live_inventory_read_failed" in status
|
||||
288
scripts/backup/verify-gitea-backup.sh
Executable file
288
scripts/backup/verify-gitea-backup.sh
Executable file
@@ -0,0 +1,288 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Isolated Gitea restore drill. It validates the latest dump without touching
|
||||
# the running container or exposing repository names, database rows, or config.
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
TEST_MODE="${GITEA_RESTORE_DRILL_TEST_MODE:-0}"
|
||||
CALLER_BACKUP_BASE="${BACKUP_BASE:-}"
|
||||
CALLER_BACKUP_LOG_DIR="${BACKUP_LOG_DIR:-}"
|
||||
CALLER_RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-}"
|
||||
if [ "${TEST_MODE}" != "1" ] && [ -f "${SCRIPT_DIR}/common.sh" ]; then
|
||||
# shellcheck disable=SC1091
|
||||
source "${SCRIPT_DIR}/common.sh"
|
||||
else
|
||||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||||
RESTIC_PASSWORD_FILE="${RESTIC_PASSWORD_FILE:-/nonexistent}"
|
||||
log_info() { printf '[INFO] %s\n' "$*"; }
|
||||
log_success() { printf '[SUCCESS] %s\n' "$*"; }
|
||||
log_error() { printf '[ERROR] %s\n' "$*" >&2; }
|
||||
notify_clawbot() { return 0; }
|
||||
fi
|
||||
if [ -n "${CALLER_BACKUP_BASE}" ]; then
|
||||
BACKUP_BASE="${CALLER_BACKUP_BASE}"
|
||||
BACKUP_LOG_DIR="${CALLER_BACKUP_LOG_DIR:-${BACKUP_BASE}/logs}"
|
||||
RESTIC_PASSWORD_FILE="${CALLER_RESTIC_PASSWORD_FILE:-${BACKUP_BASE}/scripts/.restic-password}"
|
||||
export BACKUP_BASE BACKUP_LOG_DIR RESTIC_PASSWORD_FILE
|
||||
fi
|
||||
|
||||
LOCAL_REPO="${GITEA_BACKUP_REPO:-${BACKUP_BASE}/gitea}"
|
||||
STATE_DIR="${GITEA_RESTORE_DRILL_STATE_DIR:-${BACKUP_BASE}/integrity}"
|
||||
WORK_ROOT="${GITEA_RESTORE_DRILL_WORK_ROOT:-${BACKUP_BASE}/tmp}"
|
||||
STATUS_FILE="${STATE_DIR}/gitea-restore-drill.status"
|
||||
MIN_FREE_BUFFER_BYTES="${GITEA_RESTORE_DRILL_MIN_FREE_BUFFER_BYTES:-2147483648}"
|
||||
NOTIFY=0
|
||||
SOURCE_ZIP="${GITEA_RESTORE_DRILL_SOURCE_ZIP:-}"
|
||||
LIVE_COUNTS="${GITEA_RESTORE_DRILL_LIVE_COUNTS:-}"
|
||||
LIVE_INVENTORY_FILE="${GITEA_RESTORE_DRILL_LIVE_INVENTORY_FILE:-}"
|
||||
START_TIME="$(date +%s)"
|
||||
WORK_DIR=""
|
||||
|
||||
while [ "$#" -gt 0 ]; do
|
||||
case "$1" in
|
||||
--notify) NOTIFY=1 ;;
|
||||
--no-notify) NOTIFY=0 ;;
|
||||
-h|--help)
|
||||
printf 'Usage: verify-gitea-backup.sh [--notify|--no-notify]\n'
|
||||
exit 0
|
||||
;;
|
||||
*)
|
||||
printf 'Unknown argument: %s\n' "$1" >&2
|
||||
exit 2
|
||||
;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
cleanup() {
|
||||
if [ -n "${WORK_DIR}" ] && [ -d "${WORK_DIR}" ]; then
|
||||
rm -rf "${WORK_DIR}"
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
low_priority() {
|
||||
if command -v ionice >/dev/null 2>&1; then
|
||||
ionice -c2 -n7 nice -n 10 "$@"
|
||||
else
|
||||
nice -n 10 "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
file_size() {
|
||||
stat -c '%s' "$1" 2>/dev/null || stat -f '%z' "$1"
|
||||
}
|
||||
|
||||
write_status() {
|
||||
local success="$1"
|
||||
local reason="$2"
|
||||
local snapshot_id="$3"
|
||||
local snapshot_timestamp="$4"
|
||||
local dump_size="$5"
|
||||
local backup_regular="$6"
|
||||
local backup_wiki="$7"
|
||||
local live_regular="$8"
|
||||
local live_wiki="$9"
|
||||
local database_present="${10}"
|
||||
local config_present="${11}"
|
||||
local crc_ok="${12}"
|
||||
local inventory_match="${13}"
|
||||
local duration
|
||||
local temp_status
|
||||
|
||||
duration=$(($(date +%s) - START_TIME))
|
||||
install -d -m 700 "${STATE_DIR}"
|
||||
temp_status="${STATUS_FILE}.tmp.$$"
|
||||
{
|
||||
printf 'timestamp=%s\n' "$(date +%s)"
|
||||
printf 'success=%s\n' "${success}"
|
||||
printf 'reason=%s\n' "${reason}"
|
||||
printf 'snapshot_id=%s\n' "${snapshot_id}"
|
||||
printf 'snapshot_timestamp=%s\n' "${snapshot_timestamp}"
|
||||
printf 'dump_size_bytes=%s\n' "${dump_size}"
|
||||
printf 'backup_regular_repo_count=%s\n' "${backup_regular}"
|
||||
printf 'backup_wiki_repo_count=%s\n' "${backup_wiki}"
|
||||
printf 'live_regular_repo_count=%s\n' "${live_regular}"
|
||||
printf 'live_wiki_repo_count=%s\n' "${live_wiki}"
|
||||
printf 'database_dump_present=%s\n' "${database_present}"
|
||||
printf 'config_present=%s\n' "${config_present}"
|
||||
printf 'archive_crc_ok=%s\n' "${crc_ok}"
|
||||
printf 'inventory_digest_match=%s\n' "${inventory_match}"
|
||||
printf 'duration_seconds=%s\n' "${duration}"
|
||||
} > "${temp_status}"
|
||||
chmod 600 "${temp_status}"
|
||||
mv "${temp_status}" "${STATUS_FILE}"
|
||||
}
|
||||
|
||||
fail_drill() {
|
||||
local reason="$1"
|
||||
local snapshot_id="${2:-unknown}"
|
||||
local snapshot_timestamp="${3:-unknown}"
|
||||
local dump_size="${4:-0}"
|
||||
local backup_regular="${5:-0}"
|
||||
local backup_wiki="${6:-0}"
|
||||
local live_regular="${7:-0}"
|
||||
local live_wiki="${8:-0}"
|
||||
local database_present="${9:-0}"
|
||||
local config_present="${10:-0}"
|
||||
local crc_ok="${11:-0}"
|
||||
local inventory_match="${12:-0}"
|
||||
|
||||
write_status 0 "${reason}" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
|
||||
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
|
||||
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||||
log_error "Gitea restore drill failed: ${reason}; evidence=${STATUS_FILE}"
|
||||
if [ "${NOTIFY}" -eq 1 ]; then
|
||||
notify_clawbot "failed" "gitea-restore-drill" "Gitea 備份還原演練失敗：${reason}" "$(($(date +%s) - START_TIME))"
|
||||
fi
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ "${TEST_MODE}" != "1" ] && { [ -n "${SOURCE_ZIP}" ] || [ -n "${LIVE_COUNTS}" ] || [ -n "${LIVE_INVENTORY_FILE}" ]; }; then
|
||||
fail_drill "test_override_rejected"
|
||||
fi
|
||||
|
||||
install -d -m 700 "${WORK_ROOT}" "${STATE_DIR}"
|
||||
WORK_DIR="$(mktemp -d "${WORK_ROOT%/}/gitea-restore-drill.XXXXXX")"
|
||||
chmod 700 "${WORK_DIR}"
|
||||
RESTORED_ZIP="${WORK_DIR}/gitea-dump.zip"
|
||||
|
||||
snapshot_id="unknown"
|
||||
snapshot_timestamp="unknown"
|
||||
dump_size=0
|
||||
|
||||
if [ "${TEST_MODE}" = "1" ]; then
|
||||
[ -f "${SOURCE_ZIP}" ] || fail_drill "test_source_zip_missing"
|
||||
cp "${SOURCE_ZIP}" "${RESTORED_ZIP}"
|
||||
dump_size="$(file_size "${RESTORED_ZIP}")"
|
||||
snapshot_id="synthetic"
|
||||
snapshot_timestamp="synthetic"
|
||||
else
|
||||
[ -d "${LOCAL_REPO}/data" ] || fail_drill "restic_repo_missing"
|
||||
[ -r "${RESTIC_PASSWORD_FILE}" ] || fail_drill "restic_password_file_unavailable"
|
||||
|
||||
snapshot_meta=""
|
||||
if ! snapshot_meta="$(
|
||||
restic -r "${LOCAL_REPO}" snapshots --latest 1 --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
||||
python3 -c 'import json,sys
|
||||
rows=json.load(sys.stdin)
|
||||
if rows:
|
||||
row=rows[-1]
|
||||
print("%s\t%s" % (row.get("short_id", "unknown"), row.get("time", "unknown")))'
|
||||
)"; then
|
||||
fail_drill "snapshot_metadata_read_failed"
|
||||
fi
|
||||
[ -n "${snapshot_meta}" ] || fail_drill "snapshot_metadata_missing"
|
||||
IFS=$'\t' read -r snapshot_id snapshot_timestamp <<<"${snapshot_meta}"
|
||||
|
||||
dump_meta=""
|
||||
if ! dump_meta="$(
|
||||
restic -r "${LOCAL_REPO}" ls latest --json --password-file "${RESTIC_PASSWORD_FILE}" 2>/dev/null | \
|
||||
python3 -c 'import json,os,sys
|
||||
for line in sys.stdin:
|
||||
try: item=json.loads(line)
|
||||
except json.JSONDecodeError: continue
|
||||
if item.get("struct_type")=="node" and os.path.basename(item.get("path", ""))=="gitea-dump.zip":
|
||||
print("%s\t%s" % (item.get("path", ""), item.get("size", 0)))
|
||||
raise SystemExit'
|
||||
)"; then
|
||||
fail_drill "dump_metadata_read_failed" "${snapshot_id}" "${snapshot_timestamp}"
|
||||
fi
|
||||
[ -n "${dump_meta}" ] || fail_drill "gitea_dump_member_missing" "${snapshot_id}" "${snapshot_timestamp}"
|
||||
IFS=$'\t' read -r dump_path dump_size <<<"${dump_meta}"
|
||||
|
||||
available_bytes="$(df -Pk "${WORK_ROOT}" | awk 'NR==2 {printf "%.0f\n", $4 * 1024}')"
|
||||
required_bytes=$((dump_size + MIN_FREE_BUFFER_BYTES))
|
||||
if [ "${available_bytes}" -lt "${required_bytes}" ]; then
|
||||
fail_drill "insufficient_free_space" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||||
fi
|
||||
|
||||
log_info "Restoring isolated Gitea dump snapshot=${snapshot_id} size_bytes=${dump_size}"
|
||||
if ! low_priority restic -r "${LOCAL_REPO}" dump latest "${dump_path}" \
|
||||
--password-file "${RESTIC_PASSWORD_FILE}" > "${RESTORED_ZIP}"; then
|
||||
fail_drill "restic_dump_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||||
fi
|
||||
fi
|
||||
|
||||
actual_size="$(file_size "${RESTORED_ZIP}")"
|
||||
if [ "${actual_size}" -ne "${dump_size}" ]; then
|
||||
fail_drill "dump_size_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||||
fi
|
||||
|
||||
if ! low_priority unzip -tqq "${RESTORED_ZIP}"; then
|
||||
fail_drill "archive_crc_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}"
|
||||
fi
|
||||
crc_ok=1
|
||||
|
||||
inventory=""
|
||||
if ! inventory="$(
|
||||
unzip -Z1 "${RESTORED_ZIP}" | awk '
|
||||
/\.wiki\.git\/HEAD$/ { wiki += 1 }
|
||||
/\.git\/HEAD$/ && $0 !~ /\.wiki\.git\/HEAD$/ { regular += 1 }
|
||||
/(^|\/)gitea-db\.sql$/ { database += 1 }
|
||||
/(^|\/)app\.ini$/ { config += 1 }
|
||||
END { printf "%d %d %d %d", regular, wiki, database, config }
|
||||
'
|
||||
)"; then
|
||||
fail_drill "archive_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 0 0 0 0 0 "${crc_ok}"
|
||||
fi
|
||||
read -r backup_regular backup_wiki database_present config_present <<<"${inventory}"
|
||||
|
||||
backup_inventory_file="${WORK_DIR}/backup-repo-inventory"
|
||||
live_inventory_file="${WORK_DIR}/live-repo-inventory"
|
||||
if ! unzip -Z1 "${RESTORED_ZIP}" | awk -F/ '
|
||||
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
|
||||
' | LC_ALL=C sort -u > "${backup_inventory_file}"; then
|
||||
fail_drill "backup_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||||
fi
|
||||
|
||||
if [ -n "${LIVE_INVENTORY_FILE}" ]; then
|
||||
[ -f "${LIVE_INVENTORY_FILE}" ] || fail_drill "test_live_inventory_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||||
LC_ALL=C sort -u "${LIVE_INVENTORY_FILE}" > "${live_inventory_file}"
|
||||
else
|
||||
if ! docker exec gitea sh -c 'find /data/git/repositories -type f -path "*.git/HEAD"' 2>/dev/null | awk -F/ '
|
||||
/\.git\/HEAD$/ { print $(NF - 2) "/" $(NF - 1) }
|
||||
' | LC_ALL=C sort -u > "${live_inventory_file}"; then
|
||||
fail_drill "live_inventory_read_failed" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" 0 0 "${database_present}" "${config_present}" "${crc_ok}"
|
||||
fi
|
||||
fi
|
||||
|
||||
live_inventory="$(awk '
|
||||
/\.wiki\.git$/ { wiki += 1; next }
|
||||
/\.git$/ { regular += 1 }
|
||||
END { printf "%d %d", regular, wiki }
|
||||
' "${live_inventory_file}")"
|
||||
read -r live_regular live_wiki <<<"${live_inventory}"
|
||||
|
||||
hash_file() {
|
||||
if command -v sha256sum >/dev/null 2>&1; then
|
||||
sha256sum "$1" | awk '{print $1}'
|
||||
else
|
||||
shasum -a 256 "$1" | awk '{print $1}'
|
||||
fi
|
||||
}
|
||||
|
||||
backup_inventory_digest="$(hash_file "${backup_inventory_file}")"
|
||||
live_inventory_digest="$(hash_file "${live_inventory_file}")"
|
||||
inventory_match=0
|
||||
[ "${backup_inventory_digest}" = "${live_inventory_digest}" ] && inventory_match=1
|
||||
|
||||
[ "${database_present}" -gt 0 ] || fail_drill "database_dump_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" 0 "${config_present}" "${crc_ok}"
|
||||
[ "${config_present}" -gt 0 ] || fail_drill "config_missing" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" 0 "${crc_ok}"
|
||||
[ "${backup_regular}" -gt 0 ] || fail_drill "repository_inventory_empty" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" 0 "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}"
|
||||
|
||||
if [ "${backup_regular}" -ne "${live_regular}" ] || [ "${backup_wiki}" -ne "${live_wiki}" ]; then
|
||||
fail_drill "repository_count_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||||
fi
|
||||
if [ "${inventory_match}" -ne 1 ]; then
|
||||
fail_drill "repository_inventory_mismatch" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" "${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" "${database_present}" "${config_present}" "${crc_ok}" 0
|
||||
fi
|
||||
|
||||
write_status 1 "verified" "${snapshot_id}" "${snapshot_timestamp}" "${dump_size}" \
|
||||
"${backup_regular}" "${backup_wiki}" "${live_regular}" "${live_wiki}" \
|
||||
"${database_present}" "${config_present}" "${crc_ok}" "${inventory_match}"
|
||||
log_success "Gitea restore drill verified: regular=${backup_regular}/${live_regular} wiki=${backup_wiki}/${live_wiki}; evidence=${STATUS_FILE}"
|
||||
if [ "${NOTIFY}" -eq 1 ]; then
|
||||
notify_clawbot "success" "gitea-restore-drill" "Gitea 備份還原演練完成，repo=${backup_regular}/${live_regular}" "$(($(date +%s) - START_TIME))"
|
||||
fi
|
||||
@@ -6,6 +6,9 @@ Node exporter textfile metrics use labels such as job="backup_all" locally, but
|
||||
Prometheus rewrites that metric label to exported_job because the scrape target
|
||||
already has job="node-exporter-110". Backup alerts must therefore use
|
||||
$labels.exported_job in user-facing text and exported_job="..." in expressions.
|
||||
Receipt alerts must also be demand-driven: only an actively firing backup or
|
||||
Gitea alert may require a fresh receipt. Missing markers for inactive rules are
|
||||
inventory evidence, not user-facing incidents.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -142,20 +145,30 @@ def static_check(path: Path, baseline_path: Path) -> list[str]:
|
||||
lines.append("OK BackupAlertReceiptMetricMissing188 checks 188 receipt requirement metric")
|
||||
|
||||
rule = _require_alert(alerts, "BackupAlertReceiptStageMissing")
|
||||
_require_contains(
|
||||
str(rule.get("expr", "")),
|
||||
expr = str(rule.get("expr", ""))
|
||||
for required_fragment in [
|
||||
"ALERTS{",
|
||||
'alertstate="firing"',
|
||||
'alertname=~"Backup.*|GiteaPrivateBundle.*|GiteaRestoreDrill.*"',
|
||||
'alertname!="BackupAlertReceiptStageMissing"',
|
||||
"unless on (alertname, host)",
|
||||
"label_replace(",
|
||||
"awoooi_backup_alert_receipt_fresh == 1",
|
||||
]:
|
||||
_require_contains(expr, required_fragment, "BackupAlertReceiptStageMissing expr")
|
||||
_require_not_contains(
|
||||
expr,
|
||||
"sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0",
|
||||
"BackupAlertReceiptStageMissing expr",
|
||||
)
|
||||
text = _annotation_text(rule)
|
||||
for required_label in [
|
||||
"$labels.host",
|
||||
"$labels.receipt_channel",
|
||||
"$value",
|
||||
"required_alert/scope/stage/required_notification_type",
|
||||
"$labels.alertname",
|
||||
"required_alert",
|
||||
]:
|
||||
_require_contains(text, required_label, "BackupAlertReceiptStageMissing annotations")
|
||||
lines.append("OK BackupAlertReceiptStageMissing aggregates by host/receipt channel and points to detail labels")
|
||||
lines.append("OK BackupAlertReceiptStageMissing requires receipts only for actively firing backup/Gitea alerts")
|
||||
|
||||
return lines
|
||||
|
||||
|
||||
@@ -34,6 +34,10 @@ BACKUP_OFFSITE_ENV = Path(os.environ.get("AIOPS_BACKUP_OFFSITE_ENV", "/backup/sc
|
||||
OFFSITE_STATUS_DIR = Path(os.environ.get("AIOPS_OFFSITE_STATUS_DIR", "/backup/offsite"))
|
||||
ESCROW_EVIDENCE_DIR = Path(os.environ.get("AIOPS_ESCROW_EVIDENCE_DIR", "/backup/escrow-evidence"))
|
||||
CONFIG_CAPTURE_STATUS_FILE = Path(os.environ.get("AIOPS_CONFIG_CAPTURE_STATUS_FILE", "/backup/status/backup-configs-last-status.json"))
|
||||
GITEA_RESTORE_DRILL_STATUS_FILE = Path(
|
||||
os.environ.get("AIOPS_GITEA_RESTORE_DRILL_STATUS_FILE", "/backup/integrity/gitea-restore-drill.status")
|
||||
)
|
||||
GITEA_RESTORE_DRILL_MAX_AGE_HOURS = float(os.environ.get("AIOPS_GITEA_RESTORE_DRILL_MAX_AGE_HOURS", "744"))
|
||||
BACKUP_ALERT_RECEIPT_DIR = Path(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_DIR", "/backup/alert-receipts"))
|
||||
BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS = float(os.environ.get("AIOPS_BACKUP_ALERT_RECEIPT_MAX_AGE_HOURS", "168"))
|
||||
DEFAULT_GITEA_BUNDLE_EXPECTED_REPOS = [
|
||||
@@ -73,6 +77,7 @@ BACKUP_ALERT_RECEIPT_REQUIREMENTS = [
|
||||
{"host": "110", "alertname": "BackupConfigCaptureStatusStale", "scope": "config_capture"},
|
||||
{"host": "110", "alertname": "BackupIntegrityCheckMissingOrFailed", "scope": "integrity"},
|
||||
{"host": "110", "alertname": "BackupRestoreDrillMissingOrFailed", "scope": "restore_drill"},
|
||||
{"host": "110", "alertname": "GiteaRestoreDrillMissingOrFailed", "scope": "gitea_restore_drill"},
|
||||
{"host": "110", "alertname": "BackupRestoreTestMissing", "scope": "velero_restore_test"},
|
||||
{"host": "110", "alertname": "BackupRestoreTestCronMissing", "scope": "velero_restore_test"},
|
||||
{"host": "110", "alertname": "BackupRestoreTestFailed", "scope": "velero_restore_test"},
|
||||
@@ -573,6 +578,46 @@ def _integrity_metric_lines(host: str) -> list[str]:
|
||||
return lines
|
||||
|
||||
|
||||
def _gitea_restore_drill_metric_lines(host: str) -> list[str]:
|
||||
values = _read_key_value_status(str(GITEA_RESTORE_DRILL_STATUS_FILE))
|
||||
now = int(time.time())
|
||||
timestamp = int(values.get("timestamp", 0))
|
||||
success = int(values.get("success", 0))
|
||||
dump_size = int(values.get("dump_size_bytes", 0))
|
||||
backup_regular = int(values.get("backup_regular_repo_count", 0))
|
||||
backup_wiki = int(values.get("backup_wiki_repo_count", 0))
|
||||
live_regular = int(values.get("live_regular_repo_count", 0))
|
||||
live_wiki = int(values.get("live_wiki_repo_count", 0))
|
||||
database_present = int(values.get("database_dump_present", 0))
|
||||
config_present = int(values.get("config_present", 0))
|
||||
crc_ok = int(values.get("archive_crc_ok", 0))
|
||||
inventory_match = int(values.get("inventory_digest_match", 0))
|
||||
age = max(0, now - timestamp) if timestamp else 0
|
||||
count_match = int(backup_regular == live_regular and backup_wiki == live_wiki and backup_regular > 0)
|
||||
required_artifacts_ok = int(database_present > 0 and config_present > 0)
|
||||
verified = int(success == 1 and crc_ok == 1 and required_artifacts_ok == 1 and count_match == 1 and inventory_match == 1)
|
||||
fresh = int(timestamp > 0 and age <= int(GITEA_RESTORE_DRILL_MAX_AGE_HOURS * 3600) and verified == 1)
|
||||
reason = str(values.get("reason", "status_missing"))
|
||||
labels = f'host="{_escape_label(host)}",max_age_hours="{GITEA_RESTORE_DRILL_MAX_AGE_HOURS:g}"'
|
||||
|
||||
return [
|
||||
f'awoooi_gitea_restore_drill_status_info{{{labels},reason="{_escape_label(reason)}"}} 1',
|
||||
f"awoooi_gitea_restore_drill_last_success_timestamp{{{labels}}} {timestamp if verified else 0}",
|
||||
f"awoooi_gitea_restore_drill_age_seconds{{{labels}}} {age}",
|
||||
f"awoooi_gitea_restore_drill_fresh{{{labels}}} {fresh}",
|
||||
f"awoooi_gitea_restore_drill_verified{{{labels}}} {verified}",
|
||||
f"awoooi_gitea_restore_drill_archive_crc_ok{{{labels}}} {crc_ok}",
|
||||
f"awoooi_gitea_restore_drill_required_artifacts_ok{{{labels}}} {required_artifacts_ok}",
|
||||
f"awoooi_gitea_restore_drill_repo_count_match{{{labels}}} {count_match}",
|
||||
f"awoooi_gitea_restore_drill_inventory_match{{{labels}}} {inventory_match}",
|
||||
f"awoooi_gitea_restore_drill_dump_size_bytes{{{labels}}} {dump_size}",
|
||||
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="regular"}} {backup_regular}',
|
||||
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="backup",kind="wiki"}} {backup_wiki}',
|
||||
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="regular"}} {live_regular}',
|
||||
f'awoooi_gitea_restore_drill_repo_count{{{labels},source="live",kind="wiki"}} {live_wiki}',
|
||||
]
|
||||
|
||||
|
||||
def _config_capture_metric_lines(host: str) -> list[str]:
|
||||
now = int(time.time())
|
||||
labels = f'host="{_escape_label(host)}"'
|
||||
@@ -975,6 +1020,28 @@ def _base_lines(host: str) -> list[str]:
|
||||
"# TYPE awoooi_backup_integrity_failed_repo_count gauge",
|
||||
"# HELP awoooi_backup_integrity_checked_repo_count Checked repository count from backup integrity or restore drill run.",
|
||||
"# TYPE awoooi_backup_integrity_checked_repo_count gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_status_info Bounded result reason from the isolated Gitea restore drill.",
|
||||
"# TYPE awoooi_gitea_restore_drill_status_info gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_last_success_timestamp Unix timestamp of the latest fully verified isolated Gitea restore drill.",
|
||||
"# TYPE awoooi_gitea_restore_drill_last_success_timestamp gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_age_seconds Age of the latest isolated Gitea restore drill evidence.",
|
||||
"# TYPE awoooi_gitea_restore_drill_age_seconds gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_fresh Whether the isolated Gitea restore drill is verified and within max age.",
|
||||
"# TYPE awoooi_gitea_restore_drill_fresh gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_verified Whether CRC, required assets, counts, and inventory all passed.",
|
||||
"# TYPE awoooi_gitea_restore_drill_verified gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_archive_crc_ok Whether every Gitea dump ZIP member passed CRC verification.",
|
||||
"# TYPE awoooi_gitea_restore_drill_archive_crc_ok gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_required_artifacts_ok Whether database dump and configuration are present.",
|
||||
"# TYPE awoooi_gitea_restore_drill_required_artifacts_ok gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_repo_count_match Whether backup and live regular/wiki repo counts match.",
|
||||
"# TYPE awoooi_gitea_restore_drill_repo_count_match gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_inventory_match Whether privacy-preserving backup and live repo inventories match.",
|
||||
"# TYPE awoooi_gitea_restore_drill_inventory_match gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_dump_size_bytes Size of the verified Gitea dump archive.",
|
||||
"# TYPE awoooi_gitea_restore_drill_dump_size_bytes gauge",
|
||||
"# HELP awoooi_gitea_restore_drill_repo_count Regular or wiki repo count in backup or live inventory.",
|
||||
"# TYPE awoooi_gitea_restore_drill_repo_count gauge",
|
||||
"# HELP awoooi_backup_config_capture_status_timestamp Unix timestamp of the latest config-capture coverage status.",
|
||||
"# TYPE awoooi_backup_config_capture_status_timestamp gauge",
|
||||
"# HELP awoooi_backup_config_capture_status_age_seconds Age of the latest config-capture coverage status.",
|
||||
@@ -1104,6 +1171,7 @@ def _collect_110(host: str) -> list[str]:
|
||||
"offsite_full_sync_verify": "/backup/scripts/verify-offsite-full-sync.sh --write-textfile",
|
||||
"backup_integrity_check": "/backup/scripts/check-backup-integrity.sh --mode check",
|
||||
"backup_restore_drill": "/backup/scripts/check-backup-integrity.sh --mode restore-drill",
|
||||
"gitea_bundle_sync_188": "/backup/scripts/gitea-bundle-backup-sync-188.sh",
|
||||
}
|
||||
for job, pattern in expected_crons.items():
|
||||
labels = f'host="{_escape_label(host)}",job="{_escape_label(job)}"'
|
||||
@@ -1124,6 +1192,9 @@ def _collect_110(host: str) -> list[str]:
|
||||
"verify-offsite-full-sync.sh",
|
||||
"mark-credential-escrow-verified.sh",
|
||||
"check-backup-integrity.sh",
|
||||
"verify-gitea-backup.sh",
|
||||
"gitea-repo-bundle-backup.sh",
|
||||
"gitea-bundle-backup-sync-188.sh",
|
||||
"backup-gitea.sh",
|
||||
"backup-harbor.sh",
|
||||
"backup-momo.sh",
|
||||
@@ -1193,6 +1264,7 @@ def _collect_110(host: str) -> list[str]:
|
||||
lines.append(f"awoooi_backup_last_run_failed_count{{{labels}}} {failed_count}")
|
||||
lines.append(f"awoooi_backup_job_last_success_timestamp{{{labels},type=\"aggregate\",source=\"110-cron-log\",target=\"/backup/logs/cron.log\",max_age_hours=\"48\"}} {backup_all_ts if failed_count == 0 else 0}")
|
||||
lines.extend(_integrity_metric_lines(host))
|
||||
lines.extend(_gitea_restore_drill_metric_lines(host))
|
||||
lines.extend(_config_capture_metric_lines(host))
|
||||
lines.extend(_offsite_and_escrow_metric_lines(host))
|
||||
lines.extend(_retention_metric_lines(host))
|
||||
@@ -1215,11 +1287,13 @@ def _collect_188(host: str) -> list[str]:
|
||||
for script in [
|
||||
"/home/ollama/bin/backup-from-110.sh",
|
||||
"/home/ollama/bin/momo-pg-backup.sh",
|
||||
"/home/ollama/scripts/gitea-bundle-sample-restore-dry-run.sh",
|
||||
"/home/ollama/awoooi-ops/pg-backup.sh",
|
||||
]:
|
||||
labels = f'host="{_escape_label(host)}",script="{_escape_label(Path(script).name)}"'
|
||||
lines.append(f"awoooi_backup_script_present{{{labels}}} {int(Path(script).exists() and os.access(script, os.X_OK))}")
|
||||
|
||||
backup_from_110_ts = _read_backup_110_timestamp()
|
||||
lines.extend(
|
||||
_metric_lines_for_job(
|
||||
host=host,
|
||||
@@ -1227,12 +1301,13 @@ def _collect_188(host: str) -> list[str]:
|
||||
source="188-rsync",
|
||||
target="/home/ollama/backup/110",
|
||||
backup_type="rsync",
|
||||
last_success=_read_backup_110_timestamp(),
|
||||
last_success=backup_from_110_ts,
|
||||
max_age_hours=25,
|
||||
sample_count=1,
|
||||
)
|
||||
)
|
||||
gitea_mirror_ts, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
|
||||
_, gitea_mirror_count = _newest_tree_timestamp(Path("/home/ollama/backup/110/gitea"))
|
||||
gitea_mirror_ts = backup_from_110_ts
|
||||
gitea_mirror_fresh = 1 if gitea_mirror_ts and int(time.time()) - gitea_mirror_ts <= 25 * 3600 else 0
|
||||
lines.extend(
|
||||
_metric_lines_for_job(
|
||||
@@ -1248,6 +1323,20 @@ def _collect_188(host: str) -> list[str]:
|
||||
)
|
||||
gitea_bundle_lines, gitea_bundle_ok = _gitea_bundle_metric_lines(host)
|
||||
lines.extend(gitea_bundle_lines)
|
||||
gitea_bundle_ts, gitea_bundle_count = _newest_tree_timestamp(GITEA_BUNDLE_ROOT, max_entries=20000)
|
||||
gitea_bundle_fresh = 1 if gitea_bundle_ts and int(time.time()) - gitea_bundle_ts <= 25 * 3600 else 0
|
||||
lines.extend(
|
||||
_metric_lines_for_job(
|
||||
host=host,
|
||||
job="gitea_private_bundle",
|
||||
source="188-git-bundle",
|
||||
target=str(GITEA_BUNDLE_ROOT),
|
||||
backup_type="git_bundle",
|
||||
last_success=gitea_bundle_ts,
|
||||
max_age_hours=25,
|
||||
sample_count=gitea_bundle_count,
|
||||
)
|
||||
)
|
||||
coverage_labels = (
|
||||
f'host="{_escape_label(host)}",'
|
||||
'domain="service",'
|
||||
@@ -1256,7 +1345,7 @@ def _collect_188(host: str) -> list[str]:
|
||||
lines.append(f"awoooi_backup_coverage_domain_expected_info{{{coverage_labels}}} 1")
|
||||
lines.append(
|
||||
"awoooi_backup_coverage_domain_fresh"
|
||||
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_ok else 0}"
|
||||
f"{{{coverage_labels}}} {1 if gitea_mirror_fresh and gitea_bundle_fresh and gitea_bundle_ok else 0}"
|
||||
)
|
||||
momo_ts = _newest_file_timestamp([
|
||||
"/home/ollama/momo_backups/*.sql.gz",
|
||||
|
||||
@@ -13,10 +13,16 @@ CURRENT_RULES="${CURRENT_RULES:-/home/wooo/monitoring/alerts.yml}"
|
||||
CANONICAL_RULES="${CANONICAL_RULES:-/home/wooo/monitoring/alerts-unified.canonical.yml}"
|
||||
TEXTFILE="${TEXTFILE:-/home/wooo/node_exporter_textfiles/prometheus_rule_drift_guard.prom}"
|
||||
LOG_FILE="${LOG_FILE:-/home/wooo/logs/prometheus-rule-drift-guard.log}"
|
||||
PROMETHEUS_CONTAINER="${PROMETHEUS_CONTAINER:-prometheus}"
|
||||
PROMETHEUS_RUNTIME_RULES="${PROMETHEUS_RUNTIME_RULES:-/etc/prometheus/alerts.yml}"
|
||||
PROMETHEUS_COMPOSE_FILE="${PROMETHEUS_COMPOSE_FILE:-/home/wooo/monitoring/docker-compose.yml}"
|
||||
PROMETHEUS_COMPOSE_SERVICE="${PROMETHEUS_COMPOSE_SERVICE:-prometheus}"
|
||||
ALLOW_RUNTIME_RECREATE="${ALLOW_RUNTIME_RECREATE:-1}"
|
||||
|
||||
REQUIRED_RULES=(
|
||||
"BackupCredentialEscrowEvidenceMissing"
|
||||
"BackupExpectedJobMissing"
|
||||
"GiteaRestoreDrillMissingOrFailed"
|
||||
"awoooi_recovery_core_ready"
|
||||
"awoooi_recovery_dr_offsite_ready"
|
||||
"ColdStartRecoveryBlocked"
|
||||
@@ -32,6 +38,7 @@ write_textfile() {
|
||||
local repaired="$2"
|
||||
local missing_count="$3"
|
||||
local matches_canonical="$4"
|
||||
local runtime_matches_current="$5"
|
||||
local tmp
|
||||
mkdir -p "$(dirname "$TEXTFILE")" 2>/dev/null || true
|
||||
tmp="$(mktemp "${TEXTFILE}.tmp.XXXXXX")" || return 0
|
||||
@@ -48,6 +55,9 @@ awoooi_prometheus_rule_drift_guard_missing_required_count{host="${HOST_LABEL}"}
|
||||
# HELP awoooi_prometheus_rule_drift_guard_current_matches_canonical Whether active alerts.yml matches canonical copy.
|
||||
# TYPE awoooi_prometheus_rule_drift_guard_current_matches_canonical gauge
|
||||
awoooi_prometheus_rule_drift_guard_current_matches_canonical{host="${HOST_LABEL}"} ${matches_canonical}
|
||||
# HELP awoooi_prometheus_rule_drift_guard_runtime_matches_current Whether the Prometheus container mount matches the active host rules file.
|
||||
# TYPE awoooi_prometheus_rule_drift_guard_runtime_matches_current gauge
|
||||
awoooi_prometheus_rule_drift_guard_runtime_matches_current{host="${HOST_LABEL}"} ${runtime_matches_current}
|
||||
EOF
|
||||
chmod 0644 "$tmp" 2>/dev/null || true
|
||||
mv "$tmp" "$TEXTFILE" 2>/dev/null || rm -f "$tmp"
|
||||
@@ -67,16 +77,32 @@ try:
|
||||
if payload.get("status") != "success":
|
||||
raise RuntimeError(payload)
|
||||
loaded = {
|
||||
str(rule.get("name") or rule.get("alert") or rule.get("record"))
|
||||
str(rule.get("name") or rule.get("alert") or rule.get("record")): rule
|
||||
for group in payload.get("data", {}).get("groups") or []
|
||||
for rule in group.get("rules") or []
|
||||
}
|
||||
print(len(required - loaded))
|
||||
missing = required - set(loaded)
|
||||
unhealthy = {name for name in required & set(loaded) if loaded[name].get("health") != "ok"}
|
||||
print(len(missing | unhealthy))
|
||||
except Exception as exc:
|
||||
print(f"QUERY_FAILED:{exc}")
|
||||
PY
|
||||
}
|
||||
|
||||
wait_for_required_rules() {
|
||||
local attempt missing
|
||||
for attempt in $(seq 1 30); do
|
||||
missing="$(rules_missing_count)"
|
||||
if [ "$missing" = "0" ]; then
|
||||
log "runtime rules converged after attempt=${attempt}"
|
||||
return 0
|
||||
fi
|
||||
sleep 2
|
||||
done
|
||||
log "runtime rules did not converge within 60s: ${missing}"
|
||||
return 1
|
||||
}
|
||||
|
||||
matches_canonical() {
|
||||
if [ ! -f "$CURRENT_RULES" ] || [ ! -f "$CANONICAL_RULES" ]; then
|
||||
echo 0
|
||||
@@ -89,60 +115,136 @@ matches_canonical() {
|
||||
fi
|
||||
}
|
||||
|
||||
file_sha256() {
|
||||
sha256sum "$1" 2>/dev/null | awk '{print $1}'
|
||||
}
|
||||
|
||||
runtime_rules_sha256() {
|
||||
docker exec "${PROMETHEUS_CONTAINER}" sha256sum "${PROMETHEUS_RUNTIME_RULES}" 2>/dev/null | awk '{print $1}'
|
||||
}
|
||||
|
||||
runtime_matches_current() {
|
||||
local current_hash runtime_hash
|
||||
current_hash="$(file_sha256 "${CURRENT_RULES}")"
|
||||
runtime_hash="$(runtime_rules_sha256)"
|
||||
if [ -n "${current_hash}" ] && [ "${current_hash}" = "${runtime_hash}" ]; then
|
||||
echo 1
|
||||
else
|
||||
echo 0
|
||||
fi
|
||||
}
|
||||
|
||||
validate_canonical_rules() {
|
||||
local candidate_path rc
|
||||
candidate_path="/tmp/awoooi-prometheus-rule-guard-candidate.$$.yml"
|
||||
docker cp "${CANONICAL_RULES}" "${PROMETHEUS_CONTAINER}:${candidate_path}" >/dev/null 2>&1 || return 1
|
||||
docker exec "${PROMETHEUS_CONTAINER}" promtool check rules "${candidate_path}" >>"${LOG_FILE}" 2>&1
|
||||
rc=$?
|
||||
docker exec -u 0 "${PROMETHEUS_CONTAINER}" rm -f "${candidate_path}" >/dev/null 2>&1 || true
|
||||
return "${rc}"
|
||||
}
|
||||
|
||||
recreate_prometheus_runtime() {
|
||||
[ "${ALLOW_RUNTIME_RECREATE}" = "1" ] || return 1
|
||||
[ -f "${PROMETHEUS_COMPOSE_FILE}" ] || return 1
|
||||
docker compose -f "${PROMETHEUS_COMPOSE_FILE}" up -d --no-deps --force-recreate \
|
||||
"${PROMETHEUS_COMPOSE_SERVICE}" >>"${LOG_FILE}" 2>&1
|
||||
}
|
||||
|
||||
restore_rules() {
|
||||
local backup_path
|
||||
local backup_path needs_recreate
|
||||
backup_path="${CURRENT_RULES}.guard.bak.$(date +%Y%m%d%H%M%S)"
|
||||
cp "$CURRENT_RULES" "$backup_path" 2>/dev/null || true
|
||||
cp "$CANONICAL_RULES" "$CURRENT_RULES"
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null
|
||||
if ! cp "$CURRENT_RULES" "$backup_path" 2>/dev/null; then
|
||||
log "failed to create active rules rollback copy"
|
||||
return 1
|
||||
fi
|
||||
if ! validate_canonical_rules; then
|
||||
log "canonical promtool validation failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! cat "$CANONICAL_RULES" >"$CURRENT_RULES"; then
|
||||
log "failed to update active rules file"
|
||||
return 1
|
||||
fi
|
||||
|
||||
needs_recreate=0
|
||||
[ "$(runtime_matches_current)" -eq 1 ] || needs_recreate=1
|
||||
if [ "${needs_recreate}" -eq 1 ]; then
|
||||
log "runtime bind mount differs from active file; recreating Prometheus only"
|
||||
if ! recreate_prometheus_runtime; then
|
||||
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
|
||||
log "Prometheus recreate failed; active file rolled back"
|
||||
return 1
|
||||
fi
|
||||
elif ! curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null; then
|
||||
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true
|
||||
log "Prometheus reload failed; active file rolled back"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if wait_for_required_rules && [ "$(runtime_matches_current)" -eq 1 ]; then
|
||||
return 0
|
||||
fi
|
||||
|
||||
cat "$backup_path" >"$CURRENT_RULES" 2>/dev/null || true
|
||||
if [ "${needs_recreate}" -eq 1 ]; then
|
||||
recreate_prometheus_runtime >/dev/null 2>&1 || true
|
||||
else
|
||||
curl -fsS -X POST "${PROMETHEUS_URL}/-/reload" >/dev/null 2>&1 || true
|
||||
fi
|
||||
log "post-apply verifier failed; active rules rolled back"
|
||||
return 1
|
||||
}
|
||||
|
||||
main() {
|
||||
if [ ! -f "$CANONICAL_RULES" ]; then
|
||||
log "canonical rules missing: ${CANONICAL_RULES}"
|
||||
write_textfile "canonical_missing" 0 999 0
|
||||
write_textfile "canonical_missing" 0 999 0 0
|
||||
return 1
|
||||
fi
|
||||
|
||||
local missing before_matches repaired after_missing after_matches
|
||||
local missing before_matches before_runtime_matches repaired after_missing after_matches after_runtime_matches
|
||||
missing="$(rules_missing_count)"
|
||||
before_matches="$(matches_canonical)"
|
||||
before_runtime_matches="$(runtime_matches_current)"
|
||||
repaired=0
|
||||
|
||||
if [[ "$missing" == QUERY_FAILED:* ]]; then
|
||||
log "Prometheus query failed: ${missing}"
|
||||
write_textfile "query_failed" 0 999 "$before_matches"
|
||||
write_textfile "query_failed" 0 999 "$before_matches" "$before_runtime_matches"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ]; then
|
||||
log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches}; restoring"
|
||||
if [ "$missing" -gt 0 ] || [ "$before_matches" -eq 0 ] || [ "$before_runtime_matches" -eq 0 ]; then
|
||||
log "rule drift detected: missing=${missing} current_matches_canonical=${before_matches} runtime_matches_current=${before_runtime_matches}; restoring"
|
||||
if restore_rules; then
|
||||
repaired=1
|
||||
sleep 3
|
||||
else
|
||||
log "restore failed"
|
||||
write_textfile "restore_failed" 0 "$missing" "$before_matches"
|
||||
write_textfile "restore_failed" 0 "$missing" "$before_matches" "$before_runtime_matches"
|
||||
return 1
|
||||
fi
|
||||
fi
|
||||
|
||||
after_missing="$(rules_missing_count)"
|
||||
after_matches="$(matches_canonical)"
|
||||
after_runtime_matches="$(runtime_matches_current)"
|
||||
if [[ "$after_missing" == QUERY_FAILED:* ]]; then
|
||||
log "post-restore Prometheus query failed: ${after_missing}"
|
||||
write_textfile "post_query_failed" "$repaired" 999 "$after_matches"
|
||||
write_textfile "post_query_failed" "$repaired" 999 "$after_matches" "$after_runtime_matches"
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ]; then
|
||||
write_textfile "ok" "$repaired" "$after_missing" "$after_matches"
|
||||
if [ "$after_missing" -eq 0 ] && [ "$after_matches" -eq 1 ] && [ "$after_runtime_matches" -eq 1 ]; then
|
||||
write_textfile "ok" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches"
|
||||
log "ok repaired=${repaired}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches}"
|
||||
write_textfile "drifted" "$repaired" "$after_missing" "$after_matches"
|
||||
log "still drifted after check: missing=${after_missing} current_matches_canonical=${after_matches} runtime_matches_current=${after_runtime_matches}"
|
||||
write_textfile "drifted" "$repaired" "$after_missing" "$after_matches" "$after_runtime_matches"
|
||||
return 1
|
||||
}
|
||||
|
||||
|
||||
56
scripts/ops/tests/test_backup_alert_label_contract.py
Normal file
56
scripts/ops/tests/test_backup_alert_label_contract.py
Normal file
@@ -0,0 +1,56 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import yaml
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
CHECKER = ROOT / "scripts" / "ops" / "backup-alert-label-contract-check.py"
|
||||
RULES = ROOT / "ops" / "monitoring" / "alerts-unified.yml"
|
||||
BASELINE = ROOT / "ops" / "reboot-recovery" / "full-stack-backup-baseline.yml"
|
||||
|
||||
|
||||
def _run(rules: Path) -> subprocess.CompletedProcess[str]:
|
||||
return subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(CHECKER),
|
||||
"--rules",
|
||||
str(rules),
|
||||
"--baseline",
|
||||
str(BASELINE),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=False,
|
||||
)
|
||||
|
||||
|
||||
def test_checker_accepts_firing_only_receipt_contract() -> None:
|
||||
result = _run(RULES)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "requires receipts only for actively firing backup/Gitea alerts" in result.stdout
|
||||
assert "BACKUP_ALERT_LABEL_CONTRACT_OK" in result.stdout
|
||||
|
||||
|
||||
def test_checker_rejects_non_firing_receipt_selector(tmp_path: Path) -> None:
|
||||
payload = yaml.safe_load(RULES.read_text(encoding="utf-8"))
|
||||
for group in payload["groups"]:
|
||||
for rule in group.get("rules", []):
|
||||
if rule.get("alert") == "BackupAlertReceiptStageMissing":
|
||||
rule["expr"] = str(rule["expr"]).replace(
|
||||
'alertstate="firing"',
|
||||
'alertstate="pending"',
|
||||
)
|
||||
candidate = tmp_path / "alerts.yml"
|
||||
candidate.write_text(yaml.safe_dump(payload, allow_unicode=True, sort_keys=False), encoding="utf-8")
|
||||
|
||||
result = _run(candidate)
|
||||
|
||||
assert result.returncode != 0
|
||||
assert 'alertstate="firing"' in result.stderr
|
||||
@@ -35,6 +35,50 @@ def test_default_gitea_bundle_expected_repos_cover_current_project_set() -> None
|
||||
"wooo/bitan-pharmacy",
|
||||
"wooo/vtuber",
|
||||
]
|
||||
assert any(
|
||||
row["alertname"] == "GiteaRestoreDrillMissingOrFailed"
|
||||
for row in exporter.BACKUP_ALERT_RECEIPT_REQUIREMENTS
|
||||
)
|
||||
|
||||
|
||||
def test_gitea_restore_drill_metrics_require_inventory_and_artifacts(tmp_path: Path, monkeypatch) -> None:
|
||||
exporter = load_exporter()
|
||||
status_file = tmp_path / "gitea-restore-drill.status"
|
||||
status_file.write_text(
|
||||
"\n".join(
|
||||
[
|
||||
"timestamp=2000000000",
|
||||
"success=1",
|
||||
"reason=verified",
|
||||
"dump_size_bytes=6928031514",
|
||||
"backup_regular_repo_count=15",
|
||||
"backup_wiki_repo_count=0",
|
||||
"live_regular_repo_count=15",
|
||||
"live_wiki_repo_count=0",
|
||||
"database_dump_present=1",
|
||||
"config_present=2",
|
||||
"archive_crc_ok=1",
|
||||
"inventory_digest_match=1",
|
||||
]
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
monkeypatch.setattr(exporter, "GITEA_RESTORE_DRILL_STATUS_FILE", status_file)
|
||||
monkeypatch.setattr(exporter.time, "time", lambda: 2000000060)
|
||||
|
||||
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
|
||||
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 1' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 1' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_inventory_match{host="110",max_age_hours="744"} 1' in rendered
|
||||
|
||||
status_file.write_text(
|
||||
status_file.read_text(encoding="utf-8").replace("inventory_digest_match=1", "inventory_digest_match=0"),
|
||||
encoding="utf-8",
|
||||
)
|
||||
rendered = "\n".join(exporter._gitea_restore_drill_metric_lines("110"))
|
||||
assert 'awoooi_gitea_restore_drill_fresh{host="110",max_age_hours="744"} 0' in rendered
|
||||
assert 'awoooi_gitea_restore_drill_verified{host="110",max_age_hours="744"} 0' in rendered
|
||||
|
||||
|
||||
def test_gitea_bundle_metrics_require_all_expected_repos(tmp_path: Path, monkeypatch) -> None:
|
||||
|
||||
@@ -446,6 +446,7 @@ def test_backup_exporter_emits_domain_level_backup_coverage() -> None:
|
||||
"GiteaPrivateBundleBackupIncomplete",
|
||||
"GiteaPrivateBundleRestoreDryRunMissing",
|
||||
"BackupRestoreDrillMissingOrFailed",
|
||||
"GiteaRestoreDrillMissingOrFailed",
|
||||
"BackupCredentialEscrowEvidenceMissing",
|
||||
]:
|
||||
assert alertname in exporter
|
||||
@@ -462,16 +463,19 @@ def test_backup_alerts_cover_188_gitea_mirror_subtree() -> None:
|
||||
assert "BackupAlertReceiptMetricMissing" in alerts
|
||||
assert "BackupAlertReceiptStageMissing" in alerts
|
||||
assert "awoooi_backup_alert_receipt_expected_info" in alerts
|
||||
assert "sum by (host, receipt_channel) (awoooi_backup_alert_receipt_stage_fresh == bool 0) > 0" in alerts
|
||||
assert "$labels.receipt_channel" in alerts
|
||||
assert "$value" in alerts
|
||||
assert "required_alert/scope/stage/required_notification_type" in alerts
|
||||
assert "telegram_awooop" in alerts
|
||||
assert 'ALERTS{' in alerts
|
||||
assert 'alertname!="BackupAlertReceiptStageMissing"' in alerts
|
||||
assert "label_replace(" in alerts
|
||||
assert 'awoooi_backup_alert_receipt_fresh == 1' in alerts
|
||||
assert "$labels.alertname" in alerts
|
||||
|
||||
|
||||
def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None:
|
||||
script = read("scripts/backup/gitea-repo-bundle-backup.sh")
|
||||
restore_script = read("scripts/backup/gitea-bundle-sample-restore-dry-run.sh")
|
||||
wrapper = read("scripts/backup/gitea-bundle-backup-sync-188.sh")
|
||||
exporter = read("scripts/ops/backup-health-textfile-exporter.py")
|
||||
playbook = read("infra/ansible/playbooks/110-devops.yml")
|
||||
|
||||
assert "GIT_TERMINAL_PROMPT=0" in script
|
||||
assert "GIT_REMOTE_BASE" in script
|
||||
@@ -481,6 +485,9 @@ def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None:
|
||||
assert "bundle verify" in script
|
||||
assert ".sha256" in script
|
||||
assert "manifest.tsv" in script
|
||||
assert "latest-private-complete" in script
|
||||
assert "complete links unchanged" in script
|
||||
assert 'exit 1' in script
|
||||
assert "gitea dump" in script
|
||||
assert "does not replace `gitea dump`" in script
|
||||
assert "tokens or passwords" in script
|
||||
@@ -488,3 +495,39 @@ def test_gitea_repo_bundle_backup_is_non_interactive_and_manifested() -> None:
|
||||
assert "fsck --connectivity-only" in restore_script
|
||||
assert "awoooi_gitea_bundle_sample_restore_dry_run_ok" in restore_script
|
||||
assert "never reads tokens" in restore_script
|
||||
assert "flock -n" in wrapper
|
||||
assert "latest-private-complete" in wrapper
|
||||
assert "--local-container" in wrapper
|
||||
assert "rsync -a --checksum" in wrapper
|
||||
assert "remote_sample_restore_fsck" in wrapper
|
||||
assert "refresh_remote_backup_health" in wrapper
|
||||
assert "rollback_remote_links" in wrapper
|
||||
assert "remote_manifest_checksum_ok" in wrapper
|
||||
assert "promoted=1" in wrapper
|
||||
assert '"gitea_bundle_sync_188"' in exporter
|
||||
assert 'job="gitea_private_bundle"' in exporter
|
||||
assert "gitea-bundle-backup-sync-188.sh" in playbook
|
||||
assert 'minute: "15"' in playbook
|
||||
assert 'hour: "0"' in playbook
|
||||
|
||||
|
||||
def test_prometheus_rule_guard_repairs_container_bind_mount_drift() -> None:
|
||||
guard = read("scripts/ops/prometheus-rule-drift-guard.sh")
|
||||
playbook = read("infra/ansible/playbooks/110-devops.yml")
|
||||
|
||||
assert "runtime_matches_current" in guard
|
||||
assert "awoooi_prometheus_rule_drift_guard_runtime_matches_current" in guard
|
||||
assert "promtool check rules" in guard
|
||||
assert "--no-deps --force-recreate" in guard
|
||||
assert "failed to create active rules rollback copy" in guard
|
||||
assert "post-apply verifier failed; active rules rolled back" in guard
|
||||
assert "prometheus-rule-drift-guard.sh" in playbook
|
||||
assert 'minute: "*/5"' in playbook
|
||||
|
||||
|
||||
def test_velero_restore_test_recovers_a_missed_weekly_schedule() -> None:
|
||||
manifest = read("k8s/awoooi-prod/16-cronjob-backup-restore-test.yaml")
|
||||
|
||||
assert "startingDeadlineSeconds: 604800" in manifest
|
||||
assert 'schedule: "0 18 * * 6"' in manifest
|
||||
assert "concurrencyPolicy: Forbid" in manifest
|
||||
|
||||
Reference in New Issue
Block a user