wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(backup): make Gitea dump fail-safe

Browse Source
This commit is contained in:
Your Name
2026-07-18 20:21:03 +08:00
parent e7e3bcf8a5
commit 56d1d396b8
9 changed files with 740 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
scripts/backup/tests/test_110_backup_runtime_deploy_playbook.py
View File

@@ -18,6 +18,9 @@ EXPECTED_FILES = {
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
@@ -26,7 +29,7 @@ EXPECTED_FILES = {
}
def test_backup_runtime_deploy_is_fixed_to_host_110_and_fourteen_files() -> None:
def test_backup_runtime_deploy_is_fixed_to_host_110_and_seventeen_files() -> None:
plays = yaml.safe_load(PLAYBOOK.read_text(encoding="utf-8"))
assert len(plays) == 1
play = plays[0]

44
scripts/backup/tests/test_backup_gitea_full_receipt_contract.py
View File

@@ -5,6 +5,7 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
BACKUP_GITEA = ROOT / "scripts" / "backup" / "backup-gitea.sh"
BACKUP_CONFIGS = ROOT / "scripts" / "backup" / "backup-configs.sh"
def test_backup_gitea_writes_full_server_component_receipts() -> None:
@@ -15,7 +16,12 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert "awoooi_gitea_full_backup_secret_value_collected" in text
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "gitea dump -c /data/gitea/conf/app.ini" in text
assert "offline_gitea_dump" in text
assert "--volumes-from" in text
assert "--pull=never" in text
assert "docker stop --time" in text
assert "start_and_verify_primary" in text
assert "gitea-full-backup-restore-drill.sh" in text
assert "GITEA_FULL_BACKUP_COMPONENTS" in text
for component in [
@@ -25,6 +31,9 @@ def test_backup_gitea_writes_full_server_component_receipts() -> None:
"lfs_objects",
"package_registry",
"attachments",
"actions_logs",
"actions_artifacts",
"avatars",
"hooks_custom_assets",
]:
assert component in text
@@ -37,7 +46,7 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
assert "awoooi_gitea_full_backup_production_restore_performed" in text
assert "awoooi_gitea_full_backup_restore_drill_performed" in text
assert (
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
'restore_drill_performed{host=\\"${host}\\",service=\\"${service_label}\\"} ${structural_drill_performed}'
in text
)
assert (
@@ -48,3 +57,34 @@ def test_backup_gitea_receipt_contract_stays_no_secret_no_restore() -> None:
'production_restore_performed{host=\\"${host}\\",service=\\"${service_label}\\"} 0'
in text
)
def test_backup_gitea_is_bounded_and_fail_safe_around_the_offline_window() -> None:
text = BACKUP_GITEA.read_text(encoding="utf-8")
assert 'GITEA_BACKUP_LOCK="${GITEA_BACKUP_LOCK:-/tmp/gitea-backup.lock}"' in text
assert "flock -n 9" in text
assert "trap cleanup EXIT" in text
assert 'if [ "${PRIMARY_STOPPED}" -eq 1 ]' in text
assert "wait_for_primary_health" in text
assert "http://127.0.0.1:3001/api/healthz" in text
assert '--kill-after="${GITEA_DUMP_KILL_AFTER_SECONDS}s"' in text
assert "bounded_docker_control rm -f" in text
assert 'GITEA_MAX_OUTAGE_SECONDS="${GITEA_MAX_OUTAGE_SECONDS:-600}"' in text
assert "configured_outage_bound" in text
assert "--security-opt no-new-privileges:true" in text
assert '--cpus "${BACKUP_DOCKER_CPUS}"' in text
assert '--memory "${BACKUP_DOCKER_MEMORY}"' in text
assert '--memory-swap "${BACKUP_DOCKER_MEMORY_SWAP}"' in text
assert "awoooi_gitea_full_backup_consistent_offline_window" in text
assert "awoooi_gitea_full_backup_service_restart_verified" in text
assert "awoooi_gitea_full_backup_service_outage_seconds" in text
def test_gitea_runner_configuration_is_in_the_encrypted_config_backup() -> None:
text = BACKUP_CONFIGS.read_text(encoding="utf-8")
assert "/home/wooo/vibework-act-runner/config.yaml" in text
assert 'record_config_status "110-gitea-runner-config" true true "110"' in text
assert 'record_config_status "110-gitea-runner-config" true false "110"' in text
assert "/home/wooo/vibework-act-runner/data" not in text

300
scripts/backup/tests/test_backup_gitea_offline_window_behavior.py Normal file
View File

@@ -0,0 +1,300 @@
from __future__ import annotations
import os
import subprocess
import textwrap
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-gitea.sh"
def _write_executable(path: Path, source: str) -> None:
path.write_text(textwrap.dedent(source), encoding="utf-8")
path.chmod(0o755)
def _fake_runtime(tmp_path: Path) -> tuple[dict[str, str], Path, Path]:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
state_dir = tmp_path / "state"
state_dir.mkdir()
(state_dir / "running").write_text("true\n", encoding="utf-8")
command_log = state_dir / "commands.log"
_write_executable(
fake_bin / "docker",
r"""
#!/usr/bin/env bash
set -euo pipefail
state="${FAKE_DOCKER_STATE:?}"
printf '%s\n' "$*" >> "$state/commands.log"
command_name="${1:-}"
shift || true
case "$command_name" in
inspect)
args="$*"
case "$args" in
*State.Running*) cat "$state/running" ;;
*NetworkSettings.Networks*) printf 'gitea_gitea\n' ;;
*'{{.Image}}'*) printf 'sha256:%064d\n' 0 ;;
*) exit 64 ;;
esac
;;
exec)
args="$*"
if [[ "$args" == *' id -u'* ]]; then
id -u
elif [[ "$args" == *'gitea --version'* ]]; then
printf 'gitea version 1.25.5 built with test\n'
else
exit 64
fi
;;
stop)
printf 'false\n' > "$state/running"
if [ "${FAKE_DOCKER_STOP_FAIL:-0}" = "1" ]; then
exit 42
fi
;;
start)
printf 'true\n' > "$state/running"
;;
run)
output=""
for arg in "$@"; do
case "$arg" in
type=bind,src=*,dst=/backup-out)
output="${arg#type=bind,src=}"
output="${output%,dst=/backup-out}"
;;
esac
done
[ -n "$output" ] || exit 65
if [ "${FAKE_DOCKER_RUN_FAIL:-0}" = "1" ]; then
exit 42
fi
python3 - "$output/gitea-dump.zip" <<'PY'
import sys
import zipfile
entries = {
"gitea-db.sql": "fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\n",
"data/lfs/objects/aa/bb/object": "lfs\n",
"data/packages/package.bin": "package\n",
"data/attachments/attachment.bin": "attachment\n",
"data/actions_artifacts/aa/bb/artifact.zip": "artifact\n",
"data/actions_log/aa/bb/job.log": "log\n",
"data/avatars/aa/bb/avatar.png": "avatar\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
with zipfile.ZipFile(sys.argv[1], "w") as archive:
for name, value in entries.items():
archive.writestr(name, value)
PY
;;
rm) ;;
*) exit 64 ;;
esac
""",
)
_write_executable(
fake_bin / "curl",
r"""
#!/usr/bin/env bash
url="${!#}"
if [ "$url" = "http://127.0.0.1:3001/api/healthz" ]; then
printf '{"status":"pass"}'
else
printf '<html>another service</html>'
fi
""",
)
_write_executable(
fake_bin / "timeout",
"""
#!/usr/bin/env bash
set -euo pipefail
while [[ "${1:-}" == --* ]]; do shift; done
shift
if [ "${FAKE_DOCKER_RUN_TIMEOUT:-0}" = "1" ] && [[ " $* " == *' docker run '* ]]; then
exit 124
fi
exec "$@"
""",
)
_write_executable(
fake_bin / "flock",
"""
#!/usr/bin/env bash
exit 0
""",
)
_write_executable(
fake_bin / "restic",
r"""
#!/usr/bin/env bash
set -euo pipefail
repo=""
args=("$@")
for ((i=0; i<${#args[@]}; i++)); do
if [ "${args[$i]}" = "-r" ]; then repo="${args[$((i+1))]}"; fi
done
if [[ " $* " == *' init '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' backup '* ]]; then
mkdir -p "$repo/data" "$repo/snapshots"
exit 0
fi
if [[ " $* " == *' snapshots '* ]]; then
printf '[{"short_id":"abcd1234","time":"2026-07-18T00:00:00Z"}]\n'
exit 0
fi
if [[ " $* " == *' forget '* ]]; then exit 0; fi
exit 64
""",
)
backup_root = tmp_path / "backup"
(backup_root / "scripts").mkdir(parents=True)
(backup_root / "scripts" / ".restic-password").write_text("fixture\n", encoding="utf-8")
textfile = tmp_path / "gitea.prom"
environment = os.environ | {
"PATH": f"{fake_bin}:{os.environ['PATH']}",
"BACKUP_BASE": str(backup_root),
"BACKUP_LOG_DIR": str(backup_root / "logs"),
"RESTIC_PASSWORD_FILE": str(backup_root / "scripts" / ".restic-password"),
"GITEA_FULL_BACKUP_RECEIPT_TEXTFILE": str(textfile),
"NODE_EXPORTER_TEXTFILE_DIR": str(tmp_path),
"GITEA_BACKUP_LOCK": str(tmp_path / "gitea.lock"),
"GITEA_START_TIMEOUT_SECONDS": "4",
"GITEA_DUMP_TIMEOUT_SECONDS": "30",
"GITEA_DUMP_KILL_AFTER_SECONDS": "2",
"GITEA_DOCKER_CONTROL_TIMEOUT_SECONDS": "2",
"GITEA_DOCKER_CONTROL_KILL_AFTER_SECONDS": "1",
"GITEA_STOP_TIMEOUT_SECONDS": "2",
"FAKE_DOCKER_STATE": str(state_dir),
"BACKUP_RETENTION_MODE": "latest",
}
return environment, command_log, textfile
def test_offline_backup_restarts_and_verifies_primary_on_success(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode == 0, result.stderr
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("run --pull=never")
assert commands.index("run --pull=never") < commands.index("rm -f agent99-gitea-backup-")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert "awoooi_gitea_full_backup_consistent_offline_window" in rendered
assert "awoooi_gitea_full_backup_service_restart_verified" in rendered
assert 'service="gitea"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert "stop --time 2 gitea" in commands
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
rendered = textfile.read_text(encoding="utf-8")
assert 'awoooi_gitea_full_backup_service_restart_verified{host="110",service="gitea"} 1' in rendered
assert 'status="offline_dump_failed"} 1' in rendered
def test_offline_backup_restarts_primary_when_dump_hits_hard_timeout(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_RUN_TIMEOUT"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("rm -f agent99-gitea-backup-") < commands.index("start gitea")
assert 'status="offline_dump_failed"} 1' in textfile.read_text(encoding="utf-8")
def test_offline_backup_recovers_when_stop_has_side_effect_then_fails(tmp_path: Path) -> None:
environment, command_log, textfile = _fake_runtime(tmp_path)
environment["FAKE_DOCKER_STOP_FAIL"] = "1"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert (Path(environment["FAKE_DOCKER_STATE"]) / "running").read_text(encoding="utf-8").strip() == "true"
commands = command_log.read_text(encoding="utf-8")
assert commands.index("stop --time 2 gitea") < commands.index("start gitea")
assert 'service_restart_verified{host="110",service="gitea"} 1' in textfile.read_text(encoding="utf-8")
def test_wrong_service_health_response_cannot_open_offline_window(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_LOCAL_HEALTH_URL"] = "http://127.0.0.1:3000/"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
commands = command_log.read_text(encoding="utf-8")
assert "stop --time" not in commands
def test_oversized_outage_budget_fails_before_primary_stop(tmp_path: Path) -> None:
environment, command_log, _ = _fake_runtime(tmp_path)
environment["GITEA_MAX_OUTAGE_SECONDS"] = "10"
result = subprocess.run(
["bash", str(SCRIPT)],
env=environment,
text=True,
capture_output=True,
check=False,
)
assert result.returncode != 0
assert "Configured Gitea backup outage bound exceeds 10s" in result.stdout
assert not command_log.exists() or "stop --time" not in command_log.read_text(encoding="utf-8")

77
scripts/backup/tests/test_gitea_full_backup_restore_drill.py
View File

@@ -9,15 +9,25 @@ ROOT = Path(__file__).resolve().parents[1]
SCRIPT = ROOT / "gitea-full-backup-restore-drill.sh"
def _write_dump(path: Path, *, include_lfs: bool = True) -> None:
def _write_dump(
path: Path,
*,
include_lfs: bool = True,
include_actions_artifact: bool = True,
empty_critical_payloads: bool = False,
) -> None:
entries = {
"gitea-db.sql": "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "ref: refs/heads/main\n",
"gitea-db.sql": "" if empty_critical_payloads else "-- redacted fixture\n",
"repos/wooo/awoooi.git/HEAD": "" if empty_critical_payloads else "ref: refs/heads/main\n",
"custom/conf/app.ini": "[server]\nAPP_NAME=fixture\n",
"data/packages/package.bin": "package fixture\n",
"data/attachments/attachment.bin": "attachment fixture\n",
"data/actions_log/aa/bb/job.log": "actions log fixture\n",
"data/avatars/aa/bb/avatar.png": "avatar fixture\n",
"custom/hooks/pre-receive": "#!/bin/sh\n",
}
if include_actions_artifact:
entries["data/actions_artifacts/aa/bb/artifact.zip"] = "actions artifact fixture\n"
if include_lfs:
entries["data/lfs/objects/aa/bb/object"] = "lfs fixture\n"
with zipfile.ZipFile(path, "w") as archive:
@@ -97,3 +107,64 @@ def test_gitea_full_backup_restore_drill_fails_closed_when_component_missing(
rendered = textfile.read_text(encoding="utf-8")
assert 'component="lfs_objects",drill_scope="structural_metadata_only"} 0' in rendered
assert 'production_restore_performed{host="110",service="gitea",' in rendered
def test_gitea_full_backup_restore_drill_rejects_empty_database_and_repository(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump, empty_critical_payloads=True)
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=component_coverage_gap" in result.stdout
def test_gitea_full_backup_restore_drill_requires_actions_log_and_artifact(
tmp_path: Path,
) -> None:
dump = tmp_path / "gitea-dump.zip"
textfile = tmp_path / "gitea_full_backup_restore_drill.prom"
_write_dump(dump, include_actions_artifact=False)
result = subprocess.run(
[
"bash",
str(SCRIPT),
"--dump-zip",
str(dump),
"--write-textfile",
"--textfile",
str(textfile),
],
text=True,
capture_output=True,
)
assert result.returncode == 1
rendered = textfile.read_text(encoding="utf-8")
assert 'component="actions_logs",drill_scope="structural_metadata_only"} 1' not in rendered
assert 'component="actions_artifacts",drill_scope="structural_metadata_only"} 0' in rendered
def test_gitea_full_backup_restore_drill_rejects_crc_corruption(tmp_path: Path) -> None:
dump = tmp_path / "gitea-dump.zip"
_write_dump(dump)
payload = dump.read_bytes()
fixture = b"-- redacted fixture\n"
assert fixture in payload
dump.write_bytes(payload.replace(fixture, b"X" + fixture[1:], 1))
result = subprocess.run(
["bash", str(SCRIPT), "--dump-zip", str(dump)],
text=True,
capture_output=True,
)
assert result.returncode == 1
assert "ERROR=crc_failed" in result.stdout