283 lines
11 KiB
Python
283 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
import base64
|
|
import hashlib
|
|
import importlib.util
|
|
import json
|
|
import re
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parents[3]
|
|
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
|
|
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
|
|
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
|
|
|
|
|
|
EXPECTED_RUNTIME_FILES = {
|
|
"common.sh",
|
|
"backup-all.sh",
|
|
"backup-host188-products.sh",
|
|
"verify-host188-products-backup.sh",
|
|
"verify-host188-products-archive.py",
|
|
"backup-gitea.sh",
|
|
"gitea-full-backup-restore-drill.sh",
|
|
"backup-configs.sh",
|
|
"backup-awoooi.sh",
|
|
"backup-awoooi-frequent.sh",
|
|
"backup-clawbot.sh",
|
|
"backup-sentry.sh",
|
|
"check-backup-integrity.sh",
|
|
"sync-offsite-backups.sh",
|
|
"backup-offsite-readiness-gate.sh",
|
|
"verify-offsite-full-sync.sh",
|
|
"enforce-latest-only-retention.sh",
|
|
}
|
|
|
|
|
|
def _load_verifier():
|
|
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
|
|
assert spec and spec.loader
|
|
module = importlib.util.module_from_spec(spec)
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
|
|
def _sha256(path: Path) -> str:
|
|
return hashlib.sha256(path.read_bytes()).hexdigest()
|
|
|
|
|
|
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
|
|
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
|
|
assert '$TargetHost = "192.168.0.110"' in source
|
|
assert '$TargetUser = "wooo"' in source
|
|
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
|
|
assert 'keys\\agent99_ed25519' in source
|
|
assert '"credential.helper="' in source
|
|
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
|
|
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
|
|
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
|
|
assert 'executorSha256 = $executorDigest' in source
|
|
assert 'verifierSha256 = $verifierDigest' in source
|
|
assert '"scripts/ops/backup-health-textfile-exporter.py"' in source
|
|
assert "/home/wooo/awoooi" not in source
|
|
assert '"StrictHostKeyChecking=yes"' in source
|
|
assert '"NumberOfPasswordPrompts=0"' in source
|
|
assert '"ConnectionAttempts=1"' in source
|
|
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
|
|
assert "taskkill.exe /PID $process.Id /T /F" in source
|
|
assert "$process.Kill()" in source
|
|
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
|
|
|
|
|
|
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
|
|
source = BROKER.read_text(encoding="utf-8")
|
|
|
|
assert 'RedirectStandardInput = $StandardInputPath' in source
|
|
assert "python3 - --manifest-base64" in source
|
|
assert 'remoteWritePerformed' in source
|
|
assert '[IO.File]::Move($temporary, $Path)' in source
|
|
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
|
|
assert 'run_identity_conflict' in source
|
|
assert 'run_identity_evidence_exists' in source
|
|
assert 'decisionProvider = "deterministic_only"' in source
|
|
assert 'criticProvider = "deterministic_only"' in source
|
|
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
|
|
|
|
|
|
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
|
|
source = EXECUTOR.read_text(encoding="utf-8")
|
|
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
|
|
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
|
|
|
|
assert runtime_files == EXPECTED_RUNTIME_FILES
|
|
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
|
|
assert 'readonly EXPECTED_USER="wooo"' in source
|
|
assert 'readonly DEST_ROOT="/backup/scripts"' in source
|
|
assert 'readonly EXPORTER_ROOT="/home/wooo/scripts"' in source
|
|
assert 'readonly EXPORTER_FILE="backup-health-textfile-exporter.py"' in source
|
|
assert 'readonly -a PAYLOAD_FILES=("${RUNTIME_FILES[@]}" "$EXPORTER_FILE")' in source
|
|
assert '--source-stage' in source
|
|
assert 'document.get("executorSha256"' in source
|
|
assert 'document.get("verifierSha256"' in source
|
|
assert 'executor_identity_failed' in source
|
|
assert 'verifier_identity_failed' in source
|
|
assert 'flock -x -w 30 9' in source
|
|
assert 'legacy_backup_runtime_active' in source
|
|
assert source.index("for name in \"${PAYLOAD_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
|
|
assert 'rollback_unverified' in source
|
|
assert 'failed_rolled_back' in source
|
|
assert 'working_digest "$dest_path"' in source
|
|
assert 'PREVIOUS_METADATA' in source
|
|
assert "stat -c '%u:%g:%a'" in source
|
|
assert 'os.link(temporary, path)' in source
|
|
assert 'run_identity_receipt_exists' in source
|
|
assert 'run_identity_stage_exists' in source
|
|
assert 'run_identity_rollback_exists' in source
|
|
assert 'verify-host110-backup-runtime.py' in source
|
|
assert 'selfIdentityVerified' in source
|
|
assert '"backupScriptFileCount": 17' in source
|
|
assert '"backupHealthExporterIncluded": True' in source
|
|
assert '"productionServiceRestarted": False' in source
|
|
assert '"secretValuesRead": False' in source
|
|
assert "sudo " not in source
|
|
assert "docker " not in source
|
|
assert "systemctl " not in source
|
|
|
|
|
|
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
|
|
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
|
|
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
|
|
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
|
|
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
|
|
|
for source in (common, restore, offsite):
|
|
assert "/tmp/agent99-host110-backup-runtime.lock" in source
|
|
assert "flock -s -w 60" in source
|
|
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
|
|
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
|
|
|
|
|
|
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
|
|
verifier = _load_verifier()
|
|
destination = tmp_path / "scripts"
|
|
destination.mkdir()
|
|
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
|
rows = []
|
|
for name in verifier.EXPECTED_BACKUP_FILES:
|
|
path = destination / name
|
|
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
|
path.chmod(0o755)
|
|
rows.append({"name": name, "sha256": _sha256(path)})
|
|
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
|
exporter_destination.chmod(0o755)
|
|
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
|
verifier_digest = _sha256(VERIFIER)
|
|
source_revision = "a" * 40
|
|
run_id = "unit-exact"
|
|
manifest = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": "b" * 40,
|
|
"runId": run_id,
|
|
"executorSha256": "c" * 64,
|
|
"verifierSha256": verifier_digest,
|
|
"files": rows,
|
|
}
|
|
manifest_path = tmp_path / "manifest.json"
|
|
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
|
|
before = {
|
|
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
|
for path in [*destination.iterdir(), exporter_destination]
|
|
}
|
|
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
|
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
str(VERIFIER),
|
|
"--manifest",
|
|
str(manifest_path),
|
|
"--destination",
|
|
str(destination),
|
|
"--exporter-destination",
|
|
str(exporter_destination),
|
|
"--source-revision",
|
|
source_revision,
|
|
"--run-id",
|
|
run_id,
|
|
"--expected-verifier-sha256",
|
|
verifier_digest,
|
|
],
|
|
)
|
|
|
|
verifier.main()
|
|
|
|
result = json.loads(capsys.readouterr().out)
|
|
after = {
|
|
path.name: (path.stat().st_mtime_ns, _sha256(path))
|
|
for path in [*destination.iterdir(), exporter_destination]
|
|
}
|
|
assert result["ok"] is True
|
|
assert result["fileCount"] == 18
|
|
assert result["backupScriptFileCount"] == 17
|
|
assert result["backupHealthExporterIncluded"] is True
|
|
assert result["remoteWritePerformed"] is False
|
|
assert result["selfIdentityVerified"] is True
|
|
assert before == after
|
|
|
|
|
|
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
|
|
verifier = _load_verifier()
|
|
destination = tmp_path / "scripts"
|
|
destination.mkdir()
|
|
exporter_destination = tmp_path / verifier.EXPORTER_FILE
|
|
rows = []
|
|
for name in verifier.EXPECTED_BACKUP_FILES:
|
|
path = destination / name
|
|
path.write_text(f"runtime:{name}\n", encoding="utf-8")
|
|
path.chmod(0o755)
|
|
rows.append({"name": name, "sha256": _sha256(path)})
|
|
exporter_destination.write_text("exporter\n", encoding="utf-8")
|
|
exporter_destination.chmod(0o755)
|
|
rows.append({"name": verifier.EXPORTER_FILE, "sha256": _sha256(exporter_destination)})
|
|
(destination / verifier.EXPECTED_BACKUP_FILES[0]).write_text("drifted\n", encoding="utf-8")
|
|
verifier_digest = _sha256(VERIFIER)
|
|
source_revision = "d" * 40
|
|
run_id = "unit-drift"
|
|
manifest = {
|
|
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
|
|
"sourceRevision": source_revision,
|
|
"sourceHead": "e" * 40,
|
|
"runId": run_id,
|
|
"executorSha256": "f" * 64,
|
|
"verifierSha256": verifier_digest,
|
|
"files": rows,
|
|
}
|
|
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
|
|
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
|
|
monkeypatch.setattr(verifier, "EXPECTED_EXPORTER_DESTINATION", exporter_destination)
|
|
monkeypatch.setattr(
|
|
sys,
|
|
"argv",
|
|
[
|
|
str(VERIFIER),
|
|
"--manifest-base64",
|
|
encoded,
|
|
"--destination",
|
|
str(destination),
|
|
"--exporter-destination",
|
|
str(exporter_destination),
|
|
"--source-revision",
|
|
source_revision,
|
|
"--run-id",
|
|
run_id,
|
|
"--expected-verifier-sha256",
|
|
verifier_digest,
|
|
],
|
|
)
|
|
|
|
with pytest.raises(SystemExit):
|
|
verifier.main()
|
|
assert "runtime_file_identity_failed" in capsys.readouterr().err
|
|
|
|
|
|
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
|
|
sources = [
|
|
ROOT / "agent99-bootstrap.ps1",
|
|
ROOT / "agent99-contract-check.ps1",
|
|
ROOT / "agent99-deploy.ps1",
|
|
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
|
|
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
|
|
]
|
|
marker = "agent99-host110-backup-runtime-broker.ps1"
|
|
|
|
for path in sources:
|
|
assert marker in path.read_text(encoding="utf-8"), path
|