feat(agent99): bind host110 backup runtime deploy

scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py

@@ -0,0 +1,251 @@
+from __future__ import annotations
+
+import base64
+import hashlib
+import importlib.util
+import json
+import re
+import sys
+from pathlib import Path
+
+import pytest
+
+
+ROOT = Path(__file__).resolve().parents[3]
+BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
+EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
+VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
+
+
+EXPECTED_RUNTIME_FILES = {
+    "common.sh",
+    "backup-all.sh",
+    "backup-host188-products.sh",
+    "verify-host188-products-backup.sh",
+    "verify-host188-products-archive.py",
+    "backup-gitea.sh",
+    "gitea-full-backup-restore-drill.sh",
+    "backup-configs.sh",
+    "backup-awoooi.sh",
+    "backup-awoooi-frequent.sh",
+    "backup-clawbot.sh",
+    "backup-sentry.sh",
+    "check-backup-integrity.sh",
+    "sync-offsite-backups.sh",
+    "backup-offsite-readiness-gate.sh",
+    "verify-offsite-full-sync.sh",
+    "enforce-latest-only-retention.sh",
+}
+
+
+def _load_verifier():
+    spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
+    assert spec and spec.loader
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def _sha256(path: Path) -> str:
+    return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
+    source = BROKER.read_text(encoding="utf-8")
+
+    assert '[ValidateSet("Check", "Apply", "Verify")]' in source
+    assert '$TargetHost = "192.168.0.110"' in source
+    assert '$TargetUser = "wooo"' in source
+    assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
+    assert 'keys\\agent99_ed25519' in source
+    assert '"credential.helper="' in source
+    assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
+    assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
+    assert '"checkout", "--quiet", $SourceRevision, "--"' in source
+    assert 'executorSha256 = $executorDigest' in source
+    assert 'verifierSha256 = $verifierDigest' in source
+    assert "/home/wooo/awoooi" not in source
+    assert '"StrictHostKeyChecking=yes"' in source
+    assert '"NumberOfPasswordPrompts=0"' in source
+    assert '"ConnectionAttempts=1"' in source
+    assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
+    assert "taskkill.exe /PID $process.Id /T /F" in source
+    assert "$process.Kill()" in source
+    assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
+
+
+def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
+    source = BROKER.read_text(encoding="utf-8")
+
+    assert 'RedirectStandardInput = $StandardInputPath' in source
+    assert "python3 - --manifest-base64" in source
+    assert 'remoteWritePerformed' in source
+    assert '[IO.File]::Move($temporary, $Path)' in source
+    assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
+    assert 'run_identity_conflict' in source
+    assert 'run_identity_evidence_exists' in source
+    assert 'decisionProvider = "deterministic_only"' in source
+    assert 'criticProvider = "deterministic_only"' in source
+    assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
+
+
+def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
+    source = EXECUTOR.read_text(encoding="utf-8")
+    array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
+    runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
+
+    assert runtime_files == EXPECTED_RUNTIME_FILES
+    assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
+    assert 'readonly EXPECTED_USER="wooo"' in source
+    assert 'readonly DEST_ROOT="/backup/scripts"' in source
+    assert '--source-stage' in source
+    assert 'document.get("executorSha256"' in source
+    assert 'document.get("verifierSha256"' in source
+    assert 'executor_identity_failed' in source
+    assert 'verifier_identity_failed' in source
+    assert 'flock -x -w 30 9' in source
+    assert 'legacy_backup_runtime_active' in source
+    assert source.index("for name in \"${RUNTIME_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
+    assert 'rollback_unverified' in source
+    assert 'failed_rolled_back' in source
+    assert 'working_digest "$dest_path"' in source
+    assert 'os.link(temporary, path)' in source
+    assert 'run_identity_receipt_exists' in source
+    assert 'run_identity_stage_exists' in source
+    assert 'run_identity_rollback_exists' in source
+    assert 'verify-host110-backup-runtime.py' in source
+    assert 'selfIdentityVerified' in source
+    assert '"productionServiceRestarted": False' in source
+    assert '"secretValuesRead": False' in source
+    assert "sudo " not in source
+    assert "docker " not in source
+    assert "systemctl " not in source
+
+
+def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
+    common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
+    restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
+    offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
+    archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
+
+    for source in (common, restore, offsite):
+        assert "/tmp/agent99-host110-backup-runtime.lock" in source
+        assert "flock -s -w 60" in source
+    assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
+    assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
+
+
+def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
+    verifier = _load_verifier()
+    destination = tmp_path / "scripts"
+    destination.mkdir()
+    rows = []
+    for name in verifier.EXPECTED_FILES:
+        path = destination / name
+        path.write_text(f"runtime:{name}\n", encoding="utf-8")
+        path.chmod(0o755)
+        rows.append({"name": name, "sha256": _sha256(path)})
+    verifier_digest = _sha256(VERIFIER)
+    source_revision = "a" * 40
+    run_id = "unit-exact"
+    manifest = {
+        "schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
+        "sourceRevision": source_revision,
+        "sourceHead": "b" * 40,
+        "runId": run_id,
+        "executorSha256": "c" * 64,
+        "verifierSha256": verifier_digest,
+        "files": rows,
+    }
+    manifest_path = tmp_path / "manifest.json"
+    manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
+    before = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
+    monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            str(VERIFIER),
+            "--manifest",
+            str(manifest_path),
+            "--destination",
+            str(destination),
+            "--source-revision",
+            source_revision,
+            "--run-id",
+            run_id,
+            "--expected-verifier-sha256",
+            verifier_digest,
+        ],
+    )
+
+    verifier.main()
+
+    result = json.loads(capsys.readouterr().out)
+    after = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
+    assert result["ok"] is True
+    assert result["remoteWritePerformed"] is False
+    assert result["selfIdentityVerified"] is True
+    assert before == after
+
+
+def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
+    verifier = _load_verifier()
+    destination = tmp_path / "scripts"
+    destination.mkdir()
+    rows = []
+    for name in verifier.EXPECTED_FILES:
+        path = destination / name
+        path.write_text(f"runtime:{name}\n", encoding="utf-8")
+        path.chmod(0o755)
+        rows.append({"name": name, "sha256": _sha256(path)})
+    (destination / verifier.EXPECTED_FILES[0]).write_text("drifted\n", encoding="utf-8")
+    verifier_digest = _sha256(VERIFIER)
+    source_revision = "d" * 40
+    run_id = "unit-drift"
+    manifest = {
+        "schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
+        "sourceRevision": source_revision,
+        "sourceHead": "e" * 40,
+        "runId": run_id,
+        "executorSha256": "f" * 64,
+        "verifierSha256": verifier_digest,
+        "files": rows,
+    }
+    encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
+    monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
+    monkeypatch.setattr(
+        sys,
+        "argv",
+        [
+            str(VERIFIER),
+            "--manifest-base64",
+            encoded,
+            "--destination",
+            str(destination),
+            "--source-revision",
+            source_revision,
+            "--run-id",
+            run_id,
+            "--expected-verifier-sha256",
+            verifier_digest,
+        ],
+    )
+
+    with pytest.raises(SystemExit):
+        verifier.main()
+    assert "runtime_file_identity_failed" in capsys.readouterr().err
+
+
+def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
+    sources = [
+        ROOT / "agent99-bootstrap.ps1",
+        ROOT / "agent99-contract-check.ps1",
+        ROOT / "agent99-deploy.ps1",
+        ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
+        ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
+    ]
+    marker = "agent99-host110-backup-runtime-broker.ps1"
+
+    for path in sources:
+        assert marker in path.read_text(encoding="utf-8"), path

scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py

@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
             "exists": True,
             "sourceRevision": "a" * 40,
             "runtimeMatched": True,
-            "fileCount": 18,
+            "fileCount": 19,
             "mismatchCount": 0,
             "parseError": "",
         },
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
     )
     assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
     assert "preflightGreen = [bool]$decision.deploymentEligible" in source
-    assert "-ExpectedRuntimeFileCount 18" in source
+    assert "-ExpectedRuntimeFileCount 19" in source
     assert "warningReasons = $warningReasons" in source
     assert "selfHealthPerformanceIssues = @(" in source
     assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source

scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py

@@ -26,6 +26,7 @@ EXPECTED_RUNTIME_FILES = (
     "agent99-control-plane.ps1",
     "agent99-db-bounded-executor.ps1",
     "agent99-signoz-metadata-executor.ps1",
+    "agent99-host110-backup-runtime-broker.ps1",
     "agent99-deploy.ps1",
     "agent99-register-tasks.ps1",
     "agent99-alertmanager-alertchain-poll.ps1",
@@ -88,7 +89,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
     )
     assert preflight_count is not None
     assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
-    assert "expectedRuntimeFileCount\": 18" in sender
+    assert "expectedRuntimeFileCount\": 19" in sender
     assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
     assert "required_runtime_file_missing" in receiver
     assert "duplicate_runtime_file" in receiver
@@ -263,7 +264,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
     )
     assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
     assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
-    assert '$runtimeManifest.fileCount -eq 18' in source
+    assert '$runtimeManifest.fileCount -eq 19' in source
     assert "function Invoke-AgentWindowsControlBaseline" in source
     assert "function Get-AgentRunEvidenceToken" in source
     assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source

scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py

@@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
     for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
         assert f"[string]${parameter} = \"\"" in control
         assert f'[string]`${parameter} = ""' in bootstrap
-    assert 'reason = "controlled_dispatch_identity_unsafe"' in control
+    assert '"controlled_dispatch_identity_unsafe"' in control
+    assert "reason = $identityRejectionReason" in control
     assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
     assert '$dryRun.managerPreflightStatus -eq "ready"' in function
     assert "-not $dryRun.artifactTransactionBlocked" in function