feat(agent99): bind host110 backup runtime deploy

This commit is contained in:
Your Name
2026-07-18 21:21:05 +08:00
parent e7e3bcf8a5
commit 4b2b73f1a7
19 changed files with 1453 additions and 23 deletions

View File

@@ -486,7 +486,7 @@ $decision = Get-AgentLivePreflightDecision `
-AgentRootPresent ([bool](Test-Path $AgentRoot)) `
-Manifest $manifest `
-ManifestCheckRequired ([bool]($ReadinessScope -eq "FullRuntime")) `
-ExpectedRuntimeFileCount 18 `
-ExpectedRuntimeFileCount 19 `
-TaskFailureCount $taskFailures.Count `
-RelayListenerCount $relayListeners.Count `
-RequiredEvidenceStaleCount $requiredEvidenceStale.Count `

View File

@@ -34,6 +34,7 @@ $FixedRuntimeFiles = @(
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -951,7 +952,7 @@ try {
}
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 18) { throw "invalid_expected_runtime_file_count" }
if ([int]$envelope.expectedRuntimeFileCount -ne 19) { throw "invalid_expected_runtime_file_count" }
$traceId = [string]$envelope.traceId
$runId = [string]$envelope.runId
@@ -986,7 +987,7 @@ try {
$sourceManifest = [pscustomobject]@{
schemaVersion = "agent99_remote_source_manifest_v1"
sourceRevision = $sourceRevision
fileCount = 18
fileCount = 19
manifestSha256 = $manifestDigest
files = @($FixedRuntimeFiles | ForEach-Object {
[pscustomobject]@{ name = $_; sha256 = ([string]($filesByName[$_].sha256)).ToLowerInvariant() }
@@ -1023,7 +1024,7 @@ try {
status = "check_ready"
mode = "check"
sourceRevision = $sourceRevision
expectedRuntimeFileCount = 18
expectedRuntimeFileCount = 19
manifestSha256 = $manifestDigest
plannedStagingPath = $stagePath
runtimeManifest = $currentManifest
@@ -1110,7 +1111,7 @@ try {
if (
[string]$existingSourceManifest.schemaVersion -ne "agent99_remote_source_manifest_v1" -or
[string]$existingSourceManifest.sourceRevision -ne $sourceRevision -or
[int]$existingSourceManifest.fileCount -ne 18 -or
[int]$existingSourceManifest.fileCount -ne 19 -or
[string]$existingSourceManifest.manifestSha256 -ne $manifestDigest
) { throw "existing_staging_manifest_mismatch" }
} catch {
@@ -1134,7 +1135,7 @@ try {
[string]$prior.launcherContract.sha256 -match "^[0-9a-f]{64}$" -and
$live.runtimeMatched -and
$live.sourceRevision -eq $sourceRevision -and
$live.fileCount -eq 18 -and
$live.fileCount -eq 19 -and
$live.mismatchCount -eq 0 -and
$liveLauncherContract.ok -and
[string]$liveLauncherContract.sha256 -eq [string]$prior.launcherContract.sha256
@@ -1447,7 +1448,7 @@ try {
$runtimeManifest.exists -and
$runtimeManifest.runtimeMatched -and
$runtimeManifest.sourceRevision -eq $sourceRevision -and
$runtimeManifest.fileCount -eq 18 -and
$runtimeManifest.fileCount -eq 19 -and
$runtimeManifest.mismatchCount -eq 0 -and
$launcherContract.ok -and
[string]$launcherContract.sha256 -eq [string]$deploy.payload.launcherContract.sha256 -and

View File

@@ -18,6 +18,7 @@ RUNTIME_FILES=(
"agent99-control-plane.ps1"
"agent99-db-bounded-executor.ps1"
"agent99-signoz-metadata-executor.ps1"
"agent99-host110-backup-runtime-broker.ps1"
"agent99-deploy.ps1"
"agent99-register-tasks.ps1"
"agent99-alertmanager-alertchain-poll.ps1"
@@ -227,7 +228,7 @@ trace_id, run_id, work_item_id = sys.argv[4:7]
output_path = Path(sys.argv[7])
runtime_names = sys.argv[8:]
if len(runtime_names) != 18 or len(set(runtime_names)) != 18:
if len(runtime_names) != 19 or len(set(runtime_names)) != 19:
raise SystemExit("fixed_runtime_file_contract_failed")
files: list[dict[str, str]] = []
@@ -267,7 +268,7 @@ envelope = {
"runId": run_id,
"workItemId": work_item_id,
"sourceRevision": source_revision,
"expectedRuntimeFileCount": 18,
"expectedRuntimeFileCount": 19,
"manifestSha256": manifest_sha256,
"files": files,
"livePreflight": {
@@ -419,14 +420,14 @@ stage_token = hashlib.sha256(stage_identity.encode("utf-8")).hexdigest()[:20]
script = r'''$ErrorActionPreference='Stop';$ProgressPreference='SilentlyContinue'
$a='C:\Wooo\Agent99';$b=Join-Path $a 'bin';$p=Join-Path $a 'state\runtime-manifest.json'
$r=[ordered]@{exists=$false;sourceRevision='';runtimeMatched=$false;recordedRuntimeMatched=$false;fileCount=0;mismatchCount=-1;recordedMismatchCount=-1;parseError=''}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 18-and$m.mismatchCount-eq 0-and$rows.Count-eq 18-and$seen.Count-eq 18-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
if(Test-Path -LiteralPath $p -PathType Leaf){try{$m=Get-Content -LiteralPath $p -Raw|ConvertFrom-Json;$rows=@($m.files);$seen=@{};$bad=0;foreach($x in $rows){$n=[string]$x.name;if(!$n-or[IO.Path]::GetFileName($n)-ne$n-or$seen.ContainsKey($n)){$bad++;continue};$seen[$n]=1;$q=Join-Path $b $n;$h=if(Test-Path -LiteralPath $q -PathType Leaf){(Get-FileHash -LiteralPath $q -Algorithm SHA256).Hash.ToLowerInvariant()}else{'missing'};if(([string]$x.runtimeSha256).ToLowerInvariant()-ne$h){$bad++}};$r.exists=$true;$r.sourceRevision=[string]$m.sourceRevision;$r.recordedRuntimeMatched=[bool]$m.runtimeMatched;$r.fileCount=[int]$m.fileCount;$r.mismatchCount=$bad;$r.recordedMismatchCount=[int]$m.mismatchCount;$r.runtimeMatched=[bool]($m.runtimeMatched-and$m.fileCount-eq 19-and$m.mismatchCount-eq 0-and$rows.Count-eq 19-and$seen.Count-eq 19-and$bad-eq 0)}catch{$r.exists=$true;$r.parseError=$_.Exception.GetType().Name}}
$c=[ordered]@{executed=$false;ok=$false;exitCode=-1;errorType=''}
try{& powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -File (Join-Path $b 'agent99-contract-check.ps1') -SourceRoot $b 2>$null|Out-Null;$c.executed=$true;$c.exitCode=$LASTEXITCODE;$c.ok=[bool]($c.exitCode-eq 0)}catch{$c.executed=$true;$c.errorType=$_.Exception.GetType().Name}
$z=[ordered]@{};foreach($n in 'remoteWritePerformed,liveRuntimeWritePerformed,livePromotionAttempted,livePromotionPerformed,secretValueRead,privateKeyValueRead,tokenValueRead,environmentSecretRead,uiInteraction,vmPowerChange,hostReboot,serviceRestart,scheduledTaskRestart,scheduledTaskModification'.Split(',')){$z[$n]=$false}
$ready=[bool]($c.executed-and$c.ok-and$c.exitCode-eq 0)
$status=if($ready){'check_ready'}else{'check_failed_no_apply'}
$next=if($ready){'rerun_with_apply_and_trace_run_work_item_after_source_commit_and_transport_check'}else{'repair_runtime_contract_before_apply'}
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=18;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
[ordered]@{schemaVersion='agent99_remote_atomic_deploy_receipt_v1';status=$status;mode='check';sourceRevision='__SOURCE_REVISION__';expectedRuntimeFileCount=19;manifestSha256='__MANIFEST_SHA256__';plannedStagingPath='C:\Wooo\Agent99\deploy\remote-source-__STAGE_TOKEN__';runtimeManifest=[pscustomobject]$r;runtimeContract=[pscustomobject]$c;livePreflight=[ordered]@{executed=$false;reason='apply_only_after_validate_only'};transportPayloadMode='control_command_no_stdin';transientTransportPayloadStored=$false;transientTransportPayloadDeletedBeforeReceiver=$false;transportPayloadContainsSecrets=$false;operationBoundaries=$z;nextSafeAction=$next}|ConvertTo-Json -Compress -Depth 8
if(-not$ready){exit 65}'''
script = (
script.replace("__SOURCE_REVISION__", source_revision)

View File

@@ -0,0 +1,251 @@
from __future__ import annotations
import base64
import hashlib
import importlib.util
import json
import re
import sys
from pathlib import Path
import pytest
ROOT = Path(__file__).resolve().parents[3]
BROKER = ROOT / "agent99-host110-backup-runtime-broker.ps1"
EXECUTOR = ROOT / "scripts" / "backup" / "host110-backup-runtime-executor.sh"
VERIFIER = ROOT / "scripts" / "backup" / "verify-host110-backup-runtime.py"
EXPECTED_RUNTIME_FILES = {
"common.sh",
"backup-all.sh",
"backup-host188-products.sh",
"verify-host188-products-backup.sh",
"verify-host188-products-archive.py",
"backup-gitea.sh",
"gitea-full-backup-restore-drill.sh",
"backup-configs.sh",
"backup-awoooi.sh",
"backup-awoooi-frequent.sh",
"backup-clawbot.sh",
"backup-sentry.sh",
"check-backup-integrity.sh",
"sync-offsite-backups.sh",
"backup-offsite-readiness-gate.sh",
"verify-offsite-full-sync.sh",
"enforce-latest-only-retention.sh",
}
def _load_verifier():
spec = importlib.util.spec_from_file_location("host110_runtime_verifier", VERIFIER)
assert spec and spec.loader
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
return module
def _sha256(path: Path) -> str:
return hashlib.sha256(path.read_bytes()).hexdigest()
def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
source = BROKER.read_text(encoding="utf-8")
assert '[ValidateSet("Check", "Apply", "Verify")]' in source
assert '$TargetHost = "192.168.0.110"' in source
assert '$TargetUser = "wooo"' in source
assert '$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"' in source
assert 'keys\\agent99_ed25519' in source
assert '"credential.helper="' in source
assert '$env:GIT_TERMINAL_PROMPT = "0"' in source
assert '"merge-base", "--is-ancestor", $SourceRevision, $sourceHead' in source
assert '"checkout", "--quiet", $SourceRevision, "--"' in source
assert 'executorSha256 = $executorDigest' in source
assert 'verifierSha256 = $verifierDigest' in source
assert "/home/wooo/awoooi" not in source
assert '"StrictHostKeyChecking=yes"' in source
assert '"NumberOfPasswordPrompts=0"' in source
assert '"ConnectionAttempts=1"' in source
assert "$process.WaitForExit($TimeoutSeconds * 1000)" in source
assert "taskkill.exe /PID $process.Id /T /F" in source
assert "$process.Kill()" in source
assert 'Invoke-Agent99BoundedScp $sourcePackage.localPaths' in source
def test_windows99_broker_verify_is_independent_no_write_and_evidence_is_immutable() -> None:
source = BROKER.read_text(encoding="utf-8")
assert 'RedirectStandardInput = $StandardInputPath' in source
assert "python3 - --manifest-base64" in source
assert 'remoteWritePerformed' in source
assert '[IO.File]::Move($temporary, $Path)' in source
assert 'Move-Item -LiteralPath $temporary -Destination $Path -Force' not in source
assert 'run_identity_conflict' in source
assert 'run_identity_evidence_exists' in source
assert 'decisionProvider = "deterministic_only"' in source
assert 'criticProvider = "deterministic_only"' in source
assert 'verifier = "independent_host110_backup_runtime_python_readback"' in source
def test_host110_executor_is_manifest_bound_atomic_and_fail_closed() -> None:
source = EXECUTOR.read_text(encoding="utf-8")
array_body = source.split("readonly -a RUNTIME_FILES=(", 1)[1].split(")", 1)[0]
runtime_files = set(re.findall(r"^\s+([A-Za-z0-9_.-]+)\s*$", array_body, re.MULTILINE))
assert runtime_files == EXPECTED_RUNTIME_FILES
assert 'readonly EXPECTED_HOST_IP="192.168.0.110"' in source
assert 'readonly EXPECTED_USER="wooo"' in source
assert 'readonly DEST_ROOT="/backup/scripts"' in source
assert '--source-stage' in source
assert 'document.get("executorSha256"' in source
assert 'document.get("verifierSha256"' in source
assert 'executor_identity_failed' in source
assert 'verifier_identity_failed' in source
assert 'flock -x -w 30 9' in source
assert 'legacy_backup_runtime_active' in source
assert source.index("for name in \"${RUNTIME_FILES[@]}\"") < source.index("mv -f -- \"$temporary\"")
assert 'rollback_unverified' in source
assert 'failed_rolled_back' in source
assert 'working_digest "$dest_path"' in source
assert 'os.link(temporary, path)' in source
assert 'run_identity_receipt_exists' in source
assert 'run_identity_stage_exists' in source
assert 'run_identity_rollback_exists' in source
assert 'verify-host110-backup-runtime.py' in source
assert 'selfIdentityVerified' in source
assert '"productionServiceRestarted": False' in source
assert '"secretValuesRead": False' in source
assert "sudo " not in source
assert "docker " not in source
assert "systemctl " not in source
def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
common = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8")
restore = (ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh").read_text(encoding="utf-8")
offsite = (ROOT / "scripts/backup/backup-offsite-readiness-gate.sh").read_text(encoding="utf-8")
archive = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
for source in (common, restore, offsite):
assert "/tmp/agent99-host110-backup-runtime.lock" in source
assert "flock -s -w 60" in source
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
rows = []
for name in verifier.EXPECTED_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
verifier_digest = _sha256(VERIFIER)
source_revision = "a" * 40
run_id = "unit-exact"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "b" * 40,
"runId": run_id,
"executorSha256": "c" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
manifest_path = tmp_path / "manifest.json"
manifest_path.write_text(json.dumps(manifest), encoding="utf-8")
before = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest",
str(manifest_path),
"--destination",
str(destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
verifier.main()
result = json.loads(capsys.readouterr().out)
after = {path.name: (path.stat().st_mtime_ns, _sha256(path)) for path in destination.iterdir()}
assert result["ok"] is True
assert result["remoteWritePerformed"] is False
assert result["selfIdentityVerified"] is True
assert before == after
def test_independent_verifier_rejects_drifted_bundle(tmp_path, monkeypatch, capsys) -> None:
verifier = _load_verifier()
destination = tmp_path / "scripts"
destination.mkdir()
rows = []
for name in verifier.EXPECTED_FILES:
path = destination / name
path.write_text(f"runtime:{name}\n", encoding="utf-8")
path.chmod(0o755)
rows.append({"name": name, "sha256": _sha256(path)})
(destination / verifier.EXPECTED_FILES[0]).write_text("drifted\n", encoding="utf-8")
verifier_digest = _sha256(VERIFIER)
source_revision = "d" * 40
run_id = "unit-drift"
manifest = {
"schemaVersion": "agent99_host110_backup_runtime_source_manifest_v1",
"sourceRevision": source_revision,
"sourceHead": "e" * 40,
"runId": run_id,
"executorSha256": "f" * 64,
"verifierSha256": verifier_digest,
"files": rows,
}
encoded = base64.b64encode(json.dumps(manifest).encode()).decode()
monkeypatch.setattr(verifier, "EXPECTED_DESTINATION", destination)
monkeypatch.setattr(
sys,
"argv",
[
str(VERIFIER),
"--manifest-base64",
encoded,
"--destination",
str(destination),
"--source-revision",
source_revision,
"--run-id",
run_id,
"--expected-verifier-sha256",
verifier_digest,
],
)
with pytest.raises(SystemExit):
verifier.main()
assert "runtime_file_identity_failed" in capsys.readouterr().err
def test_agent99_runtime_lists_include_the_host110_broker_once() -> None:
sources = [
ROOT / "agent99-bootstrap.ps1",
ROOT / "agent99-contract-check.ps1",
ROOT / "agent99-deploy.ps1",
ROOT / "scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh",
ROOT / "scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1",
]
marker = "agent99-host110-backup-runtime-broker.ps1"
for path in sources:
assert marker in path.read_text(encoding="utf-8"), path

View File

@@ -36,7 +36,7 @@ def _base_case() -> dict[str, Any]:
"exists": True,
"sourceRevision": "a" * 40,
"runtimeMatched": True,
"fileCount": 18,
"fileCount": 19,
"mismatchCount": 0,
"parseError": "",
},
@@ -133,7 +133,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
)
assert "deploymentEligible = [bool]($blockingReasons.Count -eq 0)" in decision
assert "preflightGreen = [bool]$decision.deploymentEligible" in source
assert "-ExpectedRuntimeFileCount 18" in source
assert "-ExpectedRuntimeFileCount 19" in source
assert "warningReasons = $warningReasons" in source
assert "selfHealthPerformanceIssues = @(" in source
assert 'if ($safeHost -notmatch "^[A-Za-z0-9.:-]{1,128}$")' in source

View File

@@ -26,6 +26,7 @@ EXPECTED_RUNTIME_FILES = (
"agent99-control-plane.ps1",
"agent99-db-bounded-executor.ps1",
"agent99-signoz-metadata-executor.ps1",
"agent99-host110-backup-runtime-broker.ps1",
"agent99-deploy.ps1",
"agent99-register-tasks.ps1",
"agent99-alertmanager-alertchain-poll.ps1",
@@ -88,7 +89,7 @@ def test_sender_and_receiver_allow_exactly_the_deployer_runtime_bundle() -> None
)
assert preflight_count is not None
assert int(preflight_count.group(1)) == len(EXPECTED_RUNTIME_FILES)
assert "expectedRuntimeFileCount\": 18" in sender
assert "expectedRuntimeFileCount\": 19" in sender
assert "$filesByName.Count -ne $FixedRuntimeFiles.Count" in receiver
assert "required_runtime_file_missing" in receiver
assert "duplicate_runtime_file" in receiver
@@ -263,7 +264,7 @@ def test_receiver_rolls_back_failed_deploy_or_failed_independent_post_verifier()
)
assert "$livePreflight.sourceRevision -eq $sourceRevision" in source
assert "$runtimeManifest.sourceRevision -eq $sourceRevision" in source
assert '$runtimeManifest.fileCount -eq 18' in source
assert '$runtimeManifest.fileCount -eq 19' in source
assert "function Invoke-AgentWindowsControlBaseline" in source
assert "function Get-AgentRunEvidenceToken" in source
assert "$evidenceRunToken = Get-AgentRunEvidenceToken -Value $RunId" in source

View File

@@ -516,7 +516,8 @@ def test_agent99_propagates_identity_and_requires_independent_receipt() -> None:
for parameter in ("AutomationRunId", "TraceId", "WorkItemId"):
assert f"[string]${parameter} = \"\"" in control
assert f'[string]`${parameter} = ""' in bootstrap
assert 'reason = "controlled_dispatch_identity_unsafe"' in control
assert '"controlled_dispatch_identity_unsafe"' in control
assert "reason = $identityRejectionReason" in control
assert 'applyBlockedReason = "control_identity_incomplete_or_unsafe"' in function
assert '$dryRun.managerPreflightStatus -eq "ready"' in function
assert "-not $dryRun.artifactTransactionBlocked" in function