wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 2 Packages Projects Releases Wiki Activity

fix(agent99): fail closed host110 runtime replay

Browse Source
This commit is contained in:
Your Name
2026-07-19 02:40:26 +08:00
parent d1797a1c87
commit 2c2dc086b0
8 changed files with 1878 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1
View File

@@ -35,11 +35,19 @@ Write-ReplayTrace "define_functions"
$functionNames = @(
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Assert-Agent99ExactJsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult"
"Get-Agent99CommittedFailureResult",
"Assert-Agent99VerifierEvidence",
"Assert-Agent99PreflightEvidence",
"Get-Agent99TransportReceiptReadback",
"New-Agent99TransportLossReconciliation",
"Assert-Agent99ExecutorEvidence",
"Assert-Agent99ReconciliationEvidence",
"Assert-Agent99ExistingBrokerEvidence"
)
foreach ($functionName in $functionNames) {
$node = $ast.Find(
@@ -56,8 +64,11 @@ foreach ($functionName in $functionNames) {
$SourceRevision = "a" * 40
$RunId = "windows99-contract-replay"
$Mode = "Apply"
$TargetHost = "192.168.0.110"
$ExpectedFileCount = 18
$sourceHead = "b" * 40
$VerifierSha256 = "c" * 64
$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
@@ -95,8 +106,12 @@ $verifier = [pscustomobject]@{
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
destination = "/backup/scripts"
exporterDestination = "/home/wooo/scripts/backup-health-textfile-exporter.py"
remoteWritePerformed = $false
secretValuesRead = $false
selfIdentityVerified = $true
verifierSha256 = $VerifierSha256
}
$success = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_executor_v1"
@@ -127,7 +142,7 @@ $success = [pscustomobject]@{
}
Write-ReplayTrace "accept_success"
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" $VerifierSha256 | Out-Null
$topLevelRequired = @(
"schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
"runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
@@ -139,21 +154,22 @@ $topLevelRequired = @(
$verifierRequired = @(
"schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
"backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
"selfIdentityVerified"
"secretValuesRead", "selfIdentityVerified", "destination", "exporterDestination",
"verifierSha256"
)
$missingFieldRejections = 0
Write-ReplayTrace "reject_missing_top"
foreach ($field in $topLevelRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_$field"
$missingFieldRejections++
}
Write-ReplayTrace "reject_missing_verifier"
foreach ($field in $verifierRequired) {
$mutated = Copy-ReplayDocument $success
$mutated.verifier.PSObject.Properties.Remove($field)
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_verifier_$field"
$missingFieldRejections++
}
@@ -169,12 +185,12 @@ Write-ReplayTrace "reject_types"
foreach ($mutation in $typeMutations) {
$mutated = Copy-ReplayDocument $success
$mutated.($mutation.field) = $mutation.value
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_$($mutation.field)"
$typeRejections++
}
$mutated = Copy-ReplayDocument $success
$mutated.verifier.remoteWritePerformed = "false"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_verifier_remoteWritePerformed"
$typeRejections++
$cleanupPending = Copy-ReplayDocument $success
@@ -204,6 +220,174 @@ $terminalReceiptFailed.verifier = $null
Write-ReplayTrace "accept_terminal_receipt_failed"
Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
$EvidencePath = "C:\Wooo\Agent99\evidence\host110-backup-runtime\agent99-host110-backup-runtime-$RunId.json"
$preflight = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
ok = $true
sourceRevision = $SourceRevision
sourceHead = $sourceHead
fileCount = 18
backupScriptFileCount = 17
backupHealthExporterIncluded = $true
remoteWritePerformed = $false
}
$brokerSuccess = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
ok = $true
status = "verified"
mode = "Apply"
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
targetHost = $TargetHost
sourceTransport = "windows99_gitea_exact_revision_manifest"
executor = "host110_backup_runtime_executor"
verifier = "independent_host110_backup_runtime_python_readback"
decisionProvider = "deterministic_only"
criticProvider = "deterministic_only"
agentAction = "controlled_apply"
check = $preflight
apply = $success
verify = $verifier
cleanupVerified = $true
errorCode = ""
elapsedSeconds = 1.25
evidence = $EvidencePath
secretValuesRead = $false
rawSessionStored = $false
}
Write-ReplayTrace "accept_existing_success"
Assert-Agent99ExistingBrokerEvidence $brokerSuccess $EvidencePath
$brokerRequired = @(
"schemaVersion", "ok", "status", "mode", "sourceRevision", "sourceHead",
"runId", "targetHost", "sourceTransport", "executor", "verifier",
"decisionProvider", "criticProvider", "agentAction", "check", "apply",
"verify", "cleanupVerified", "errorCode", "elapsedSeconds", "evidence",
"secretValuesRead", "rawSessionStored"
)
$existingEvidenceRejections = 0
Write-ReplayTrace "reject_existing_missing"
foreach ($field in $brokerRequired) {
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated.PSObject.Properties.Remove($field)
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_missing_$field"
$existingEvidenceRejections++
}
$existingMutations = @(
@{ name = "ok_string_false"; field = "ok"; value = "false" },
@{ name = "wrong_mode"; field = "mode"; value = "Verify" },
@{ name = "wrong_run_id"; field = "runId"; value = "other-run" },
@{ name = "wrong_status"; field = "status"; value = "readback_ok" },
@{ name = "wrong_cleanup"; field = "cleanupVerified"; value = $false },
@{ name = "cleanup_string"; field = "cleanupVerified"; value = "true" },
@{ name = "elapsed_string"; field = "elapsedSeconds"; value = "1.25" }
)
Write-ReplayTrace "reject_existing_conflicts"
foreach ($mutation in $existingMutations) {
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated.($mutation.field) = $mutation.value
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_$($mutation.name)"
$existingEvidenceRejections++
}
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated.apply.runId = "other-run"
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_apply_run_conflict"
$existingEvidenceRejections++
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated.verify.PSObject.Properties.Remove("selfIdentityVerified")
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_verify_nested_missing"
$existingEvidenceRejections++
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated | Add-Member -NotePropertyName unexpected -NotePropertyValue $true
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_top"
$existingEvidenceRejections++
$mutated = Copy-ReplayDocument $brokerSuccess
$mutated.apply | Add-Member -NotePropertyName unexpected -NotePropertyValue $true
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_apply"
$existingEvidenceRejections++
$brokerFailure = Copy-ReplayDocument $brokerSuccess
$brokerFailure.ok = $false
$brokerFailure.status = "failed"
$brokerFailure.cleanupVerified = $false
$brokerFailure.errorCode = "host110_apply_cleanup_pending"
$brokerFailure.apply = $cleanupPending
$brokerFailure.verify = $null
Write-ReplayTrace "accept_existing_failure"
Assert-Agent99ExistingBrokerEvidence $brokerFailure $EvidencePath
$failureWithoutPreflight = Copy-ReplayDocument $brokerFailure
$failureWithoutPreflight.check = $null
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithoutPreflight $EvidencePath } "failure_preflight_missing"
$existingEvidenceRejections++
$failureWithImpossibleVerifier = Copy-ReplayDocument $brokerFailure
$failureWithImpossibleVerifier.verify = $verifier
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithImpossibleVerifier $EvidencePath } "failure_verifier_conflict"
$existingEvidenceRejections++
$postcommitCleanupFailure = Copy-ReplayDocument $brokerSuccess
$postcommitCleanupFailure.ok = $false
$postcommitCleanupFailure.status = "failed"
$postcommitCleanupFailure.cleanupVerified = $false
$postcommitCleanupFailure.errorCode = "remote_source_stage_cleanup_failed"
Assert-Agent99ExistingBrokerEvidence $postcommitCleanupFailure $EvidencePath
$malformedFailure = Copy-ReplayDocument $brokerFailure
$malformedFailure.ok = "false"
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $malformedFailure $EvidencePath } "malformed_failure_not_upgraded"
$existingEvidenceRejections++
$sourcePackage = [pscustomobject]@{ sourceHead = $sourceHead }
$transportLoss = [pscustomobject]@{ reason = "transport_timeout"; exitCode = -2 }
$receiptReadback = [pscustomobject]@{
schemaVersion = "agent99_host110_backup_transport_receipt_readback_v1"
ok = $true
sourceRevision = $SourceRevision
sourceHead = $sourceHead
runId = $RunId
payloadReceipt = $payloadReceipt
payloadState = "valid"
terminalReceipt = $terminalReceipt
terminalState = "missing"
terminalStatus = ""
terminalCleanupVerified = $false
terminalExitCode = -1
remoteWritePerformed = $false
secretValuesRead = $false
}
Write-ReplayTrace "reconcile_terminal_missing"
$terminalMissing = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $receiptReadback $verifier
if (
$terminalMissing.status -ne "committed_verified_terminal_missing" -or
-not $terminalMissing.payloadCommitted -or
-not $terminalMissing.destinationVerified -or
$terminalMissing.retryApplyAllowed
) { throw "transport_terminal_missing_reconciliation_failed" }
Assert-Agent99ReconciliationEvidence $terminalMissing $sourceHead
$terminalPresentReadback = Copy-ReplayDocument $receiptReadback
$terminalPresentReadback.terminalState = "valid"
$terminalPresentReadback.terminalStatus = "verified"
$terminalPresentReadback.terminalCleanupVerified = $true
$terminalPresentReadback.terminalExitCode = 0
$transportRecovered = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $terminalPresentReadback $verifier
if ($transportRecovered.status -ne "committed_verified_transport_lost" -or $transportRecovered.retryApplyAllowed) {
throw "transport_verified_reconciliation_failed"
}
Assert-Agent99ReconciliationEvidence $transportRecovered $sourceHead
$conflictingTransport = Copy-ReplayDocument $transportRecovered
$conflictingTransport.terminalStatus = "cleanup_pending"
Assert-Rejected { Assert-Agent99ReconciliationEvidence $conflictingTransport $sourceHead } "transport_terminal_status_conflict"
$existingEvidenceRejections++
$committedUnknown = New-Agent99TransportLossReconciliation `
$transportLoss $sourcePackage $null $null "receipt_readback_unavailable" "verifier_unavailable"
if (
$committedUnknown.status -ne "committed_unknown" -or
$committedUnknown.payloadCommitted -or
$committedUnknown.destinationVerified -or
$committedUnknown.retryApplyAllowed
) { throw "transport_unknown_reconciliation_failed" }
Assert-Agent99ReconciliationEvidence $committedUnknown $sourceHead
Write-ReplayTrace "complete"
[pscustomobject]@{
schemaVersion = "agent99_host110_broker_contract_replay_v1"
@@ -213,6 +397,11 @@ Write-ReplayTrace "complete"
cleanupPendingPreserved = $true
committedSignalPreserved = $true
terminalReceiptFailurePreserved = $true
existingSuccessAccepted = $true
existingFailurePreserved = $true
existingEvidenceRejections = $existingEvidenceRejections
terminalMissingReconciled = $true
committedUnknownPreserved = $true
missingFieldRejections = $missingFieldRejections
typeRejections = $typeRejections
remoteWritePerformed = $false

523
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py
View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import base64
import fcntl
import grp
import hashlib
import importlib.util
@@ -94,6 +95,88 @@ def _write_executable(path: Path, source: str) -> None:
path.chmod(0o755)
def _write_fcntl_flock(path: Path) -> None:
_write_executable(
path,
"""#!/usr/bin/env python3
import fcntl
import sys
import time
args = sys.argv[1:]
operation = fcntl.LOCK_SH if "-s" in args else fcntl.LOCK_EX
wait_seconds = float(args[args.index("-w") + 1]) if "-w" in args else 0.0
descriptor = int(args[-1])
deadline = time.monotonic() + wait_seconds
while True:
try:
fcntl.flock(descriptor, operation | fcntl.LOCK_NB)
raise SystemExit(0)
except BlockingIOError:
if time.monotonic() >= deadline:
raise SystemExit(1)
time.sleep(0.02)
""",
)
def _build_shared_lock_gate_fixture(
tmp_path: Path,
entrypoint: str,
*,
reject_fd_reopen: bool = False,
body_sleep_seconds: float = 0.0,
) -> tuple[Path, Path, Path, dict[str, str]]:
runtime_root = tmp_path / "runtime-scripts"
runtime_root.mkdir()
lock_path = tmp_path / "runtime.lock"
fake_bin = tmp_path / "fake-bin"
fake_bin.mkdir()
_write_fcntl_flock(fake_bin / "flock")
source_map = {
"common": ROOT / "scripts/backup/common.sh",
"restore": ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh",
"offsite": ROOT / "scripts/backup/backup-offsite-readiness-gate.sh",
}
source = source_map[entrypoint].read_text(encoding="utf-8")
source = source.replace("/backup/scripts", str(runtime_root))
source = source.replace("/tmp/agent99-host110-backup-runtime.lock", str(lock_path))
source = source.replace("flock -s -w 60", "flock -s -w 1")
if reject_fd_reopen:
source, replacement_count = re.subn(
r"(?m)^(\s*)exec 197>>.*$",
r"\1printf 'FD_REOPENED=1\\n' >&2; exit 98",
source,
)
assert replacement_count == 1
body = "printf 'BODY_REACHED=1\\n'\n"
if body_sleep_seconds:
body += f"sleep {body_sleep_seconds}\n"
if entrypoint == "common":
common_path = runtime_root / "common.sh"
gate_source = source.split("# WOOO AIOps - 備份共用函式庫", 1)[0]
_write_executable(common_path, gate_source)
script_path = runtime_root / "backup-gitea.sh"
_write_executable(
script_path,
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
f"source {str(common_path)!r}\n"
+ body,
)
else:
product_start = "DUMP_ZIP=" if entrypoint == "restore" else "BACKUP_BASE="
gate_source = source.split(product_start, 1)[0]
script_path = runtime_root / source_map[entrypoint].name
_write_executable(script_path, gate_source + body)
environment = os.environ.copy()
environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
return script_path, runtime_root, lock_path, environment
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
destination = tmp_path / "backup-scripts"
exporter_root = tmp_path / "exporter"
@@ -337,6 +420,10 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
assert '$property = $Object.PSObject.Properties[$Name]' in source
assert 'if ($null -eq $property) { return $false }' in source
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
assert 'Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage' in source
assert 'status = "committed_unknown"' in source
assert '"committed_verified_terminal_missing"' in source
assert 'retryApplyAllowed = $false' in source
assert 'throw "host110_apply_$([string]$apply.status)"' in source
@@ -405,11 +492,19 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w
for function_name in (
"Test-Agent99JsonField",
"Assert-Agent99JsonFields",
"Assert-Agent99ExactJsonFields",
"Convert-Agent99ExecutorDocument",
"Assert-Agent99ApplyResultFields",
"Assert-Agent99VerifierFields",
"Get-Agent99ExecutorResult",
"Get-Agent99CommittedFailureResult",
"Assert-Agent99VerifierEvidence",
"Assert-Agent99PreflightEvidence",
"Get-Agent99TransportReceiptReadback",
"New-Agent99TransportLossReconciliation",
"Assert-Agent99ExecutorEvidence",
"Assert-Agent99ReconciliationEvidence",
"Assert-Agent99ExistingBrokerEvidence",
):
assert f'"{function_name}"' in source
assert "$mutated.PSObject.Properties.Remove($field)" in source
@@ -417,12 +512,188 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w
assert 'status = "cleanup_pending"' in source
assert 'status = "committed_signal_deferred"' in source
assert 'status = "terminal_receipt_failed"' in source
assert 'ok = "false"' in source
assert 'status -ne "committed_verified_terminal_missing"' in source
assert 'status -ne "committed_unknown"' in source
assert "existingEvidenceRejections" in source
assert "malformed_failure_not_upgraded" in source
assert "remoteWritePerformed = $false" in source
assert "secretValuesRead = $false" in source
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
assert forbidden not in source
def test_existing_broker_evidence_replay_is_exact_typed_and_nested_fail_closed() -> None:
source = BROKER.read_text(encoding="utf-8")
exact_guard = _powershell_function(source, "Assert-Agent99ExactJsonFields")
existing = _powershell_function(source, "Assert-Agent99ExistingBrokerEvidence")
reconciliation = _powershell_function(source, "Assert-Agent99ReconciliationEvidence")
assert "$properties.Count -ne $Contract.Count" in exact_guard
assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard
for field in (
"schemaVersion",
"ok",
"status",
"mode",
"sourceRevision",
"sourceHead",
"runId",
"targetHost",
"sourceTransport",
"executor",
"verifier",
"decisionProvider",
"criticProvider",
"agentAction",
"check",
"apply",
"verify",
"cleanupVerified",
"errorCode",
"elapsedSeconds",
"evidence",
"secretValuesRead",
"rawSessionStored",
):
assert re.search(rf"^\s+{field} = \"", existing, re.MULTILINE), field
assert "Assert-Agent99PreflightEvidence $Existing.check" in existing
assert "Assert-Agent99ExecutorEvidence $Existing.apply" in existing
assert "Assert-Agent99ReconciliationEvidence $Existing.apply" in existing
assert "Assert-Agent99VerifierEvidence $Existing.verify" in existing
assert 'throw "existing_broker_apply_failure_evidence_missing"' in existing
assert 'retryApplyAllowed = "Boolean"' in reconciliation
assert "host110_existing_reconciliation_payload_truth_mismatch" in reconciliation
def test_transport_loss_reconciliation_is_read_only_run_bound_and_never_retries_apply() -> None:
source = BROKER.read_text(encoding="utf-8")
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
reconcile = _powershell_function(source, "Invoke-Agent99TransportLossReconciliation")
classify = _powershell_function(source, "New-Agent99TransportLossReconciliation")
for expected in (
"agent99-backup-runtime-deploy-{args.run_id}.json",
"agent99-backup-runtime-terminal-{args.run_id}.json",
'set(document) == set(contract)',
'payload["status"] == "payload_verified"',
'terminal["payloadReceipt"] == str(payload_path)',
'"remoteWritePerformed": False',
'"secretValuesRead": False',
):
assert expected in reader
assert "Invoke-Agent99BoundedSsh $command 60" in reconcile
assert "Invoke-Agent99NoWriteVerifier $SourcePackage" in reconcile
assert 'status = "committed_unknown"' in classify
assert '"committed_verified_terminal_missing"' in classify
assert '"committed_verified_transport_lost"' in classify
assert "retryApplyAllowed = $false" in classify
for forbidden in ("rm ", "mv ", "install ", "scp", "--mode apply"):
assert forbidden not in reconcile
def test_transport_receipt_readback_rejects_extra_and_wrong_typed_documents(tmp_path: Path) -> None:
source = BROKER.read_text(encoding="utf-8")
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
status_root = tmp_path / "status"
status_root.mkdir()
run_id = "transport-loss-replay"
source_revision = "a" * 40
source_head = "b" * 40
verifier_sha256 = "c" * 64
payload_path = status_root / f"agent99-backup-runtime-deploy-{run_id}.json"
terminal_path = status_root / f"agent99-backup-runtime-terminal-{run_id}.json"
payload = {
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
"status": "payload_verified",
"sourceRevision": source_revision,
"sourceHead": source_head,
"runId": run_id,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"rollbackAttempted": False,
"rollbackPerformed": False,
"rollbackVerified": False,
"zeroResidueVerified": True,
"zeroResidueScope": "run_owned_destination_temps",
"verifierSha256": verifier_sha256,
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": 1,
}
terminal = {
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
"status": "verified",
"ok": True,
"payloadCommitted": True,
"payloadReceipt": str(payload_path),
"sourceRevision": source_revision,
"sourceHead": source_head,
"runId": run_id,
"fileCount": 18,
"backupScriptFileCount": 17,
"backupHealthExporterIncluded": True,
"stageCleanupVerified": True,
"rollbackPrestateCleanupVerified": True,
"cleanupVerified": True,
"independentVerifierVerified": True,
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
"deferredSignal": None,
"terminalExitCode": 0,
"verifierSha256": verifier_sha256,
"executorHost": "192.168.0.110",
"productionServiceRestarted": False,
"secretValuesRead": False,
"writtenAt": 2,
}
def run_readback() -> dict[str, object]:
result = subprocess.run(
[
sys.executable,
"-",
"--status-root",
str(status_root),
"--source-revision",
source_revision,
"--source-head",
source_head,
"--run-id",
run_id,
"--verifier-sha256",
verifier_sha256,
],
input=reader,
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
return json.loads(result.stdout)
payload_path.write_text(json.dumps(payload), encoding="utf-8")
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
valid = run_readback()
assert valid["payloadState"] == "valid"
assert valid["terminalState"] == "valid"
assert valid["terminalStatus"] == "verified"
payload["unexpected"] = True
payload_path.write_text(json.dumps(payload), encoding="utf-8")
assert run_readback()["payloadState"] == "invalid"
payload.pop("unexpected")
payload["writtenAt"] = "1"
payload_path.write_text(json.dumps(payload), encoding="utf-8")
assert run_readback()["payloadState"] == "invalid"
payload["writtenAt"] = 1
payload_path.write_text(json.dumps(payload), encoding="utf-8")
terminal["ok"] = "false"
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
assert run_readback()["terminalState"] == "invalid"
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
broker = BROKER.read_text(encoding="utf-8")
executor = EXECUTOR.read_text(encoding="utf-8")
@@ -854,8 +1125,258 @@ def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
for source in (common, restore, offsite):
assert "/tmp/agent99-host110-backup-runtime.lock" in source
assert "flock -s -w 60" in source
assert "realpath -e" in source
assert "readlink -f" in source
assert "identity unresolved" in source
assert "/proc/$$/fd/197" in source
assert "/dev/fd/197" in source
assert "inherited fd 197 identity mismatch" in source
assert source.index("backup_runtime_inherited_fd_status") < source.index("exec 197>>")
assert 'resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}"' in common
assert '${PWD}/${raw_path}' in common
assert '[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0' not in common
for source in (restore, offsite):
assert 'resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}"' in source
assert '&& [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]' not in source
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" not in archive
assert 'Path(f"/proc/{os.getpid()}/fd")' in archive
assert 'Path("/dev/fd/197")' in archive
assert "os.dup(197)" in archive
def test_common_shared_lock_preserves_resolved_nonproduction_fixture_behavior(tmp_path: Path) -> None:
gate_source = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8").split(
"# WOOO AIOps - 備份共用函式庫",
1,
)[0]
gate_fixture = tmp_path / "common-gate.sh"
_write_executable(gate_fixture, gate_source)
wrapper = tmp_path / "fixture-entrypoint.sh"
_write_executable(
wrapper,
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
f"source {str(gate_fixture)!r}\n"
"printf 'NONPRODUCTION_FIXTURE_OK=1\\n'\n",
)
result = subprocess.run(
[str(wrapper)],
check=False,
capture_output=True,
text=True,
)
assert result.returncode == 0, result.stderr
assert "NONPRODUCTION_FIXTURE_OK=1" in result.stdout
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
@pytest.mark.parametrize("invocation", ["absolute", "relative", "symlink"])
def test_backup_runtime_shared_lock_blocks_absolute_relative_and_symlink_invocation(
tmp_path: Path,
entrypoint: str,
invocation: str,
) -> None:
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
tmp_path,
entrypoint,
)
command_path = str(script_path)
cwd = runtime_root
if invocation == "relative":
command_path = f"./{script_path.name}"
elif invocation == "symlink":
link_path = tmp_path / f"{entrypoint}-entrypoint-link"
link_path.symlink_to(script_path)
command_path = str(link_path)
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
lock_handle = lock_path.open("a+")
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
result = subprocess.run(
[str(command_path)],
cwd=cwd,
check=False,
capture_output=True,
text=True,
env=environment,
timeout=3,
)
finally:
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
lock_handle.close()
assert result.returncode == 75
assert "backup runtime deployment is active" in result.stderr
assert "BODY_REACHED=1" not in result.stdout
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
def test_backup_runtime_shared_lock_reuses_exact_inherited_fd_without_reopen(
tmp_path: Path,
entrypoint: str,
) -> None:
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
tmp_path,
entrypoint,
reject_fd_reopen=True,
body_sleep_seconds=0.5,
)
wrapper = tmp_path / "run-with-inherited-lock.sh"
_write_executable(
wrapper,
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
f"exec 197>>{str(lock_path)!r}\n"
"flock -s -w 1 197\n"
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=forged_external_value\n"
f"exec {str(script_path)!r}\n",
)
process = subprocess.Popen(
[str(wrapper)],
cwd=runtime_root,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
env=environment,
)
assert process.stdout is not None
assert process.stdout.readline().strip() == "BODY_REACHED=1"
contender = lock_path.open("a+")
try:
with pytest.raises(BlockingIOError):
fcntl.flock(contender.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
finally:
contender.close()
stdout, stderr = process.communicate(timeout=3)
assert process.returncode == 0, stderr
assert "FD_REOPENED=1" not in stderr
assert stdout == ""
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
def test_backup_runtime_shared_lock_rejects_inherited_fd_for_another_file(
tmp_path: Path,
entrypoint: str,
) -> None:
script_path, runtime_root, _, environment = _build_shared_lock_gate_fixture(tmp_path, entrypoint)
wrong_lock = tmp_path / "wrong-runtime.lock"
wrapper = tmp_path / "run-with-wrong-inherited-lock.sh"
_write_executable(
wrapper,
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
f"exec 197>>{str(wrong_lock)!r}\n"
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=1\n"
f"exec {str(script_path)!r}\n",
)
result = subprocess.run(
[str(wrapper)],
cwd=runtime_root,
check=False,
capture_output=True,
text=True,
env=environment,
timeout=3,
)
assert result.returncode == 69
assert "inherited fd 197 identity mismatch" in result.stderr
assert "BODY_REACHED=1" not in result.stdout
def test_host188_archive_lock_ignores_forged_env_and_validates_inherited_fd(tmp_path: Path) -> None:
runtime_root = tmp_path / "runtime-scripts"
runtime_root.mkdir()
lock_path = tmp_path / "runtime.lock"
archive_path = runtime_root / "verify-host188-products-archive.py"
archive_source = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
archive_source = archive_source.replace('Path("/backup/scripts")', f"Path({str(runtime_root)!r})")
archive_source = archive_source.replace(
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")',
f"RUNTIME_LOCK = Path({str(lock_path)!r})",
)
archive_source = archive_source.replace("time.monotonic() + 60", "time.monotonic() + 0.2")
_write_executable(archive_path, archive_source)
runner = tmp_path / "archive-lock-replay.py"
runner.write_text(
"""import fcntl
import importlib.util
import os
import pathlib
import sys
archive_path = pathlib.Path(sys.argv[1])
lock_path = pathlib.Path(sys.argv[2])
mode = sys.argv[3]
spec = importlib.util.spec_from_file_location("host188_archive_lock_replay", archive_path)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
if mode in {"exact", "wrong"}:
selected = lock_path if mode == "exact" else lock_path.with_name("wrong-runtime.lock")
selected.touch()
inherited = selected.open("a+")
fcntl.flock(inherited.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
os.dup2(inherited.fileno(), 197)
try:
handle = module.acquire_runtime_lock()
except RuntimeError as exc:
print(str(exc))
raise SystemExit(69)
else:
print("LOCK_OK=1")
if handle is not None:
handle.close()
""",
encoding="utf-8",
)
environment = os.environ.copy()
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
lock_handle = lock_path.open("a+")
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
try:
forged = subprocess.run(
[sys.executable, str(runner), str(archive_path), str(lock_path), "forged"],
check=False,
capture_output=True,
text=True,
env=environment,
timeout=3,
)
finally:
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
lock_handle.close()
exact = subprocess.run(
[sys.executable, str(runner), str(archive_path), str(lock_path), "exact"],
check=False,
capture_output=True,
text=True,
env=environment,
timeout=3,
)
wrong = subprocess.run(
[sys.executable, str(runner), str(archive_path), str(lock_path), "wrong"],
check=False,
capture_output=True,
text=True,
env=environment,
timeout=3,
)
assert forged.returncode == 69
assert "backup_runtime_deployment_active" in forged.stdout
assert exact.returncode == 0, exact.stdout + exact.stderr
assert "LOCK_OK=1" in exact.stdout
assert wrong.returncode == 69
assert "backup_runtime_inherited_fd_mismatch" in wrong.stdout
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None: