fix(agent99): fail closed host110 runtime replay
This commit is contained in:
Your Name
@@ -44,6 +44,212 @@ $RuntimeRelativePaths = @(
|
||||
)
|
||||
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
|
||||
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
|
||||
$TransportReconciliationReadback = @'
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def is_string(value):
|
||||
return type(value) is str
|
||||
|
||||
|
||||
def is_boolean(value):
|
||||
return type(value) is bool
|
||||
|
||||
|
||||
def is_integer(value):
|
||||
return type(value) is int
|
||||
|
||||
|
||||
def exact_document(document, contract):
|
||||
return (
|
||||
type(document) is dict
|
||||
and set(document) == set(contract)
|
||||
and all(contract[name](document[name]) for name in contract)
|
||||
)
|
||||
|
||||
|
||||
def read_document(path):
|
||||
if path.is_symlink():
|
||||
return "invalid", None
|
||||
if not path.exists():
|
||||
return "missing", None
|
||||
if not path.is_file() or path.stat().st_size > 65536:
|
||||
return "invalid", None
|
||||
try:
|
||||
return "present", json.loads(path.read_text(encoding="utf-8"))
|
||||
except (OSError, UnicodeError, json.JSONDecodeError):
|
||||
return "invalid", None
|
||||
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument("--status-root", required=True)
|
||||
parser.add_argument("--source-revision", required=True)
|
||||
parser.add_argument("--source-head", required=True)
|
||||
parser.add_argument("--run-id", required=True)
|
||||
parser.add_argument("--verifier-sha256", required=True)
|
||||
args = parser.parse_args()
|
||||
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
|
||||
raise SystemExit(64)
|
||||
if not re.fullmatch(r"[0-9a-f]{40}", args.source_head):
|
||||
raise SystemExit(64)
|
||||
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
|
||||
raise SystemExit(64)
|
||||
if not re.fullmatch(r"[0-9a-f]{64}", args.verifier_sha256):
|
||||
raise SystemExit(64)
|
||||
|
||||
status_root = Path(args.status_root)
|
||||
if status_root.is_symlink() or not status_root.is_dir():
|
||||
raise SystemExit(66)
|
||||
payload_path = status_root / f"agent99-backup-runtime-deploy-{args.run_id}.json"
|
||||
terminal_path = status_root / f"agent99-backup-runtime-terminal-{args.run_id}.json"
|
||||
|
||||
payload_contract = {
|
||||
"schemaVersion": is_string,
|
||||
"status": is_string,
|
||||
"sourceRevision": is_string,
|
||||
"sourceHead": is_string,
|
||||
"runId": is_string,
|
||||
"fileCount": is_integer,
|
||||
"backupScriptFileCount": is_integer,
|
||||
"backupHealthExporterIncluded": is_boolean,
|
||||
"rollbackAttempted": is_boolean,
|
||||
"rollbackPerformed": is_boolean,
|
||||
"rollbackVerified": is_boolean,
|
||||
"zeroResidueVerified": is_boolean,
|
||||
"zeroResidueScope": is_string,
|
||||
"verifierSha256": is_string,
|
||||
"executorHost": is_string,
|
||||
"productionServiceRestarted": is_boolean,
|
||||
"secretValuesRead": is_boolean,
|
||||
"writtenAt": is_integer,
|
||||
}
|
||||
terminal_contract = {
|
||||
"schemaVersion": is_string,
|
||||
"status": is_string,
|
||||
"ok": is_boolean,
|
||||
"payloadCommitted": is_boolean,
|
||||
"payloadReceipt": is_string,
|
||||
"sourceRevision": is_string,
|
||||
"sourceHead": is_string,
|
||||
"runId": is_string,
|
||||
"fileCount": is_integer,
|
||||
"backupScriptFileCount": is_integer,
|
||||
"backupHealthExporterIncluded": is_boolean,
|
||||
"stageCleanupVerified": is_boolean,
|
||||
"rollbackPrestateCleanupVerified": is_boolean,
|
||||
"cleanupVerified": is_boolean,
|
||||
"independentVerifierVerified": is_boolean,
|
||||
"zeroResidueScope": is_string,
|
||||
"deferredSignal": lambda value: value is None or is_string(value),
|
||||
"terminalExitCode": is_integer,
|
||||
"verifierSha256": is_string,
|
||||
"executorHost": is_string,
|
||||
"productionServiceRestarted": is_boolean,
|
||||
"secretValuesRead": is_boolean,
|
||||
"writtenAt": is_integer,
|
||||
}
|
||||
|
||||
payload_state, payload = read_document(payload_path)
|
||||
if payload_state == "present":
|
||||
valid_payload = exact_document(payload, payload_contract) and (
|
||||
payload["schemaVersion"] == "agent99_host110_backup_runtime_deploy_receipt_v3"
|
||||
and payload["status"] == "payload_verified"
|
||||
and payload["sourceRevision"] == args.source_revision
|
||||
and payload["sourceHead"] == args.source_head
|
||||
and payload["runId"] == args.run_id
|
||||
and payload["fileCount"] == 18
|
||||
and payload["backupScriptFileCount"] == 17
|
||||
and payload["backupHealthExporterIncluded"] is True
|
||||
and payload["rollbackAttempted"] is False
|
||||
and payload["rollbackPerformed"] is False
|
||||
and payload["rollbackVerified"] is False
|
||||
and payload["zeroResidueVerified"] is True
|
||||
and payload["zeroResidueScope"] == "run_owned_destination_temps"
|
||||
and payload["verifierSha256"] == args.verifier_sha256
|
||||
and payload["executorHost"] == "192.168.0.110"
|
||||
and payload["productionServiceRestarted"] is False
|
||||
and payload["secretValuesRead"] is False
|
||||
)
|
||||
payload_state = "valid" if valid_payload else "invalid"
|
||||
|
||||
terminal_state, terminal = read_document(terminal_path)
|
||||
terminal_status = ""
|
||||
terminal_cleanup_verified = False
|
||||
terminal_exit_code = -1
|
||||
if terminal_state == "present":
|
||||
valid_terminal = exact_document(terminal, terminal_contract) and (
|
||||
terminal["schemaVersion"] == "agent99_host110_backup_runtime_terminal_receipt_v1"
|
||||
and terminal["status"] in {
|
||||
"verified", "cleanup_pending", "committed_signal_deferred"
|
||||
}
|
||||
and terminal["ok"] is (terminal["status"] == "verified")
|
||||
and terminal["payloadCommitted"] is True
|
||||
and terminal["payloadReceipt"] == str(payload_path)
|
||||
and terminal["sourceRevision"] == args.source_revision
|
||||
and terminal["sourceHead"] == args.source_head
|
||||
and terminal["runId"] == args.run_id
|
||||
and terminal["fileCount"] == 18
|
||||
and terminal["backupScriptFileCount"] == 17
|
||||
and terminal["backupHealthExporterIncluded"] is True
|
||||
and terminal["cleanupVerified"] is (
|
||||
terminal["stageCleanupVerified"]
|
||||
and terminal["rollbackPrestateCleanupVerified"]
|
||||
)
|
||||
and terminal["independentVerifierVerified"] is True
|
||||
and terminal["zeroResidueScope"]
|
||||
== "run_owned_destination_temps_and_internal_stage_prestate"
|
||||
and terminal["verifierSha256"] == args.verifier_sha256
|
||||
and terminal["executorHost"] == "192.168.0.110"
|
||||
and terminal["productionServiceRestarted"] is False
|
||||
and terminal["secretValuesRead"] is False
|
||||
)
|
||||
if valid_terminal and terminal["status"] == "verified":
|
||||
valid_terminal = (
|
||||
terminal["cleanupVerified"] is True
|
||||
and terminal["deferredSignal"] is None
|
||||
and terminal["terminalExitCode"] == 0
|
||||
)
|
||||
elif valid_terminal and terminal["status"] == "cleanup_pending":
|
||||
valid_terminal = (
|
||||
terminal["cleanupVerified"] is False
|
||||
and terminal["terminalExitCode"] == 75
|
||||
)
|
||||
elif valid_terminal:
|
||||
signal_codes = {"INT": 130, "HUP": 129, "TERM": 143}
|
||||
valid_terminal = (
|
||||
terminal["cleanupVerified"] is True
|
||||
and terminal["deferredSignal"] in signal_codes
|
||||
and terminal["terminalExitCode"]
|
||||
== signal_codes[terminal["deferredSignal"]]
|
||||
)
|
||||
terminal_state = "valid" if valid_terminal else "invalid"
|
||||
if valid_terminal:
|
||||
terminal_status = terminal["status"]
|
||||
terminal_cleanup_verified = terminal["cleanupVerified"]
|
||||
terminal_exit_code = terminal["terminalExitCode"]
|
||||
|
||||
print(json.dumps({
|
||||
"schemaVersion": "agent99_host110_backup_transport_receipt_readback_v1",
|
||||
"ok": True,
|
||||
"sourceRevision": args.source_revision,
|
||||
"sourceHead": args.source_head,
|
||||
"runId": args.run_id,
|
||||
"payloadReceipt": str(payload_path),
|
||||
"payloadState": payload_state,
|
||||
"terminalReceipt": str(terminal_path),
|
||||
"terminalState": terminal_state,
|
||||
"terminalStatus": terminal_status,
|
||||
"terminalCleanupVerified": terminal_cleanup_verified,
|
||||
"terminalExitCode": terminal_exit_code,
|
||||
"remoteWritePerformed": False,
|
||||
"secretValuesRead": False,
|
||||
}, ensure_ascii=True, sort_keys=True, separators=(",", ":")))
|
||||
'@
|
||||
|
||||
function Test-Agent99SafeIdentity {
|
||||
param([string]$Value)
|
||||
@@ -245,6 +451,9 @@ function New-Agent99SourcePackage {
|
||||
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
|
||||
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
||||
$reconciliationPath = Join-Path $SourceRoot "host110-backup-transport-reconciliation.py"
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($reconciliationPath, $TransportReconciliationReadback, $encoding)
|
||||
$manifest = [ordered]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
|
||||
sourceRevision = $SourceRevision
|
||||
@@ -254,7 +463,6 @@ function New-Agent99SourcePackage {
|
||||
verifierSha256 = $verifierDigest
|
||||
files = $rows
|
||||
}
|
||||
$encoding = New-Object System.Text.UTF8Encoding($false)
|
||||
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
|
||||
$localPaths += $executorPath, $verifierPath, $ManifestPath
|
||||
return [pscustomobject]@{
|
||||
@@ -262,6 +470,7 @@ function New-Agent99SourcePackage {
|
||||
executorSha256 = $executorDigest
|
||||
verifierSha256 = $verifierDigest
|
||||
verifierPath = $verifierPath
|
||||
reconciliationPath = $reconciliationPath
|
||||
manifestPath = $ManifestPath
|
||||
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
|
||||
localPaths = $localPaths
|
||||
@@ -272,7 +481,7 @@ function Test-Agent99JsonField {
|
||||
param(
|
||||
[object]$Object,
|
||||
[string]$Name,
|
||||
[ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")]
|
||||
[ValidateSet("String", "Boolean", "Integer", "Number", "Object", "ObjectOrNull", "Null", "NullOrString")]
|
||||
[string]$Kind
|
||||
)
|
||||
|
||||
@@ -284,7 +493,9 @@ function Test-Agent99JsonField {
|
||||
"String" { return [bool]($value -is [string]) }
|
||||
"Boolean" { return [bool]($value -is [bool]) }
|
||||
"Integer" { return [bool]($value -is [int] -or $value -is [long]) }
|
||||
"Number" { return [bool]($value -is [int] -or $value -is [long] -or $value -is [double] -or $value -is [decimal]) }
|
||||
"Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) }
|
||||
"ObjectOrNull" { return [bool]($null -eq $value -or $value -is [System.Management.Automation.PSCustomObject]) }
|
||||
"Null" { return [bool]($null -eq $value) }
|
||||
"NullOrString" { return [bool]($null -eq $value -or $value -is [string]) }
|
||||
}
|
||||
@@ -301,6 +512,17 @@ function Assert-Agent99JsonFields {
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99ExactJsonFields {
|
||||
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
|
||||
|
||||
Assert-Agent99JsonFields $Object $Contract $ErrorCode
|
||||
$properties = @($Object.PSObject.Properties)
|
||||
if ($properties.Count -ne $Contract.Count) { throw $ErrorCode }
|
||||
foreach ($property in $properties) {
|
||||
if (-not $Contract.ContainsKey([string]$property.Name)) { throw $ErrorCode }
|
||||
}
|
||||
}
|
||||
|
||||
function Convert-Agent99ExecutorDocument {
|
||||
param(
|
||||
[object]$Transport,
|
||||
@@ -352,8 +574,17 @@ function Convert-Agent99ExecutorDocument {
|
||||
function Assert-Agent99ApplyResultFields {
|
||||
param([object]$Parsed, [string]$VerifierKind)
|
||||
|
||||
Assert-Agent99JsonFields $Parsed @{
|
||||
Assert-Agent99ExactJsonFields $Parsed @{
|
||||
schemaVersion = "String"
|
||||
mode = "String"
|
||||
status = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
runId = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
payloadCommitted = "Boolean"
|
||||
rollbackAttempted = "Boolean"
|
||||
rollbackPerformed = "Boolean"
|
||||
@@ -375,7 +606,7 @@ function Assert-Agent99ApplyResultFields {
|
||||
function Assert-Agent99VerifierFields {
|
||||
param([object]$Verifier)
|
||||
|
||||
Assert-Agent99JsonFields $Verifier @{
|
||||
Assert-Agent99ExactJsonFields $Verifier @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
@@ -383,13 +614,17 @@ function Assert-Agent99VerifierFields {
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
destination = "String"
|
||||
exporterDestination = "String"
|
||||
remoteWritePerformed = "Boolean"
|
||||
secretValuesRead = "Boolean"
|
||||
selfIdentityVerified = "Boolean"
|
||||
verifierSha256 = "String"
|
||||
} "host110_apply_verifier_required_field_failed"
|
||||
}
|
||||
|
||||
function Get-Agent99ExecutorResult {
|
||||
param([object]$Transport, [string]$ExpectedMode)
|
||||
param([object]$Transport, [string]$ExpectedMode, [string]$ExpectedVerifierSha256 = "")
|
||||
|
||||
$parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode
|
||||
if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" }
|
||||
@@ -419,8 +654,13 @@ function Get-Agent99ExecutorResult {
|
||||
[int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$parsed.verifier.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$parsed.verifier.backupHealthExporterIncluded -or
|
||||
[string]$parsed.verifier.destination -ne "/backup/scripts" -or
|
||||
[string]$parsed.verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
||||
[bool]$parsed.verifier.remoteWritePerformed -or
|
||||
-not [bool]$parsed.verifier.selfIdentityVerified
|
||||
[bool]$parsed.verifier.secretValuesRead -or
|
||||
-not [bool]$parsed.verifier.selfIdentityVerified -or
|
||||
$ExpectedVerifierSha256 -notmatch "^[0-9a-f]{64}$" -or
|
||||
[string]$parsed.verifier.verifierSha256 -ne $ExpectedVerifierSha256
|
||||
) {
|
||||
throw "host110_apply_independent_verifier_contract_failed"
|
||||
}
|
||||
@@ -497,17 +737,7 @@ function Get-Agent99VerifierResult {
|
||||
} catch {
|
||||
throw "host110_verify_receipt_invalid"
|
||||
}
|
||||
Assert-Agent99JsonFields $parsed @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
runId = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
remoteWritePerformed = "Boolean"
|
||||
verifierSha256 = "String"
|
||||
} "host110_verify_receipt_required_field_failed"
|
||||
Assert-Agent99VerifierFields $parsed
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$parsed.ok -or
|
||||
@@ -516,7 +746,11 @@ function Get-Agent99VerifierResult {
|
||||
[int]$parsed.fileCount -ne $ExpectedFileCount -or
|
||||
[int]$parsed.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$parsed.backupHealthExporterIncluded -or
|
||||
[string]$parsed.destination -ne "/backup/scripts" -or
|
||||
[string]$parsed.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
||||
[bool]$parsed.remoteWritePerformed -or
|
||||
[bool]$parsed.secretValuesRead -or
|
||||
-not [bool]$parsed.selfIdentityVerified -or
|
||||
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
|
||||
) {
|
||||
throw "host110_verify_receipt_contract_failed"
|
||||
@@ -524,6 +758,463 @@ function Get-Agent99VerifierResult {
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function Assert-Agent99VerifierEvidence {
|
||||
param([object]$Verifier, [string]$ExpectedVerifierSha256 = "")
|
||||
|
||||
Assert-Agent99VerifierFields $Verifier
|
||||
if (
|
||||
[string]$Verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
||||
-not [bool]$Verifier.ok -or
|
||||
[string]$Verifier.sourceRevision -ne $SourceRevision -or
|
||||
[string]$Verifier.runId -ne $RunId -or
|
||||
[int64]$Verifier.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$Verifier.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$Verifier.backupHealthExporterIncluded -or
|
||||
[string]$Verifier.destination -ne "/backup/scripts" -or
|
||||
[string]$Verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
||||
[bool]$Verifier.remoteWritePerformed -or
|
||||
[bool]$Verifier.secretValuesRead -or
|
||||
-not [bool]$Verifier.selfIdentityVerified -or
|
||||
[string]$Verifier.verifierSha256 -notmatch "^[0-9a-f]{64}$" -or
|
||||
($ExpectedVerifierSha256 -and [string]$Verifier.verifierSha256 -ne $ExpectedVerifierSha256)
|
||||
) {
|
||||
throw "host110_existing_verifier_contract_failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99PreflightEvidence {
|
||||
param([object]$Preflight, [string]$ExpectedSourceHead)
|
||||
|
||||
Assert-Agent99ExactJsonFields $Preflight @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
fileCount = "Integer"
|
||||
backupScriptFileCount = "Integer"
|
||||
backupHealthExporterIncluded = "Boolean"
|
||||
remoteWritePerformed = "Boolean"
|
||||
} "host110_existing_preflight_required_field_failed"
|
||||
if (
|
||||
[string]$Preflight.schemaVersion -ne "agent99_host110_backup_runtime_preflight_v1" -or
|
||||
-not [bool]$Preflight.ok -or
|
||||
[string]$Preflight.sourceRevision -ne $SourceRevision -or
|
||||
[string]$Preflight.sourceHead -ne $ExpectedSourceHead -or
|
||||
[int64]$Preflight.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$Preflight.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$Preflight.backupHealthExporterIncluded -or
|
||||
[bool]$Preflight.remoteWritePerformed
|
||||
) {
|
||||
throw "host110_existing_preflight_contract_failed"
|
||||
}
|
||||
}
|
||||
|
||||
function Get-Agent99TransportReceiptReadback {
|
||||
param([object]$Transport, [object]$SourcePackage)
|
||||
|
||||
if (-not $Transport.ok) { throw "host110_transport_receipt_readback_failed" }
|
||||
try { $parsed = [string]$Transport.stdout | ConvertFrom-Json } catch {
|
||||
throw "host110_transport_receipt_readback_invalid"
|
||||
}
|
||||
Assert-Agent99ExactJsonFields $parsed @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
runId = "String"
|
||||
payloadReceipt = "String"
|
||||
payloadState = "String"
|
||||
terminalReceipt = "String"
|
||||
terminalState = "String"
|
||||
terminalStatus = "String"
|
||||
terminalCleanupVerified = "Boolean"
|
||||
terminalExitCode = "Integer"
|
||||
remoteWritePerformed = "Boolean"
|
||||
secretValuesRead = "Boolean"
|
||||
} "host110_transport_receipt_readback_required_field_failed"
|
||||
if (
|
||||
[string]$parsed.schemaVersion -ne "agent99_host110_backup_transport_receipt_readback_v1" -or
|
||||
-not [bool]$parsed.ok -or
|
||||
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
||||
[string]$parsed.sourceHead -ne [string]$SourcePackage.sourceHead -or
|
||||
[string]$parsed.runId -ne $RunId -or
|
||||
[string]$parsed.payloadReceipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
||||
[string]$parsed.payloadState -notin @("missing", "invalid", "valid") -or
|
||||
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
||||
[string]$parsed.terminalState -notin @("missing", "invalid", "valid") -or
|
||||
[bool]$parsed.remoteWritePerformed -or
|
||||
[bool]$parsed.secretValuesRead
|
||||
) {
|
||||
throw "host110_transport_receipt_readback_contract_failed"
|
||||
}
|
||||
if ([string]$parsed.terminalState -eq "valid") {
|
||||
if (
|
||||
[string]$parsed.terminalStatus -notin @("verified", "cleanup_pending", "committed_signal_deferred") -or
|
||||
([string]$parsed.terminalStatus -eq "verified" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 0)) -or
|
||||
([string]$parsed.terminalStatus -eq "cleanup_pending" -and ([bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 75)) -or
|
||||
([string]$parsed.terminalStatus -eq "committed_signal_deferred" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -notin @(129, 130, 143)))
|
||||
) {
|
||||
throw "host110_transport_terminal_readback_contract_failed"
|
||||
}
|
||||
} elseif (
|
||||
[string]$parsed.terminalStatus -ne "" -or
|
||||
[bool]$parsed.terminalCleanupVerified -or
|
||||
[int64]$parsed.terminalExitCode -ne -1
|
||||
) {
|
||||
throw "host110_transport_terminal_absence_contract_failed"
|
||||
}
|
||||
return $parsed
|
||||
}
|
||||
|
||||
function New-Agent99TransportLossReconciliation {
|
||||
param(
|
||||
[object]$Transport,
|
||||
[object]$SourcePackage,
|
||||
[object]$ReceiptReadback,
|
||||
[object]$VerifierReadback,
|
||||
[string]$ReceiptReadbackError = "",
|
||||
[string]$VerifierReadbackError = ""
|
||||
)
|
||||
|
||||
$payloadCommitted = [bool]($ReceiptReadback -and [string]$ReceiptReadback.payloadState -eq "valid")
|
||||
$destinationVerified = [bool]($VerifierReadback -and [bool]$VerifierReadback.ok)
|
||||
$terminalState = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalState } else { "unavailable" }
|
||||
$terminalStatus = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalStatus } else { "" }
|
||||
$status = "committed_unknown"
|
||||
if ($payloadCommitted -and $destinationVerified -and $terminalState -eq "missing") {
|
||||
$status = "committed_verified_terminal_missing"
|
||||
} elseif ($payloadCommitted -and $destinationVerified -and $terminalState -eq "valid") {
|
||||
$status = switch ($terminalStatus) {
|
||||
"verified" { "committed_verified_transport_lost" }
|
||||
"cleanup_pending" { "committed_cleanup_pending_transport_lost" }
|
||||
"committed_signal_deferred" { "committed_signal_deferred_transport_lost" }
|
||||
default { "committed_unknown" }
|
||||
}
|
||||
}
|
||||
return [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_transport_reconciliation_v1"
|
||||
ok = $false
|
||||
status = $status
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = [string]$SourcePackage.sourceHead
|
||||
runId = $RunId
|
||||
payloadCommitted = $payloadCommitted
|
||||
payloadReceiptState = if ($ReceiptReadback) { [string]$ReceiptReadback.payloadState } else { "unavailable" }
|
||||
terminalReceiptState = $terminalState
|
||||
terminalStatus = $terminalStatus
|
||||
destinationVerified = $destinationVerified
|
||||
retryApplyAllowed = $false
|
||||
transportReason = if ($Transport -and $Transport.PSObject.Properties["reason"]) { [string]$Transport.reason } else { "unknown" }
|
||||
transportExitCode = if ($Transport -and $Transport.PSObject.Properties["exitCode"]) { [int]$Transport.exitCode } else { -1 }
|
||||
receiptReadback = $ReceiptReadback
|
||||
verifier = $VerifierReadback
|
||||
receiptReadbackError = $ReceiptReadbackError
|
||||
verifierReadbackError = $VerifierReadbackError
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99TransportLossReconciliation {
|
||||
param([object]$Transport, [object]$SourcePackage)
|
||||
|
||||
$receiptReadback = $null
|
||||
$verifierReadback = $null
|
||||
$receiptReadbackError = ""
|
||||
$verifierReadbackError = ""
|
||||
try {
|
||||
$command = "python3 - --status-root /backup/status --source-revision $SourceRevision --source-head $($SourcePackage.sourceHead) --run-id $RunId --verifier-sha256 $($SourcePackage.verifierSha256)"
|
||||
$readbackTransport = Invoke-Agent99BoundedSsh $command 60 ([string]$SourcePackage.reconciliationPath)
|
||||
$receiptReadback = Get-Agent99TransportReceiptReadback $readbackTransport $SourcePackage
|
||||
} catch {
|
||||
$receiptReadbackError = [string]$_.Exception.Message
|
||||
}
|
||||
try {
|
||||
$verifierReadback = Invoke-Agent99NoWriteVerifier $SourcePackage
|
||||
} catch {
|
||||
$verifierReadbackError = [string]$_.Exception.Message
|
||||
}
|
||||
return New-Agent99TransportLossReconciliation `
|
||||
$Transport $SourcePackage $receiptReadback $verifierReadback `
|
||||
$receiptReadbackError $verifierReadbackError
|
||||
}
|
||||
|
||||
function Assert-Agent99ExecutorEvidence {
|
||||
param([object]$Executor, [string]$ExpectedSourceHead)
|
||||
|
||||
Assert-Agent99ApplyResultFields $Executor $(if ([bool]$Executor.ok) { "Object" } else { "Null" })
|
||||
if (
|
||||
[string]$Executor.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
|
||||
[string]$Executor.mode -ne "apply" -or
|
||||
[string]$Executor.sourceRevision -ne $SourceRevision -or
|
||||
[string]$Executor.sourceHead -ne $ExpectedSourceHead -or
|
||||
[string]$Executor.runId -ne $RunId -or
|
||||
[int64]$Executor.fileCount -ne $ExpectedFileCount -or
|
||||
[int64]$Executor.backupScriptFileCount -ne 17 -or
|
||||
-not [bool]$Executor.backupHealthExporterIncluded -or
|
||||
-not [bool]$Executor.payloadCommitted -or
|
||||
[bool]$Executor.rollbackAttempted -or
|
||||
[bool]$Executor.rollbackPerformed -or
|
||||
[bool]$Executor.rollbackVerified -or
|
||||
-not [bool]$Executor.zeroResidueVerified -or
|
||||
[string]$Executor.zeroResidueScope -ne "run_owned_destination_temps" -or
|
||||
[string]$Executor.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
||||
[string]$Executor.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
||||
-not [bool]$Executor.independentVerifierVerified
|
||||
) {
|
||||
throw "host110_existing_apply_contract_failed"
|
||||
}
|
||||
if ([bool]$Executor.ok) {
|
||||
Assert-Agent99VerifierEvidence $Executor.verifier
|
||||
if (
|
||||
[string]$Executor.status -ne "verified" -or
|
||||
-not [bool]$Executor.stageCleanupVerified -or
|
||||
-not [bool]$Executor.rollbackPrestateCleanupVerified -or
|
||||
-not [bool]$Executor.terminalReceiptPublished -or
|
||||
[int64]$Executor.terminalExitCode -ne 0 -or
|
||||
$null -ne $Executor.deferredSignal
|
||||
) {
|
||||
throw "host110_existing_apply_success_contract_failed"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if ($null -ne $Executor.verifier) { throw "host110_existing_apply_failure_verifier_present" }
|
||||
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
|
||||
if ([string]$Executor.status -eq "cleanup_pending") {
|
||||
if (
|
||||
([bool]$Executor.stageCleanupVerified -and [bool]$Executor.rollbackPrestateCleanupVerified) -or
|
||||
-not [bool]$Executor.terminalReceiptPublished -or
|
||||
[int64]$Executor.terminalExitCode -ne 75
|
||||
) { throw "host110_existing_cleanup_pending_contract_failed" }
|
||||
} elseif ([string]$Executor.status -eq "committed_signal_deferred") {
|
||||
$signalName = [string]$Executor.deferredSignal
|
||||
if (
|
||||
-not [bool]$Executor.stageCleanupVerified -or
|
||||
-not [bool]$Executor.rollbackPrestateCleanupVerified -or
|
||||
-not [bool]$Executor.terminalReceiptPublished -or
|
||||
-not $signalExitCodes.ContainsKey($signalName) -or
|
||||
[int64]$Executor.terminalExitCode -ne [int]$signalExitCodes[$signalName]
|
||||
) { throw "host110_existing_committed_signal_contract_failed" }
|
||||
} elseif ([string]$Executor.status -eq "terminal_receipt_failed") {
|
||||
if (
|
||||
[bool]$Executor.terminalReceiptPublished -or
|
||||
[int64]$Executor.terminalExitCode -ne 76
|
||||
) { throw "host110_existing_terminal_receipt_failure_contract_failed" }
|
||||
} else {
|
||||
throw "host110_existing_apply_failure_status_invalid"
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99ReconciliationEvidence {
|
||||
param([object]$Reconciliation, [string]$ExpectedSourceHead)
|
||||
|
||||
Assert-Agent99ExactJsonFields $Reconciliation @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
status = "String"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
runId = "String"
|
||||
payloadCommitted = "Boolean"
|
||||
payloadReceiptState = "String"
|
||||
terminalReceiptState = "String"
|
||||
terminalStatus = "String"
|
||||
destinationVerified = "Boolean"
|
||||
retryApplyAllowed = "Boolean"
|
||||
transportReason = "String"
|
||||
transportExitCode = "Integer"
|
||||
receiptReadback = "ObjectOrNull"
|
||||
verifier = "ObjectOrNull"
|
||||
receiptReadbackError = "String"
|
||||
verifierReadbackError = "String"
|
||||
remoteWritePerformed = "Boolean"
|
||||
secretValuesRead = "Boolean"
|
||||
} "host110_existing_reconciliation_required_field_failed"
|
||||
if (
|
||||
[string]$Reconciliation.schemaVersion -ne "agent99_host110_backup_transport_reconciliation_v1" -or
|
||||
[bool]$Reconciliation.ok -or
|
||||
[string]$Reconciliation.sourceRevision -ne $SourceRevision -or
|
||||
[string]$Reconciliation.sourceHead -ne $ExpectedSourceHead -or
|
||||
[string]$Reconciliation.runId -ne $RunId -or
|
||||
[string]$Reconciliation.payloadReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or
|
||||
[string]$Reconciliation.terminalReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or
|
||||
[bool]$Reconciliation.retryApplyAllowed -or
|
||||
[bool]$Reconciliation.remoteWritePerformed -or
|
||||
[bool]$Reconciliation.secretValuesRead
|
||||
) {
|
||||
throw "host110_existing_reconciliation_contract_failed"
|
||||
}
|
||||
if ($Reconciliation.receiptReadback) {
|
||||
if ($Reconciliation.receiptReadbackError) { throw "host110_existing_reconciliation_readback_error_conflict" }
|
||||
$sourcePackage = [pscustomobject]@{ sourceHead = $ExpectedSourceHead }
|
||||
$transport = [pscustomobject]@{
|
||||
ok = $true
|
||||
stdout = $Reconciliation.receiptReadback | ConvertTo-Json -Compress -Depth 8
|
||||
}
|
||||
Get-Agent99TransportReceiptReadback $transport $sourcePackage | Out-Null
|
||||
} elseif (-not $Reconciliation.receiptReadbackError) {
|
||||
throw "host110_existing_reconciliation_readback_error_missing"
|
||||
}
|
||||
if ($Reconciliation.verifier) {
|
||||
if ($Reconciliation.verifierReadbackError) { throw "host110_existing_reconciliation_verifier_error_conflict" }
|
||||
Assert-Agent99VerifierEvidence $Reconciliation.verifier
|
||||
} elseif (-not $Reconciliation.verifierReadbackError) {
|
||||
throw "host110_existing_reconciliation_verifier_error_missing"
|
||||
}
|
||||
if ([bool]$Reconciliation.payloadCommitted -ne [bool]($Reconciliation.payloadReceiptState -eq "valid")) {
|
||||
throw "host110_existing_reconciliation_payload_truth_mismatch"
|
||||
}
|
||||
if ($Reconciliation.payloadCommitted -and -not $Reconciliation.receiptReadback) {
|
||||
throw "host110_existing_reconciliation_payload_readback_missing"
|
||||
}
|
||||
if ([bool]$Reconciliation.destinationVerified -ne [bool]($null -ne $Reconciliation.verifier)) {
|
||||
throw "host110_existing_reconciliation_destination_truth_mismatch"
|
||||
}
|
||||
$status = [string]$Reconciliation.status
|
||||
$knownStatuses = @(
|
||||
"committed_verified_terminal_missing",
|
||||
"committed_verified_transport_lost",
|
||||
"committed_cleanup_pending_transport_lost",
|
||||
"committed_signal_deferred_transport_lost",
|
||||
"committed_unknown"
|
||||
)
|
||||
if ($status -notin $knownStatuses) { throw "host110_existing_reconciliation_status_invalid" }
|
||||
if ($status -ne "committed_unknown") {
|
||||
if (-not $Reconciliation.payloadCommitted -or -not $Reconciliation.destinationVerified) {
|
||||
throw "host110_existing_reconciliation_verified_state_invalid"
|
||||
}
|
||||
}
|
||||
if ($status -eq "committed_verified_terminal_missing" -and $Reconciliation.terminalReceiptState -ne "missing") {
|
||||
throw "host110_existing_reconciliation_terminal_missing_mismatch"
|
||||
}
|
||||
$terminalStatusByReconciliation = @{
|
||||
committed_verified_transport_lost = "verified"
|
||||
committed_cleanup_pending_transport_lost = "cleanup_pending"
|
||||
committed_signal_deferred_transport_lost = "committed_signal_deferred"
|
||||
}
|
||||
if ($terminalStatusByReconciliation.ContainsKey($status)) {
|
||||
if (
|
||||
$Reconciliation.terminalReceiptState -ne "valid" -or
|
||||
[string]$Reconciliation.terminalStatus -ne [string]$terminalStatusByReconciliation[$status]
|
||||
) { throw "host110_existing_reconciliation_terminal_state_mismatch" }
|
||||
}
|
||||
}
|
||||
|
||||
function Assert-Agent99ExistingBrokerEvidence {
|
||||
param([object]$Existing, [string]$ExpectedEvidencePath)
|
||||
|
||||
Assert-Agent99ExactJsonFields $Existing @{
|
||||
schemaVersion = "String"
|
||||
ok = "Boolean"
|
||||
status = "String"
|
||||
mode = "String"
|
||||
sourceRevision = "String"
|
||||
sourceHead = "String"
|
||||
runId = "String"
|
||||
targetHost = "String"
|
||||
sourceTransport = "String"
|
||||
executor = "String"
|
||||
verifier = "String"
|
||||
decisionProvider = "String"
|
||||
criticProvider = "String"
|
||||
agentAction = "String"
|
||||
check = "ObjectOrNull"
|
||||
apply = "ObjectOrNull"
|
||||
verify = "ObjectOrNull"
|
||||
cleanupVerified = "Boolean"
|
||||
errorCode = "String"
|
||||
elapsedSeconds = "Number"
|
||||
evidence = "String"
|
||||
secretValuesRead = "Boolean"
|
||||
rawSessionStored = "Boolean"
|
||||
} "existing_broker_evidence_required_field_failed"
|
||||
if (
|
||||
[string]$Existing.mode -ne $Mode -or
|
||||
[string]$Existing.sourceRevision -ne $SourceRevision -or
|
||||
[string]$Existing.runId -ne $RunId -or
|
||||
[string]$Existing.evidence -ne $ExpectedEvidencePath
|
||||
) {
|
||||
throw "run_identity_conflict"
|
||||
}
|
||||
if (
|
||||
[string]$Existing.schemaVersion -ne "agent99_host110_backup_runtime_broker_receipt_v3" -or
|
||||
[string]$Existing.targetHost -ne $TargetHost -or
|
||||
[string]$Existing.sourceTransport -ne "windows99_gitea_exact_revision_manifest" -or
|
||||
[string]$Existing.executor -ne "host110_backup_runtime_executor" -or
|
||||
[string]$Existing.verifier -ne "independent_host110_backup_runtime_python_readback" -or
|
||||
[string]$Existing.decisionProvider -ne "deterministic_only" -or
|
||||
[string]$Existing.criticProvider -ne "deterministic_only" -or
|
||||
[string]$Existing.agentAction -ne $(if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }) -or
|
||||
([string]$Existing.sourceHead -ne "" -and [string]$Existing.sourceHead -notmatch "^[0-9a-f]{40}$") -or
|
||||
[double]$Existing.elapsedSeconds -lt 0 -or
|
||||
[bool]$Existing.secretValuesRead -or
|
||||
[bool]$Existing.rawSessionStored
|
||||
) {
|
||||
throw "existing_broker_evidence_contract_failed"
|
||||
}
|
||||
if ($Existing.check) { Assert-Agent99PreflightEvidence $Existing.check ([string]$Existing.sourceHead) }
|
||||
if ($Existing.verify) { Assert-Agent99VerifierEvidence $Existing.verify }
|
||||
if ($Existing.apply) {
|
||||
if ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_runtime_executor_v1") {
|
||||
Assert-Agent99ExecutorEvidence $Existing.apply ([string]$Existing.sourceHead)
|
||||
} elseif ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_transport_reconciliation_v1") {
|
||||
Assert-Agent99ReconciliationEvidence $Existing.apply ([string]$Existing.sourceHead)
|
||||
} else {
|
||||
throw "existing_broker_apply_schema_invalid"
|
||||
}
|
||||
}
|
||||
if (($Existing.apply -or $Existing.verify) -and -not $Existing.check) {
|
||||
throw "existing_broker_preflight_evidence_missing"
|
||||
}
|
||||
if ($Mode -ne "Apply" -and $Existing.apply) { throw "existing_broker_apply_mode_conflict" }
|
||||
if ($Mode -eq "Check" -and $Existing.verify) { throw "existing_broker_verify_mode_conflict" }
|
||||
|
||||
if ([bool]$Existing.ok) {
|
||||
if ([string]$Existing.errorCode -ne "" -or -not [bool]$Existing.cleanupVerified) {
|
||||
throw "existing_broker_success_terminal_invalid"
|
||||
}
|
||||
if ($Mode -eq "Apply") {
|
||||
if (
|
||||
[string]$Existing.status -ne "verified" -or
|
||||
-not $Existing.check -or
|
||||
-not $Existing.apply -or
|
||||
-not [bool]$Existing.apply.ok -or
|
||||
-not $Existing.verify -or
|
||||
[string]$Existing.apply.verifier.verifierSha256 -ne [string]$Existing.verify.verifierSha256
|
||||
) { throw "existing_broker_apply_success_invalid" }
|
||||
} elseif ($Mode -eq "Verify") {
|
||||
if ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or -not $Existing.verify) {
|
||||
throw "existing_broker_verify_success_invalid"
|
||||
}
|
||||
} elseif ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or $Existing.verify) {
|
||||
throw "existing_broker_check_success_invalid"
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
if ([string]$Existing.status -ne "failed" -or [string]::IsNullOrWhiteSpace([string]$Existing.errorCode)) {
|
||||
throw "existing_broker_failure_terminal_invalid"
|
||||
}
|
||||
if ($Existing.verify -and [string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed") {
|
||||
throw "existing_broker_failure_verifier_conflict"
|
||||
}
|
||||
if ([string]$Existing.errorCode -like "host110_apply_*" -and -not $Existing.apply) {
|
||||
throw "existing_broker_apply_failure_evidence_missing"
|
||||
}
|
||||
if ($Existing.apply) {
|
||||
if ([bool]$Existing.apply.ok) {
|
||||
if (
|
||||
[string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed" -or
|
||||
-not $Existing.verify -or
|
||||
[bool]$Existing.cleanupVerified
|
||||
) { throw "existing_broker_postcommit_cleanup_failure_conflict" }
|
||||
} elseif ([string]$Existing.errorCode -ne "host110_apply_$([string]$Existing.apply.status)") {
|
||||
throw "existing_broker_apply_failure_conflict"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-Agent99ReadOnlyPreflight {
|
||||
param([object]$SourcePackage)
|
||||
|
||||
@@ -573,9 +1264,7 @@ New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-N
|
||||
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
|
||||
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
|
||||
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
|
||||
if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) {
|
||||
throw "run_identity_conflict"
|
||||
}
|
||||
Assert-Agent99ExistingBrokerEvidence $existing $evidencePath
|
||||
$existing | ConvertTo-Json -Compress -Depth 12
|
||||
if (-not [bool]$existing.ok) { exit 1 }
|
||||
exit 0
|
||||
@@ -620,9 +1309,12 @@ try {
|
||||
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
|
||||
$applyTransport = Invoke-Agent99BoundedSsh $command 330
|
||||
if ($applyTransport.ok) {
|
||||
$apply = Get-Agent99ExecutorResult $applyTransport "Apply"
|
||||
$apply = Get-Agent99ExecutorResult $applyTransport "Apply" ([string]$sourcePackage.verifierSha256)
|
||||
} else {
|
||||
$apply = Get-Agent99CommittedFailureResult $applyTransport
|
||||
try { $apply = Get-Agent99CommittedFailureResult $applyTransport } catch { $apply = $null }
|
||||
if (-not $apply) {
|
||||
$apply = Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage
|
||||
}
|
||||
throw "host110_apply_$([string]$apply.status)"
|
||||
}
|
||||
$verify = $apply.verifier
|
||||
|
||||
202
docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md
Normal file
202
docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md
Normal file
@@ -0,0 +1,202 @@
|
||||
# AWOOOI Agent99 Host110 Backup Runtime V11
|
||||
|
||||
Status: proposed. Central review and explicit owner approval are required before
|
||||
any production apply. This candidate must not be pushed to `main`, sent through
|
||||
CD, or applied to Windows99 or Host110 while it remains proposed.
|
||||
|
||||
This artifact supersedes V10. Every V10 identity below is void for approval and
|
||||
remains audit evidence only:
|
||||
|
||||
- branch/ref: `codex/host110-backup-runtime-v10-20260718`
|
||||
- candidate/revision: `d1797a1c8766722ae00ac14e20fed00f4e3f3a5b`
|
||||
- artifact SHA-256: `eb48661bbecc4e169111879bb18dfab63be29878d5f5c3651d9917f81f195d28`
|
||||
- exact diff SHA-256: `f7efb98d5f91fa6899713b692c8bb015fcb6ae6dba4291cc6ae429290072964e`
|
||||
- every other V10 proposal, content, revision, review, and derived hash
|
||||
|
||||
No V10 hash, ref, review, receipt, or test grants production execution
|
||||
authority.
|
||||
|
||||
## Exact Scope
|
||||
|
||||
- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity
|
||||
test.
|
||||
- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and
|
||||
`backup-health-textfile-exporter.py`.
|
||||
- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file
|
||||
payload, fixed executor, independent verifier, and `manifest.json`.
|
||||
- Fixed target: `wooo@192.168.0.110`, dispatched only by the Windows99 Agent99
|
||||
typed broker from an exact Gitea revision and digest-bound manifest.
|
||||
- V11 changes the Windows99 broker, three shell shared-lock entrypoints, the
|
||||
Host188 archive verifier's shared-lock acquisition, and their no-write replay
|
||||
tests.
|
||||
- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No
|
||||
Gitea primary stop/start or backup-container creation is part of this apply.
|
||||
- The restore drill and offsite gate gain only canonical shared-lock behavior.
|
||||
Applying this candidate replaces those scripts but does not invoke a drill,
|
||||
run a backup, or start an offsite sync.
|
||||
- Prior proposal documents remain tracked audit artifacts and are not runtime
|
||||
authority.
|
||||
|
||||
## V10 Defects Closed
|
||||
|
||||
### Persisted Broker Evidence Is Exact And Fail-Closed
|
||||
|
||||
Before reusing a run-bound broker receipt, the broker now validates exact key
|
||||
sets, required native JSON types, schema version, run ID, source revision, mode,
|
||||
target, evidence path binding, status, error, cleanup, and nested Apply/Verify
|
||||
contracts. Missing, extra, wrong-type, wrong-mode, wrong-run, conflicting, or
|
||||
malformed evidence fails closed before any replay decision.
|
||||
|
||||
A non-empty string such as `"false"` can no longer be cast to a truthy Boolean.
|
||||
An existing failure remains a failure and cannot be upgraded to success merely
|
||||
because it is parseable or shares a source revision.
|
||||
|
||||
### Mandatory Post-Commit Transport Reconciliation
|
||||
|
||||
If Apply loses SSH transport or times out, the broker performs a read-only,
|
||||
exact-RunId receipt readback under `/backup/status` and an independent
|
||||
destination verifier readback. It never blindly retries Apply and never
|
||||
collapses a post-commit state into a generic precommit failure.
|
||||
|
||||
The reconciliation states are explicit and non-success:
|
||||
|
||||
- `committed_verified_terminal_missing`
|
||||
- `committed_verified_transport_lost`
|
||||
- `committed_cleanup_pending_transport_lost`
|
||||
- `committed_signal_deferred_transport_lost`
|
||||
- `committed_unknown`
|
||||
|
||||
Every state records `retryApplyAllowed=false`. Receipt schema, native types,
|
||||
source/run/head identity, payload paths and counts, verifier identity, cleanup,
|
||||
signal, and terminal semantics are validated before classification. The
|
||||
readback performs no remote write and reads no secret value.
|
||||
|
||||
### Canonical Shared Lock And Nested Re-entry
|
||||
|
||||
Production backup entrypoints canonicalize absolute, relative, and symlink
|
||||
invocation paths before deciding whether the Host110 runtime lock applies.
|
||||
Unresolvable production identity fails closed.
|
||||
|
||||
For nested calls, an inherited FD 197 is accepted only when its canonical
|
||||
target is the exact runtime lock. Linux uses `/proc/$$/fd/197`; the bounded
|
||||
non-Linux fallback resolves the descriptor path with `F_GETPATH`. The entrypoint
|
||||
then requests the shared lock on the same descriptor. It does not close and
|
||||
reopen a valid inherited descriptor, so there is no release/reacquire gap.
|
||||
|
||||
If FD 197 is absent, the entrypoint opens the lock and acquires the shared side.
|
||||
If FD 197 exists but points elsewhere or cannot be resolved, execution fails
|
||||
closed. `BACKUP_RUNTIME_SHARED_LOCK_HELD` and all other external environment
|
||||
values grant no lock authority.
|
||||
|
||||
The Host188 archive verifier follows the same contract: exact inherited FD 197
|
||||
is duplicated from the same open file description, wrong identity fails
|
||||
closed, and a forged environment value cannot bypass an active exclusive lock.
|
||||
|
||||
## Apply Effects
|
||||
|
||||
- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded
|
||||
manifest and receipt state.
|
||||
- Host110 Apply replaces the ordered 18 payload paths through one exclusive,
|
||||
digest-bound filesystem transaction and writes payload/terminal receipts
|
||||
below `/backup/status`.
|
||||
- The deployed common library, restore drill, offsite gate, and Host188 archive
|
||||
verifier gain the shared-lock behavior described above. Apply does not invoke
|
||||
their backup, drill, archive, or sync behavior.
|
||||
- Apply does not stop/start Gitea, create a backup container, call Google Drive,
|
||||
prune snapshots, restart a VM or service, change a database/network/firewall,
|
||||
or read/change a credential or secret.
|
||||
|
||||
## Reproducible Validation
|
||||
|
||||
Expanded focused and compatibility suite:
|
||||
|
||||
```sh
|
||||
pytest -q \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \
|
||||
scripts/backup/tests/test_verify_host188_products_archive.py \
|
||||
scripts/backup/tests/test_backup_host188_products_contract.py \
|
||||
scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \
|
||||
scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \
|
||||
scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \
|
||||
scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py
|
||||
```
|
||||
|
||||
Current result: `120 passed, 4 skipped`.
|
||||
|
||||
The shared-lock subset includes real `fcntl` contention for absolute,
|
||||
relative, symlink, forged-environment, inherited-exact-FD, inherited-wrong-FD,
|
||||
and Host188 archive paths. Current result: `18 passed, 31 deselected`.
|
||||
|
||||
Static, syntax, and repository checks:
|
||||
|
||||
```sh
|
||||
ruff check \
|
||||
scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \
|
||||
scripts/backup/verify-host188-products-archive.py
|
||||
bash -n \
|
||||
scripts/backup/common.sh \
|
||||
scripts/backup/gitea-full-backup-restore-drill.sh \
|
||||
scripts/backup/backup-offsite-readiness-gate.sh \
|
||||
scripts/backup/host110-backup-runtime-executor.sh
|
||||
git diff --check
|
||||
PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh
|
||||
```
|
||||
|
||||
`ansible-validate.sh` parses
|
||||
`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs
|
||||
`ansible-playbook --syntax-check` only for actual playbooks. Current checks are
|
||||
green; `ansible-lint` is not installed and is reported as skipped.
|
||||
|
||||
Windows99 no-write PowerShell validation consists of:
|
||||
|
||||
1. parsing the complete candidate broker with Windows PowerShell;
|
||||
2. loading the candidate broker's actual function AST in the contract replay;
|
||||
3. replaying valid success, committed non-success, existing evidence rejection,
|
||||
terminal-missing reconciliation, and committed-unknown cases.
|
||||
|
||||
Current readback is parser `errorCount=0`, plus 38 existing-evidence
|
||||
rejections, 38 missing-field rejections, 6 type rejections,
|
||||
`terminalMissingReconciled=true`, `committedUnknownPreserved=true`,
|
||||
`remoteWritePerformed=false`, and `secretValuesRead=false`.
|
||||
|
||||
## Exact Identity And Diff Serialization
|
||||
|
||||
The central-review intake must bind all of the following independently:
|
||||
|
||||
- live Gitea feature ref `codex/host110-backup-runtime-v11-20260718`;
|
||||
- exact candidate Git SHA;
|
||||
- this tracked artifact path and exact artifact-bytes SHA-256;
|
||||
- exact binary diff SHA-256, path count, and insertion/deletion counts;
|
||||
- the 19/18/21 count contracts;
|
||||
- `scripts/backup/backup-gitea.sh` zero diff;
|
||||
- the Windows99 no-write replay result.
|
||||
|
||||
Exact diff bytes are serialized with:
|
||||
|
||||
```sh
|
||||
git diff --binary --full-index \
|
||||
e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3..<candidate_git_sha> |
|
||||
shasum -a 256
|
||||
```
|
||||
|
||||
No alternative diff serialization or untracked artifact bytes may be used for
|
||||
approval identity.
|
||||
|
||||
## Rollback
|
||||
|
||||
- Source rollback reverts the exact V11 diff to
|
||||
`e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing.
|
||||
- Agent99 rollback restores prior runtime files and manifest and verifies prior
|
||||
hashes independently.
|
||||
- Before payload commit, Host110 rollback restores prior payload content,
|
||||
SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned
|
||||
destination temporary remains.
|
||||
- After payload commit, transport loss cannot trigger a blind Apply retry.
|
||||
Exact receipts and independent destination readback determine whether the
|
||||
state is committed or unknown; every uncertain state remains non-success.
|
||||
- Source rollback restores prior lock behavior. It does not execute a backup,
|
||||
restore drill, offsite sync, container lifecycle, or service restart.
|
||||
- Any identity, schema, type, receipt, reconciliation, lock, cleanup, or
|
||||
residue mismatch fails closed and cannot produce broker `verified`.
|
||||
@@ -9,14 +9,74 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
resolve_backup_runtime_script_path() {
|
||||
local raw_path="$1" candidate="" resolved="" resolved_directory=""
|
||||
[ -n "${raw_path}" ] || return 1
|
||||
case "${raw_path}" in /*) candidate="${raw_path}" ;; *) candidate="${PWD}/${raw_path}" ;; esac
|
||||
if command -v realpath >/dev/null 2>&1; then
|
||||
resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
||||
fi
|
||||
if command -v readlink >/dev/null 2>&1; then
|
||||
resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
||||
fi
|
||||
[ ! -L "${candidate}" ] || return 1
|
||||
resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1
|
||||
[ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1
|
||||
printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")"
|
||||
}
|
||||
|
||||
backup_runtime_inherited_fd_status() {
|
||||
local expected_path="$1" fd_path="" fd_target=""
|
||||
if [ -d "/proc/$$/fd" ]; then
|
||||
fd_path="/proc/$$/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
fd_target="$(resolve_backup_runtime_script_path "${fd_path}")" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
return 0
|
||||
fi
|
||||
fd_path="/dev/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
command -v python3 >/dev/null 2>&1 || return 2
|
||||
fd_target="$(python3 - <<'PY'
|
||||
import fcntl
|
||||
import os
|
||||
|
||||
try:
|
||||
target = os.readlink("/dev/fd/197")
|
||||
except OSError:
|
||||
value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
||||
target = os.fsdecode(value.split(b"\0", 1)[0])
|
||||
print(os.path.realpath(target))
|
||||
PY
|
||||
)" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
}
|
||||
|
||||
runtime_script_path="$(resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}")" || {
|
||||
echo "backup runtime gate unavailable: script identity unresolved" >&2
|
||||
exit 69
|
||||
}
|
||||
if [[ "${runtime_script_path}" == /backup/scripts/* ]]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
runtime_lock_path="$(resolve_backup_runtime_script_path /tmp/agent99-host110-backup-runtime.lock)" \
|
||||
|| { echo "backup runtime gate unavailable: lock identity unresolved" >&2; exit 69; }
|
||||
if backup_runtime_inherited_fd_status "${runtime_lock_path}"; then
|
||||
:
|
||||
else
|
||||
inherited_fd_status=$?
|
||||
if [ "${inherited_fd_status}" -eq 1 ]; then
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
else
|
||||
echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2
|
||||
exit 69
|
||||
fi
|
||||
fi
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
BACKUP_BASE="${BACKUP_BASE:-/backup}"
|
||||
|
||||
@@ -3,14 +3,78 @@
|
||||
|
||||
# Production backup entrypoints share this stable gate. Agent99 takes the
|
||||
# exclusive side while promoting a complete, digest-bound runtime bundle.
|
||||
resolve_backup_runtime_source_path() {
|
||||
local raw_path="$1"
|
||||
local candidate=""
|
||||
local resolved=""
|
||||
local resolved_directory=""
|
||||
[ -n "${raw_path}" ] || return 1
|
||||
case "${raw_path}" in
|
||||
/*) candidate="${raw_path}" ;;
|
||||
*) candidate="${PWD}/${raw_path}" ;;
|
||||
esac
|
||||
if command -v realpath >/dev/null 2>&1; then
|
||||
resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then
|
||||
printf '%s\n' "${resolved}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
if command -v readlink >/dev/null 2>&1; then
|
||||
resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then
|
||||
printf '%s\n' "${resolved}"
|
||||
return 0
|
||||
fi
|
||||
fi
|
||||
[ ! -L "${candidate}" ] || return 1
|
||||
resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1
|
||||
[ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1
|
||||
printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")"
|
||||
}
|
||||
|
||||
backup_runtime_inherited_fd_status() {
|
||||
local expected_path="$1"
|
||||
local fd_path=""
|
||||
local fd_target=""
|
||||
if [ -d "/proc/$$/fd" ]; then
|
||||
fd_path="/proc/$$/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
fd_target="$(resolve_backup_runtime_source_path "${fd_path}")" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
return 0
|
||||
fi
|
||||
fd_path="/dev/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
command -v python3 >/dev/null 2>&1 || return 2
|
||||
fd_target="$(python3 - <<'PY'
|
||||
import fcntl
|
||||
import os
|
||||
|
||||
try:
|
||||
target = os.readlink("/dev/fd/197")
|
||||
except OSError:
|
||||
value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
||||
target = os.fsdecode(value.split(b"\0", 1)[0])
|
||||
print(os.path.realpath(target))
|
||||
PY
|
||||
)" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
}
|
||||
|
||||
acquire_backup_runtime_shared_lock() {
|
||||
local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}"
|
||||
local caller_path=""
|
||||
local lock_path="/tmp/agent99-host110-backup-runtime.lock"
|
||||
local lock_canonical=""
|
||||
local inherited_fd_status=0
|
||||
caller_path="$(resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}")" || {
|
||||
echo "backup runtime gate unavailable: caller identity unresolved" >&2
|
||||
return 69
|
||||
}
|
||||
case "${caller_path}" in
|
||||
/backup/scripts/*) ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0
|
||||
command -v flock >/dev/null 2>&1 || {
|
||||
echo "backup runtime gate unavailable: flock missing" >&2
|
||||
return 69
|
||||
@@ -21,12 +85,25 @@ acquire_backup_runtime_shared_lock() {
|
||||
}
|
||||
(umask 077; : >> "${lock_path}") || return 69
|
||||
[ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69
|
||||
exec 197>>"${lock_path}"
|
||||
lock_canonical="$(resolve_backup_runtime_source_path "${lock_path}")" || {
|
||||
echo "backup runtime gate unavailable: lock identity unresolved" >&2
|
||||
return 69
|
||||
}
|
||||
if backup_runtime_inherited_fd_status "${lock_canonical}"; then
|
||||
:
|
||||
else
|
||||
inherited_fd_status=$?
|
||||
if [ "${inherited_fd_status}" -eq 1 ]; then
|
||||
exec 197>>"${lock_path}"
|
||||
else
|
||||
echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2
|
||||
return 69
|
||||
fi
|
||||
fi
|
||||
flock -s -w 60 197 || {
|
||||
echo "backup runtime deployment is active; retry later" >&2
|
||||
return 75
|
||||
}
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
}
|
||||
|
||||
acquire_backup_runtime_shared_lock || {
|
||||
|
||||
@@ -5,14 +5,74 @@
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then
|
||||
resolve_backup_runtime_script_path() {
|
||||
local raw_path="$1" candidate="" resolved="" resolved_directory=""
|
||||
[ -n "${raw_path}" ] || return 1
|
||||
case "${raw_path}" in /*) candidate="${raw_path}" ;; *) candidate="${PWD}/${raw_path}" ;; esac
|
||||
if command -v realpath >/dev/null 2>&1; then
|
||||
resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
||||
fi
|
||||
if command -v readlink >/dev/null 2>&1; then
|
||||
resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved=""
|
||||
if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi
|
||||
fi
|
||||
[ ! -L "${candidate}" ] || return 1
|
||||
resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1
|
||||
[ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1
|
||||
printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")"
|
||||
}
|
||||
|
||||
backup_runtime_inherited_fd_status() {
|
||||
local expected_path="$1" fd_path="" fd_target=""
|
||||
if [ -d "/proc/$$/fd" ]; then
|
||||
fd_path="/proc/$$/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
fd_target="$(resolve_backup_runtime_script_path "${fd_path}")" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
return 0
|
||||
fi
|
||||
fd_path="/dev/fd/197"
|
||||
[ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1
|
||||
command -v python3 >/dev/null 2>&1 || return 2
|
||||
fd_target="$(python3 - <<'PY'
|
||||
import fcntl
|
||||
import os
|
||||
|
||||
try:
|
||||
target = os.readlink("/dev/fd/197")
|
||||
except OSError:
|
||||
value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
||||
target = os.fsdecode(value.split(b"\0", 1)[0])
|
||||
print(os.path.realpath(target))
|
||||
PY
|
||||
)" || return 2
|
||||
[ "${fd_target}" = "${expected_path}" ] || return 2
|
||||
}
|
||||
|
||||
runtime_script_path="$(resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}")" || {
|
||||
echo "backup runtime gate unavailable: script identity unresolved" >&2
|
||||
exit 69
|
||||
}
|
||||
if [[ "${runtime_script_path}" == /backup/scripts/* ]]; then
|
||||
command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; }
|
||||
[ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; }
|
||||
(umask 077; : >> /tmp/agent99-host110-backup-runtime.lock)
|
||||
[ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; }
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
runtime_lock_path="$(resolve_backup_runtime_script_path /tmp/agent99-host110-backup-runtime.lock)" \
|
||||
|| { echo "backup runtime gate unavailable: lock identity unresolved" >&2; exit 69; }
|
||||
if backup_runtime_inherited_fd_status "${runtime_lock_path}"; then
|
||||
:
|
||||
else
|
||||
inherited_fd_status=$?
|
||||
if [ "${inherited_fd_status}" -eq 1 ]; then
|
||||
exec 197>>/tmp/agent99-host110-backup-runtime.lock
|
||||
else
|
||||
echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2
|
||||
exit 69
|
||||
fi
|
||||
fi
|
||||
flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; }
|
||||
export BACKUP_RUNTIME_SHARED_LOCK_HELD=1
|
||||
fi
|
||||
|
||||
DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}"
|
||||
|
||||
@@ -27,11 +27,46 @@ RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")
|
||||
|
||||
|
||||
def acquire_runtime_lock():
|
||||
if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1":
|
||||
if Path(__file__).resolve().parent != Path("/backup/scripts"):
|
||||
return None
|
||||
if RUNTIME_LOCK.is_symlink():
|
||||
raise RuntimeError("backup_runtime_gate_unsafe")
|
||||
handle = RUNTIME_LOCK.open("a+")
|
||||
inherited_fd_path: Path | None = None
|
||||
proc_fd_root = Path(f"/proc/{os.getpid()}/fd")
|
||||
if proc_fd_root.is_dir():
|
||||
candidate = proc_fd_root / "197"
|
||||
if candidate.exists() or candidate.is_symlink():
|
||||
inherited_fd_path = candidate
|
||||
else:
|
||||
candidate = Path("/dev/fd/197")
|
||||
if candidate.exists() or candidate.is_symlink():
|
||||
inherited_fd_path = candidate
|
||||
|
||||
if inherited_fd_path is None:
|
||||
handle = RUNTIME_LOCK.open("a+")
|
||||
else:
|
||||
if not RUNTIME_LOCK.is_file():
|
||||
raise RuntimeError("backup_runtime_gate_unavailable")
|
||||
if proc_fd_root.is_dir():
|
||||
try:
|
||||
inherited_target = inherited_fd_path.resolve(strict=True)
|
||||
expected_target = RUNTIME_LOCK.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RuntimeError("backup_runtime_inherited_fd_unresolved") from exc
|
||||
else:
|
||||
try:
|
||||
raw_target = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024)
|
||||
inherited_target = Path(os.fsdecode(raw_target.split(b"\0", 1)[0])).resolve(strict=True)
|
||||
expected_target = RUNTIME_LOCK.resolve(strict=True)
|
||||
except OSError as exc:
|
||||
raise RuntimeError("backup_runtime_inherited_fd_unresolved") from exc
|
||||
if inherited_target != expected_target:
|
||||
raise RuntimeError("backup_runtime_inherited_fd_mismatch")
|
||||
try:
|
||||
handle = os.fdopen(os.dup(197), "a+")
|
||||
except OSError as exc:
|
||||
raise RuntimeError("backup_runtime_inherited_fd_unavailable") from exc
|
||||
|
||||
if RUNTIME_LOCK.stat().st_uid != os.getuid():
|
||||
handle.close()
|
||||
raise RuntimeError("backup_runtime_gate_owner_invalid")
|
||||
|
||||
@@ -35,11 +35,19 @@ Write-ReplayTrace "define_functions"
|
||||
$functionNames = @(
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Assert-Agent99ExactJsonFields",
|
||||
"Convert-Agent99ExecutorDocument",
|
||||
"Assert-Agent99ApplyResultFields",
|
||||
"Assert-Agent99VerifierFields",
|
||||
"Get-Agent99ExecutorResult",
|
||||
"Get-Agent99CommittedFailureResult"
|
||||
"Get-Agent99CommittedFailureResult",
|
||||
"Assert-Agent99VerifierEvidence",
|
||||
"Assert-Agent99PreflightEvidence",
|
||||
"Get-Agent99TransportReceiptReadback",
|
||||
"New-Agent99TransportLossReconciliation",
|
||||
"Assert-Agent99ExecutorEvidence",
|
||||
"Assert-Agent99ReconciliationEvidence",
|
||||
"Assert-Agent99ExistingBrokerEvidence"
|
||||
)
|
||||
foreach ($functionName in $functionNames) {
|
||||
$node = $ast.Find(
|
||||
@@ -56,8 +64,11 @@ foreach ($functionName in $functionNames) {
|
||||
|
||||
$SourceRevision = "a" * 40
|
||||
$RunId = "windows99-contract-replay"
|
||||
$Mode = "Apply"
|
||||
$TargetHost = "192.168.0.110"
|
||||
$ExpectedFileCount = 18
|
||||
$sourceHead = "b" * 40
|
||||
$VerifierSha256 = "c" * 64
|
||||
$payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json"
|
||||
$terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
|
||||
|
||||
@@ -95,8 +106,12 @@ $verifier = [pscustomobject]@{
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
destination = "/backup/scripts"
|
||||
exporterDestination = "/home/wooo/scripts/backup-health-textfile-exporter.py"
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
selfIdentityVerified = $true
|
||||
verifierSha256 = $VerifierSha256
|
||||
}
|
||||
$success = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_executor_v1"
|
||||
@@ -127,7 +142,7 @@ $success = [pscustomobject]@{
|
||||
}
|
||||
|
||||
Write-ReplayTrace "accept_success"
|
||||
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null
|
||||
Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" $VerifierSha256 | Out-Null
|
||||
$topLevelRequired = @(
|
||||
"schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead",
|
||||
"runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded",
|
||||
@@ -139,21 +154,22 @@ $topLevelRequired = @(
|
||||
$verifierRequired = @(
|
||||
"schemaVersion", "ok", "sourceRevision", "runId", "fileCount",
|
||||
"backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed",
|
||||
"selfIdentityVerified"
|
||||
"secretValuesRead", "selfIdentityVerified", "destination", "exporterDestination",
|
||||
"verifierSha256"
|
||||
)
|
||||
$missingFieldRejections = 0
|
||||
Write-ReplayTrace "reject_missing_top"
|
||||
foreach ($field in $topLevelRequired) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_$field"
|
||||
$missingFieldRejections++
|
||||
}
|
||||
Write-ReplayTrace "reject_missing_verifier"
|
||||
foreach ($field in $verifierRequired) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.verifier.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_verifier_$field"
|
||||
$missingFieldRejections++
|
||||
}
|
||||
|
||||
@@ -169,12 +185,12 @@ Write-ReplayTrace "reject_types"
|
||||
foreach ($mutation in $typeMutations) {
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.($mutation.field) = $mutation.value
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_$($mutation.field)"
|
||||
$typeRejections++
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $success
|
||||
$mutated.verifier.remoteWritePerformed = "false"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed"
|
||||
Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_verifier_remoteWritePerformed"
|
||||
$typeRejections++
|
||||
|
||||
$cleanupPending = Copy-ReplayDocument $success
|
||||
@@ -204,6 +220,174 @@ $terminalReceiptFailed.verifier = $null
|
||||
Write-ReplayTrace "accept_terminal_receipt_failed"
|
||||
Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null
|
||||
|
||||
$EvidencePath = "C:\Wooo\Agent99\evidence\host110-backup-runtime\agent99-host110-backup-runtime-$RunId.json"
|
||||
$preflight = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
fileCount = 18
|
||||
backupScriptFileCount = 17
|
||||
backupHealthExporterIncluded = $true
|
||||
remoteWritePerformed = $false
|
||||
}
|
||||
$brokerSuccess = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
|
||||
ok = $true
|
||||
status = "verified"
|
||||
mode = "Apply"
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $RunId
|
||||
targetHost = $TargetHost
|
||||
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
||||
executor = "host110_backup_runtime_executor"
|
||||
verifier = "independent_host110_backup_runtime_python_readback"
|
||||
decisionProvider = "deterministic_only"
|
||||
criticProvider = "deterministic_only"
|
||||
agentAction = "controlled_apply"
|
||||
check = $preflight
|
||||
apply = $success
|
||||
verify = $verifier
|
||||
cleanupVerified = $true
|
||||
errorCode = ""
|
||||
elapsedSeconds = 1.25
|
||||
evidence = $EvidencePath
|
||||
secretValuesRead = $false
|
||||
rawSessionStored = $false
|
||||
}
|
||||
Write-ReplayTrace "accept_existing_success"
|
||||
Assert-Agent99ExistingBrokerEvidence $brokerSuccess $EvidencePath
|
||||
|
||||
$brokerRequired = @(
|
||||
"schemaVersion", "ok", "status", "mode", "sourceRevision", "sourceHead",
|
||||
"runId", "targetHost", "sourceTransport", "executor", "verifier",
|
||||
"decisionProvider", "criticProvider", "agentAction", "check", "apply",
|
||||
"verify", "cleanupVerified", "errorCode", "elapsedSeconds", "evidence",
|
||||
"secretValuesRead", "rawSessionStored"
|
||||
)
|
||||
$existingEvidenceRejections = 0
|
||||
Write-ReplayTrace "reject_existing_missing"
|
||||
foreach ($field in $brokerRequired) {
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.PSObject.Properties.Remove($field)
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_missing_$field"
|
||||
$existingEvidenceRejections++
|
||||
}
|
||||
$existingMutations = @(
|
||||
@{ name = "ok_string_false"; field = "ok"; value = "false" },
|
||||
@{ name = "wrong_mode"; field = "mode"; value = "Verify" },
|
||||
@{ name = "wrong_run_id"; field = "runId"; value = "other-run" },
|
||||
@{ name = "wrong_status"; field = "status"; value = "readback_ok" },
|
||||
@{ name = "wrong_cleanup"; field = "cleanupVerified"; value = $false },
|
||||
@{ name = "cleanup_string"; field = "cleanupVerified"; value = "true" },
|
||||
@{ name = "elapsed_string"; field = "elapsedSeconds"; value = "1.25" }
|
||||
)
|
||||
Write-ReplayTrace "reject_existing_conflicts"
|
||||
foreach ($mutation in $existingMutations) {
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.($mutation.field) = $mutation.value
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_$($mutation.name)"
|
||||
$existingEvidenceRejections++
|
||||
}
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.apply.runId = "other-run"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_apply_run_conflict"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.verify.PSObject.Properties.Remove("selfIdentityVerified")
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_verify_nested_missing"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated | Add-Member -NotePropertyName unexpected -NotePropertyValue $true
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_top"
|
||||
$existingEvidenceRejections++
|
||||
$mutated = Copy-ReplayDocument $brokerSuccess
|
||||
$mutated.apply | Add-Member -NotePropertyName unexpected -NotePropertyValue $true
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_apply"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$brokerFailure = Copy-ReplayDocument $brokerSuccess
|
||||
$brokerFailure.ok = $false
|
||||
$brokerFailure.status = "failed"
|
||||
$brokerFailure.cleanupVerified = $false
|
||||
$brokerFailure.errorCode = "host110_apply_cleanup_pending"
|
||||
$brokerFailure.apply = $cleanupPending
|
||||
$brokerFailure.verify = $null
|
||||
Write-ReplayTrace "accept_existing_failure"
|
||||
Assert-Agent99ExistingBrokerEvidence $brokerFailure $EvidencePath
|
||||
$failureWithoutPreflight = Copy-ReplayDocument $brokerFailure
|
||||
$failureWithoutPreflight.check = $null
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithoutPreflight $EvidencePath } "failure_preflight_missing"
|
||||
$existingEvidenceRejections++
|
||||
$failureWithImpossibleVerifier = Copy-ReplayDocument $brokerFailure
|
||||
$failureWithImpossibleVerifier.verify = $verifier
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithImpossibleVerifier $EvidencePath } "failure_verifier_conflict"
|
||||
$existingEvidenceRejections++
|
||||
$postcommitCleanupFailure = Copy-ReplayDocument $brokerSuccess
|
||||
$postcommitCleanupFailure.ok = $false
|
||||
$postcommitCleanupFailure.status = "failed"
|
||||
$postcommitCleanupFailure.cleanupVerified = $false
|
||||
$postcommitCleanupFailure.errorCode = "remote_source_stage_cleanup_failed"
|
||||
Assert-Agent99ExistingBrokerEvidence $postcommitCleanupFailure $EvidencePath
|
||||
$malformedFailure = Copy-ReplayDocument $brokerFailure
|
||||
$malformedFailure.ok = "false"
|
||||
Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $malformedFailure $EvidencePath } "malformed_failure_not_upgraded"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$sourcePackage = [pscustomobject]@{ sourceHead = $sourceHead }
|
||||
$transportLoss = [pscustomobject]@{ reason = "transport_timeout"; exitCode = -2 }
|
||||
$receiptReadback = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_backup_transport_receipt_readback_v1"
|
||||
ok = $true
|
||||
sourceRevision = $SourceRevision
|
||||
sourceHead = $sourceHead
|
||||
runId = $RunId
|
||||
payloadReceipt = $payloadReceipt
|
||||
payloadState = "valid"
|
||||
terminalReceipt = $terminalReceipt
|
||||
terminalState = "missing"
|
||||
terminalStatus = ""
|
||||
terminalCleanupVerified = $false
|
||||
terminalExitCode = -1
|
||||
remoteWritePerformed = $false
|
||||
secretValuesRead = $false
|
||||
}
|
||||
Write-ReplayTrace "reconcile_terminal_missing"
|
||||
$terminalMissing = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $receiptReadback $verifier
|
||||
if (
|
||||
$terminalMissing.status -ne "committed_verified_terminal_missing" -or
|
||||
-not $terminalMissing.payloadCommitted -or
|
||||
-not $terminalMissing.destinationVerified -or
|
||||
$terminalMissing.retryApplyAllowed
|
||||
) { throw "transport_terminal_missing_reconciliation_failed" }
|
||||
Assert-Agent99ReconciliationEvidence $terminalMissing $sourceHead
|
||||
|
||||
$terminalPresentReadback = Copy-ReplayDocument $receiptReadback
|
||||
$terminalPresentReadback.terminalState = "valid"
|
||||
$terminalPresentReadback.terminalStatus = "verified"
|
||||
$terminalPresentReadback.terminalCleanupVerified = $true
|
||||
$terminalPresentReadback.terminalExitCode = 0
|
||||
$transportRecovered = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $terminalPresentReadback $verifier
|
||||
if ($transportRecovered.status -ne "committed_verified_transport_lost" -or $transportRecovered.retryApplyAllowed) {
|
||||
throw "transport_verified_reconciliation_failed"
|
||||
}
|
||||
Assert-Agent99ReconciliationEvidence $transportRecovered $sourceHead
|
||||
$conflictingTransport = Copy-ReplayDocument $transportRecovered
|
||||
$conflictingTransport.terminalStatus = "cleanup_pending"
|
||||
Assert-Rejected { Assert-Agent99ReconciliationEvidence $conflictingTransport $sourceHead } "transport_terminal_status_conflict"
|
||||
$existingEvidenceRejections++
|
||||
|
||||
$committedUnknown = New-Agent99TransportLossReconciliation `
|
||||
$transportLoss $sourcePackage $null $null "receipt_readback_unavailable" "verifier_unavailable"
|
||||
if (
|
||||
$committedUnknown.status -ne "committed_unknown" -or
|
||||
$committedUnknown.payloadCommitted -or
|
||||
$committedUnknown.destinationVerified -or
|
||||
$committedUnknown.retryApplyAllowed
|
||||
) { throw "transport_unknown_reconciliation_failed" }
|
||||
Assert-Agent99ReconciliationEvidence $committedUnknown $sourceHead
|
||||
|
||||
Write-ReplayTrace "complete"
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host110_broker_contract_replay_v1"
|
||||
@@ -213,6 +397,11 @@ Write-ReplayTrace "complete"
|
||||
cleanupPendingPreserved = $true
|
||||
committedSignalPreserved = $true
|
||||
terminalReceiptFailurePreserved = $true
|
||||
existingSuccessAccepted = $true
|
||||
existingFailurePreserved = $true
|
||||
existingEvidenceRejections = $existingEvidenceRejections
|
||||
terminalMissingReconciled = $true
|
||||
committedUnknownPreserved = $true
|
||||
missingFieldRejections = $missingFieldRejections
|
||||
typeRejections = $typeRejections
|
||||
remoteWritePerformed = $false
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import fcntl
|
||||
import grp
|
||||
import hashlib
|
||||
import importlib.util
|
||||
@@ -94,6 +95,88 @@ def _write_executable(path: Path, source: str) -> None:
|
||||
path.chmod(0o755)
|
||||
|
||||
|
||||
def _write_fcntl_flock(path: Path) -> None:
|
||||
_write_executable(
|
||||
path,
|
||||
"""#!/usr/bin/env python3
|
||||
import fcntl
|
||||
import sys
|
||||
import time
|
||||
|
||||
args = sys.argv[1:]
|
||||
operation = fcntl.LOCK_SH if "-s" in args else fcntl.LOCK_EX
|
||||
wait_seconds = float(args[args.index("-w") + 1]) if "-w" in args else 0.0
|
||||
descriptor = int(args[-1])
|
||||
deadline = time.monotonic() + wait_seconds
|
||||
while True:
|
||||
try:
|
||||
fcntl.flock(descriptor, operation | fcntl.LOCK_NB)
|
||||
raise SystemExit(0)
|
||||
except BlockingIOError:
|
||||
if time.monotonic() >= deadline:
|
||||
raise SystemExit(1)
|
||||
time.sleep(0.02)
|
||||
""",
|
||||
)
|
||||
|
||||
|
||||
def _build_shared_lock_gate_fixture(
|
||||
tmp_path: Path,
|
||||
entrypoint: str,
|
||||
*,
|
||||
reject_fd_reopen: bool = False,
|
||||
body_sleep_seconds: float = 0.0,
|
||||
) -> tuple[Path, Path, Path, dict[str, str]]:
|
||||
runtime_root = tmp_path / "runtime-scripts"
|
||||
runtime_root.mkdir()
|
||||
lock_path = tmp_path / "runtime.lock"
|
||||
fake_bin = tmp_path / "fake-bin"
|
||||
fake_bin.mkdir()
|
||||
_write_fcntl_flock(fake_bin / "flock")
|
||||
|
||||
source_map = {
|
||||
"common": ROOT / "scripts/backup/common.sh",
|
||||
"restore": ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh",
|
||||
"offsite": ROOT / "scripts/backup/backup-offsite-readiness-gate.sh",
|
||||
}
|
||||
source = source_map[entrypoint].read_text(encoding="utf-8")
|
||||
source = source.replace("/backup/scripts", str(runtime_root))
|
||||
source = source.replace("/tmp/agent99-host110-backup-runtime.lock", str(lock_path))
|
||||
source = source.replace("flock -s -w 60", "flock -s -w 1")
|
||||
if reject_fd_reopen:
|
||||
source, replacement_count = re.subn(
|
||||
r"(?m)^(\s*)exec 197>>.*$",
|
||||
r"\1printf 'FD_REOPENED=1\\n' >&2; exit 98",
|
||||
source,
|
||||
)
|
||||
assert replacement_count == 1
|
||||
|
||||
body = "printf 'BODY_REACHED=1\\n'\n"
|
||||
if body_sleep_seconds:
|
||||
body += f"sleep {body_sleep_seconds}\n"
|
||||
if entrypoint == "common":
|
||||
common_path = runtime_root / "common.sh"
|
||||
gate_source = source.split("# WOOO AIOps - 備份共用函式庫", 1)[0]
|
||||
_write_executable(common_path, gate_source)
|
||||
script_path = runtime_root / "backup-gitea.sh"
|
||||
_write_executable(
|
||||
script_path,
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
f"source {str(common_path)!r}\n"
|
||||
+ body,
|
||||
)
|
||||
else:
|
||||
product_start = "DUMP_ZIP=" if entrypoint == "restore" else "BACKUP_BASE="
|
||||
gate_source = source.split(product_start, 1)[0]
|
||||
script_path = runtime_root / source_map[entrypoint].name
|
||||
_write_executable(script_path, gate_source + body)
|
||||
|
||||
environment = os.environ.copy()
|
||||
environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
|
||||
return script_path, runtime_root, lock_path, environment
|
||||
|
||||
|
||||
def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]:
|
||||
destination = tmp_path / "backup-scripts"
|
||||
exporter_root = tmp_path / "exporter"
|
||||
@@ -337,6 +420,10 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None:
|
||||
assert '$property = $Object.PSObject.Properties[$Name]' in source
|
||||
assert 'if ($null -eq $property) { return $false }' in source
|
||||
assert 'Get-Agent99CommittedFailureResult $applyTransport' in source
|
||||
assert 'Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage' in source
|
||||
assert 'status = "committed_unknown"' in source
|
||||
assert '"committed_verified_terminal_missing"' in source
|
||||
assert 'retryApplyAllowed = $false' in source
|
||||
assert 'throw "host110_apply_$([string]$apply.status)"' in source
|
||||
|
||||
|
||||
@@ -405,11 +492,19 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w
|
||||
for function_name in (
|
||||
"Test-Agent99JsonField",
|
||||
"Assert-Agent99JsonFields",
|
||||
"Assert-Agent99ExactJsonFields",
|
||||
"Convert-Agent99ExecutorDocument",
|
||||
"Assert-Agent99ApplyResultFields",
|
||||
"Assert-Agent99VerifierFields",
|
||||
"Get-Agent99ExecutorResult",
|
||||
"Get-Agent99CommittedFailureResult",
|
||||
"Assert-Agent99VerifierEvidence",
|
||||
"Assert-Agent99PreflightEvidence",
|
||||
"Get-Agent99TransportReceiptReadback",
|
||||
"New-Agent99TransportLossReconciliation",
|
||||
"Assert-Agent99ExecutorEvidence",
|
||||
"Assert-Agent99ReconciliationEvidence",
|
||||
"Assert-Agent99ExistingBrokerEvidence",
|
||||
):
|
||||
assert f'"{function_name}"' in source
|
||||
assert "$mutated.PSObject.Properties.Remove($field)" in source
|
||||
@@ -417,12 +512,188 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w
|
||||
assert 'status = "cleanup_pending"' in source
|
||||
assert 'status = "committed_signal_deferred"' in source
|
||||
assert 'status = "terminal_receipt_failed"' in source
|
||||
assert 'ok = "false"' in source
|
||||
assert 'status -ne "committed_verified_terminal_missing"' in source
|
||||
assert 'status -ne "committed_unknown"' in source
|
||||
assert "existingEvidenceRejections" in source
|
||||
assert "malformed_failure_not_upgraded" in source
|
||||
assert "remoteWritePerformed = $false" in source
|
||||
assert "secretValuesRead = $false" in source
|
||||
for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"):
|
||||
assert forbidden not in source
|
||||
|
||||
|
||||
def test_existing_broker_evidence_replay_is_exact_typed_and_nested_fail_closed() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
exact_guard = _powershell_function(source, "Assert-Agent99ExactJsonFields")
|
||||
existing = _powershell_function(source, "Assert-Agent99ExistingBrokerEvidence")
|
||||
reconciliation = _powershell_function(source, "Assert-Agent99ReconciliationEvidence")
|
||||
|
||||
assert "$properties.Count -ne $Contract.Count" in exact_guard
|
||||
assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard
|
||||
for field in (
|
||||
"schemaVersion",
|
||||
"ok",
|
||||
"status",
|
||||
"mode",
|
||||
"sourceRevision",
|
||||
"sourceHead",
|
||||
"runId",
|
||||
"targetHost",
|
||||
"sourceTransport",
|
||||
"executor",
|
||||
"verifier",
|
||||
"decisionProvider",
|
||||
"criticProvider",
|
||||
"agentAction",
|
||||
"check",
|
||||
"apply",
|
||||
"verify",
|
||||
"cleanupVerified",
|
||||
"errorCode",
|
||||
"elapsedSeconds",
|
||||
"evidence",
|
||||
"secretValuesRead",
|
||||
"rawSessionStored",
|
||||
):
|
||||
assert re.search(rf"^\s+{field} = \"", existing, re.MULTILINE), field
|
||||
assert "Assert-Agent99PreflightEvidence $Existing.check" in existing
|
||||
assert "Assert-Agent99ExecutorEvidence $Existing.apply" in existing
|
||||
assert "Assert-Agent99ReconciliationEvidence $Existing.apply" in existing
|
||||
assert "Assert-Agent99VerifierEvidence $Existing.verify" in existing
|
||||
assert 'throw "existing_broker_apply_failure_evidence_missing"' in existing
|
||||
assert 'retryApplyAllowed = "Boolean"' in reconciliation
|
||||
assert "host110_existing_reconciliation_payload_truth_mismatch" in reconciliation
|
||||
|
||||
|
||||
def test_transport_loss_reconciliation_is_read_only_run_bound_and_never_retries_apply() -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
|
||||
reconcile = _powershell_function(source, "Invoke-Agent99TransportLossReconciliation")
|
||||
classify = _powershell_function(source, "New-Agent99TransportLossReconciliation")
|
||||
|
||||
for expected in (
|
||||
"agent99-backup-runtime-deploy-{args.run_id}.json",
|
||||
"agent99-backup-runtime-terminal-{args.run_id}.json",
|
||||
'set(document) == set(contract)',
|
||||
'payload["status"] == "payload_verified"',
|
||||
'terminal["payloadReceipt"] == str(payload_path)',
|
||||
'"remoteWritePerformed": False',
|
||||
'"secretValuesRead": False',
|
||||
):
|
||||
assert expected in reader
|
||||
assert "Invoke-Agent99BoundedSsh $command 60" in reconcile
|
||||
assert "Invoke-Agent99NoWriteVerifier $SourcePackage" in reconcile
|
||||
assert 'status = "committed_unknown"' in classify
|
||||
assert '"committed_verified_terminal_missing"' in classify
|
||||
assert '"committed_verified_transport_lost"' in classify
|
||||
assert "retryApplyAllowed = $false" in classify
|
||||
for forbidden in ("rm ", "mv ", "install ", "scp", "--mode apply"):
|
||||
assert forbidden not in reconcile
|
||||
|
||||
|
||||
def test_transport_receipt_readback_rejects_extra_and_wrong_typed_documents(tmp_path: Path) -> None:
|
||||
source = BROKER.read_text(encoding="utf-8")
|
||||
reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0]
|
||||
status_root = tmp_path / "status"
|
||||
status_root.mkdir()
|
||||
run_id = "transport-loss-replay"
|
||||
source_revision = "a" * 40
|
||||
source_head = "b" * 40
|
||||
verifier_sha256 = "c" * 64
|
||||
payload_path = status_root / f"agent99-backup-runtime-deploy-{run_id}.json"
|
||||
terminal_path = status_root / f"agent99-backup-runtime-terminal-{run_id}.json"
|
||||
payload = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3",
|
||||
"status": "payload_verified",
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": source_head,
|
||||
"runId": run_id,
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"rollbackAttempted": False,
|
||||
"rollbackPerformed": False,
|
||||
"rollbackVerified": False,
|
||||
"zeroResidueVerified": True,
|
||||
"zeroResidueScope": "run_owned_destination_temps",
|
||||
"verifierSha256": verifier_sha256,
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": 1,
|
||||
}
|
||||
terminal = {
|
||||
"schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1",
|
||||
"status": "verified",
|
||||
"ok": True,
|
||||
"payloadCommitted": True,
|
||||
"payloadReceipt": str(payload_path),
|
||||
"sourceRevision": source_revision,
|
||||
"sourceHead": source_head,
|
||||
"runId": run_id,
|
||||
"fileCount": 18,
|
||||
"backupScriptFileCount": 17,
|
||||
"backupHealthExporterIncluded": True,
|
||||
"stageCleanupVerified": True,
|
||||
"rollbackPrestateCleanupVerified": True,
|
||||
"cleanupVerified": True,
|
||||
"independentVerifierVerified": True,
|
||||
"zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate",
|
||||
"deferredSignal": None,
|
||||
"terminalExitCode": 0,
|
||||
"verifierSha256": verifier_sha256,
|
||||
"executorHost": "192.168.0.110",
|
||||
"productionServiceRestarted": False,
|
||||
"secretValuesRead": False,
|
||||
"writtenAt": 2,
|
||||
}
|
||||
|
||||
def run_readback() -> dict[str, object]:
|
||||
result = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
"-",
|
||||
"--status-root",
|
||||
str(status_root),
|
||||
"--source-revision",
|
||||
source_revision,
|
||||
"--source-head",
|
||||
source_head,
|
||||
"--run-id",
|
||||
run_id,
|
||||
"--verifier-sha256",
|
||||
verifier_sha256,
|
||||
],
|
||||
input=reader,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
assert result.returncode == 0, result.stderr
|
||||
return json.loads(result.stdout)
|
||||
|
||||
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
|
||||
valid = run_readback()
|
||||
assert valid["payloadState"] == "valid"
|
||||
assert valid["terminalState"] == "valid"
|
||||
assert valid["terminalStatus"] == "verified"
|
||||
|
||||
payload["unexpected"] = True
|
||||
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
assert run_readback()["payloadState"] == "invalid"
|
||||
payload.pop("unexpected")
|
||||
payload["writtenAt"] = "1"
|
||||
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
assert run_readback()["payloadState"] == "invalid"
|
||||
payload["writtenAt"] = 1
|
||||
payload_path.write_text(json.dumps(payload), encoding="utf-8")
|
||||
terminal["ok"] = "false"
|
||||
terminal_path.write_text(json.dumps(terminal), encoding="utf-8")
|
||||
assert run_readback()["terminalState"] == "invalid"
|
||||
|
||||
|
||||
def test_broker_scp_basenames_match_executor_source_stage_contract() -> None:
|
||||
broker = BROKER.read_text(encoding="utf-8")
|
||||
executor = EXECUTOR.read_text(encoding="utf-8")
|
||||
@@ -854,8 +1125,258 @@ def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None:
|
||||
for source in (common, restore, offsite):
|
||||
assert "/tmp/agent99-host110-backup-runtime.lock" in source
|
||||
assert "flock -s -w 60" in source
|
||||
assert "realpath -e" in source
|
||||
assert "readlink -f" in source
|
||||
assert "identity unresolved" in source
|
||||
assert "/proc/$$/fd/197" in source
|
||||
assert "/dev/fd/197" in source
|
||||
assert "inherited fd 197 identity mismatch" in source
|
||||
assert source.index("backup_runtime_inherited_fd_status") < source.index("exec 197>>")
|
||||
assert 'resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}"' in common
|
||||
assert '${PWD}/${raw_path}' in common
|
||||
assert '[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0' not in common
|
||||
for source in (restore, offsite):
|
||||
assert 'resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}"' in source
|
||||
assert '&& [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]' not in source
|
||||
assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive
|
||||
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive
|
||||
assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" not in archive
|
||||
assert 'Path(f"/proc/{os.getpid()}/fd")' in archive
|
||||
assert 'Path("/dev/fd/197")' in archive
|
||||
assert "os.dup(197)" in archive
|
||||
|
||||
|
||||
def test_common_shared_lock_preserves_resolved_nonproduction_fixture_behavior(tmp_path: Path) -> None:
|
||||
gate_source = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8").split(
|
||||
"# WOOO AIOps - 備份共用函式庫",
|
||||
1,
|
||||
)[0]
|
||||
gate_fixture = tmp_path / "common-gate.sh"
|
||||
_write_executable(gate_fixture, gate_source)
|
||||
wrapper = tmp_path / "fixture-entrypoint.sh"
|
||||
_write_executable(
|
||||
wrapper,
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
f"source {str(gate_fixture)!r}\n"
|
||||
"printf 'NONPRODUCTION_FIXTURE_OK=1\\n'\n",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[str(wrapper)],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert "NONPRODUCTION_FIXTURE_OK=1" in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
||||
@pytest.mark.parametrize("invocation", ["absolute", "relative", "symlink"])
|
||||
def test_backup_runtime_shared_lock_blocks_absolute_relative_and_symlink_invocation(
|
||||
tmp_path: Path,
|
||||
entrypoint: str,
|
||||
invocation: str,
|
||||
) -> None:
|
||||
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
|
||||
tmp_path,
|
||||
entrypoint,
|
||||
)
|
||||
|
||||
command_path = str(script_path)
|
||||
cwd = runtime_root
|
||||
if invocation == "relative":
|
||||
command_path = f"./{script_path.name}"
|
||||
elif invocation == "symlink":
|
||||
link_path = tmp_path / f"{entrypoint}-entrypoint-link"
|
||||
link_path.symlink_to(script_path)
|
||||
command_path = str(link_path)
|
||||
|
||||
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
|
||||
lock_handle = lock_path.open("a+")
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(command_path)],
|
||||
cwd=cwd,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=3,
|
||||
)
|
||||
finally:
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
|
||||
lock_handle.close()
|
||||
|
||||
assert result.returncode == 75
|
||||
assert "backup runtime deployment is active" in result.stderr
|
||||
assert "BODY_REACHED=1" not in result.stdout
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
||||
def test_backup_runtime_shared_lock_reuses_exact_inherited_fd_without_reopen(
|
||||
tmp_path: Path,
|
||||
entrypoint: str,
|
||||
) -> None:
|
||||
script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture(
|
||||
tmp_path,
|
||||
entrypoint,
|
||||
reject_fd_reopen=True,
|
||||
body_sleep_seconds=0.5,
|
||||
)
|
||||
wrapper = tmp_path / "run-with-inherited-lock.sh"
|
||||
_write_executable(
|
||||
wrapper,
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
f"exec 197>>{str(lock_path)!r}\n"
|
||||
"flock -s -w 1 197\n"
|
||||
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=forged_external_value\n"
|
||||
f"exec {str(script_path)!r}\n",
|
||||
)
|
||||
|
||||
process = subprocess.Popen(
|
||||
[str(wrapper)],
|
||||
cwd=runtime_root,
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
assert process.stdout is not None
|
||||
assert process.stdout.readline().strip() == "BODY_REACHED=1"
|
||||
contender = lock_path.open("a+")
|
||||
try:
|
||||
with pytest.raises(BlockingIOError):
|
||||
fcntl.flock(contender.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
finally:
|
||||
contender.close()
|
||||
stdout, stderr = process.communicate(timeout=3)
|
||||
|
||||
assert process.returncode == 0, stderr
|
||||
assert "FD_REOPENED=1" not in stderr
|
||||
assert stdout == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"])
|
||||
def test_backup_runtime_shared_lock_rejects_inherited_fd_for_another_file(
|
||||
tmp_path: Path,
|
||||
entrypoint: str,
|
||||
) -> None:
|
||||
script_path, runtime_root, _, environment = _build_shared_lock_gate_fixture(tmp_path, entrypoint)
|
||||
wrong_lock = tmp_path / "wrong-runtime.lock"
|
||||
wrapper = tmp_path / "run-with-wrong-inherited-lock.sh"
|
||||
_write_executable(
|
||||
wrapper,
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
f"exec 197>>{str(wrong_lock)!r}\n"
|
||||
"export BACKUP_RUNTIME_SHARED_LOCK_HELD=1\n"
|
||||
f"exec {str(script_path)!r}\n",
|
||||
)
|
||||
|
||||
result = subprocess.run(
|
||||
[str(wrapper)],
|
||||
cwd=runtime_root,
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
assert result.returncode == 69
|
||||
assert "inherited fd 197 identity mismatch" in result.stderr
|
||||
assert "BODY_REACHED=1" not in result.stdout
|
||||
|
||||
|
||||
def test_host188_archive_lock_ignores_forged_env_and_validates_inherited_fd(tmp_path: Path) -> None:
|
||||
runtime_root = tmp_path / "runtime-scripts"
|
||||
runtime_root.mkdir()
|
||||
lock_path = tmp_path / "runtime.lock"
|
||||
archive_path = runtime_root / "verify-host188-products-archive.py"
|
||||
archive_source = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8")
|
||||
archive_source = archive_source.replace('Path("/backup/scripts")', f"Path({str(runtime_root)!r})")
|
||||
archive_source = archive_source.replace(
|
||||
'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")',
|
||||
f"RUNTIME_LOCK = Path({str(lock_path)!r})",
|
||||
)
|
||||
archive_source = archive_source.replace("time.monotonic() + 60", "time.monotonic() + 0.2")
|
||||
_write_executable(archive_path, archive_source)
|
||||
runner = tmp_path / "archive-lock-replay.py"
|
||||
runner.write_text(
|
||||
"""import fcntl
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
|
||||
archive_path = pathlib.Path(sys.argv[1])
|
||||
lock_path = pathlib.Path(sys.argv[2])
|
||||
mode = sys.argv[3]
|
||||
spec = importlib.util.spec_from_file_location("host188_archive_lock_replay", archive_path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(module)
|
||||
if mode in {"exact", "wrong"}:
|
||||
selected = lock_path if mode == "exact" else lock_path.with_name("wrong-runtime.lock")
|
||||
selected.touch()
|
||||
inherited = selected.open("a+")
|
||||
fcntl.flock(inherited.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB)
|
||||
os.dup2(inherited.fileno(), 197)
|
||||
try:
|
||||
handle = module.acquire_runtime_lock()
|
||||
except RuntimeError as exc:
|
||||
print(str(exc))
|
||||
raise SystemExit(69)
|
||||
else:
|
||||
print("LOCK_OK=1")
|
||||
if handle is not None:
|
||||
handle.close()
|
||||
""",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
environment = os.environ.copy()
|
||||
environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1"
|
||||
lock_handle = lock_path.open("a+")
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
try:
|
||||
forged = subprocess.run(
|
||||
[sys.executable, str(runner), str(archive_path), str(lock_path), "forged"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=3,
|
||||
)
|
||||
finally:
|
||||
fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN)
|
||||
lock_handle.close()
|
||||
exact = subprocess.run(
|
||||
[sys.executable, str(runner), str(archive_path), str(lock_path), "exact"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=3,
|
||||
)
|
||||
wrong = subprocess.run(
|
||||
[sys.executable, str(runner), str(archive_path), str(lock_path), "wrong"],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
timeout=3,
|
||||
)
|
||||
|
||||
assert forged.returncode == 69
|
||||
assert "backup_runtime_deployment_active" in forged.stdout
|
||||
assert exact.returncode == 0, exact.stdout + exact.stderr
|
||||
assert "LOCK_OK=1" in exact.stdout
|
||||
assert wrong.returncode == 69
|
||||
assert "backup_runtime_inherited_fd_mismatch" in wrong.stdout
|
||||
|
||||
|
||||
def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None:
|
||||
|
||||
Reference in New Issue
Block a user