1377 lines
55 KiB
PowerShell
1377 lines
55 KiB
PowerShell
[CmdletBinding()]
|
|
param(
|
|
[Parameter(Mandatory = $true)]
|
|
[ValidateSet("Check", "Apply", "Verify")]
|
|
[string]$Mode,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$SourceRevision,
|
|
[Parameter(Mandatory = $true)]
|
|
[string]$RunId,
|
|
[string]$AgentRoot = "C:\Wooo\Agent99"
|
|
)
|
|
|
|
$ErrorActionPreference = "Stop"
|
|
$ExpectedAgentRoot = "C:\Wooo\Agent99"
|
|
$TargetHost = "192.168.0.110"
|
|
$TargetUser = "wooo"
|
|
$GiteaSourceUrl = "https://gitea.wooo.work/wooo/awoooi.git"
|
|
$GitPath = "C:\Program Files\Git\cmd\git.exe"
|
|
$IdentityFile = Join-Path $ExpectedAgentRoot "keys\agent99_ed25519"
|
|
$EvidenceRoot = Join-Path $ExpectedAgentRoot "evidence\host110-backup-runtime"
|
|
$SourceRootBase = Join-Path $ExpectedAgentRoot "deploy"
|
|
$TransportMutexName = "Global\WoooAgent99Host110BackupRuntimeV2"
|
|
$ExpectedFileCount = 18
|
|
$ExpectedRemoteStageFileCount = 21
|
|
$RuntimeRelativePaths = @(
|
|
"scripts/backup/common.sh",
|
|
"scripts/backup/backup-all.sh",
|
|
"scripts/backup/backup-host188-products.sh",
|
|
"scripts/backup/verify-host188-products-backup.sh",
|
|
"scripts/backup/verify-host188-products-archive.py",
|
|
"scripts/backup/backup-gitea.sh",
|
|
"scripts/backup/gitea-full-backup-restore-drill.sh",
|
|
"scripts/backup/backup-configs.sh",
|
|
"scripts/backup/backup-awoooi.sh",
|
|
"scripts/backup/backup-awoooi-frequent.sh",
|
|
"scripts/backup/backup-clawbot.sh",
|
|
"scripts/backup/backup-sentry.sh",
|
|
"scripts/backup/check-backup-integrity.sh",
|
|
"scripts/backup/sync-offsite-backups.sh",
|
|
"scripts/backup/backup-offsite-readiness-gate.sh",
|
|
"scripts/backup/verify-offsite-full-sync.sh",
|
|
"scripts/backup/enforce-latest-only-retention.sh",
|
|
"scripts/ops/backup-health-textfile-exporter.py"
|
|
)
|
|
$ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh"
|
|
$VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py"
|
|
$TransportReconciliationReadback = @'
|
|
#!/usr/bin/env python3
|
|
import argparse
|
|
import json
|
|
import re
|
|
from pathlib import Path
|
|
|
|
|
|
def is_string(value):
|
|
return type(value) is str
|
|
|
|
|
|
def is_boolean(value):
|
|
return type(value) is bool
|
|
|
|
|
|
def is_integer(value):
|
|
return type(value) is int
|
|
|
|
|
|
def exact_document(document, contract):
|
|
return (
|
|
type(document) is dict
|
|
and set(document) == set(contract)
|
|
and all(contract[name](document[name]) for name in contract)
|
|
)
|
|
|
|
|
|
def read_document(path):
|
|
if path.is_symlink():
|
|
return "invalid", None
|
|
if not path.exists():
|
|
return "missing", None
|
|
if not path.is_file() or path.stat().st_size > 65536:
|
|
return "invalid", None
|
|
try:
|
|
return "present", json.loads(path.read_text(encoding="utf-8"))
|
|
except (OSError, UnicodeError, json.JSONDecodeError):
|
|
return "invalid", None
|
|
|
|
|
|
parser = argparse.ArgumentParser()
|
|
parser.add_argument("--status-root", required=True)
|
|
parser.add_argument("--source-revision", required=True)
|
|
parser.add_argument("--source-head", required=True)
|
|
parser.add_argument("--run-id", required=True)
|
|
parser.add_argument("--verifier-sha256", required=True)
|
|
args = parser.parse_args()
|
|
|
|
if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision):
|
|
raise SystemExit(64)
|
|
if not re.fullmatch(r"[0-9a-f]{40}", args.source_head):
|
|
raise SystemExit(64)
|
|
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id):
|
|
raise SystemExit(64)
|
|
if not re.fullmatch(r"[0-9a-f]{64}", args.verifier_sha256):
|
|
raise SystemExit(64)
|
|
|
|
status_root = Path(args.status_root)
|
|
if status_root.is_symlink() or not status_root.is_dir():
|
|
raise SystemExit(66)
|
|
payload_path = status_root / f"agent99-backup-runtime-deploy-{args.run_id}.json"
|
|
terminal_path = status_root / f"agent99-backup-runtime-terminal-{args.run_id}.json"
|
|
|
|
payload_contract = {
|
|
"schemaVersion": is_string,
|
|
"status": is_string,
|
|
"sourceRevision": is_string,
|
|
"sourceHead": is_string,
|
|
"runId": is_string,
|
|
"fileCount": is_integer,
|
|
"backupScriptFileCount": is_integer,
|
|
"backupHealthExporterIncluded": is_boolean,
|
|
"rollbackAttempted": is_boolean,
|
|
"rollbackPerformed": is_boolean,
|
|
"rollbackVerified": is_boolean,
|
|
"zeroResidueVerified": is_boolean,
|
|
"zeroResidueScope": is_string,
|
|
"verifierSha256": is_string,
|
|
"executorHost": is_string,
|
|
"productionServiceRestarted": is_boolean,
|
|
"secretValuesRead": is_boolean,
|
|
"writtenAt": is_integer,
|
|
}
|
|
terminal_contract = {
|
|
"schemaVersion": is_string,
|
|
"status": is_string,
|
|
"ok": is_boolean,
|
|
"payloadCommitted": is_boolean,
|
|
"payloadReceipt": is_string,
|
|
"sourceRevision": is_string,
|
|
"sourceHead": is_string,
|
|
"runId": is_string,
|
|
"fileCount": is_integer,
|
|
"backupScriptFileCount": is_integer,
|
|
"backupHealthExporterIncluded": is_boolean,
|
|
"stageCleanupVerified": is_boolean,
|
|
"rollbackPrestateCleanupVerified": is_boolean,
|
|
"cleanupVerified": is_boolean,
|
|
"independentVerifierVerified": is_boolean,
|
|
"zeroResidueScope": is_string,
|
|
"deferredSignal": lambda value: value is None or is_string(value),
|
|
"terminalExitCode": is_integer,
|
|
"verifierSha256": is_string,
|
|
"executorHost": is_string,
|
|
"productionServiceRestarted": is_boolean,
|
|
"secretValuesRead": is_boolean,
|
|
"writtenAt": is_integer,
|
|
}
|
|
|
|
payload_state, payload = read_document(payload_path)
|
|
if payload_state == "present":
|
|
valid_payload = exact_document(payload, payload_contract) and (
|
|
payload["schemaVersion"] == "agent99_host110_backup_runtime_deploy_receipt_v3"
|
|
and payload["status"] == "payload_verified"
|
|
and payload["sourceRevision"] == args.source_revision
|
|
and payload["sourceHead"] == args.source_head
|
|
and payload["runId"] == args.run_id
|
|
and payload["fileCount"] == 18
|
|
and payload["backupScriptFileCount"] == 17
|
|
and payload["backupHealthExporterIncluded"] is True
|
|
and payload["rollbackAttempted"] is False
|
|
and payload["rollbackPerformed"] is False
|
|
and payload["rollbackVerified"] is False
|
|
and payload["zeroResidueVerified"] is True
|
|
and payload["zeroResidueScope"] == "run_owned_destination_temps"
|
|
and payload["verifierSha256"] == args.verifier_sha256
|
|
and payload["executorHost"] == "192.168.0.110"
|
|
and payload["productionServiceRestarted"] is False
|
|
and payload["secretValuesRead"] is False
|
|
)
|
|
payload_state = "valid" if valid_payload else "invalid"
|
|
|
|
terminal_state, terminal = read_document(terminal_path)
|
|
terminal_status = ""
|
|
terminal_cleanup_verified = False
|
|
terminal_exit_code = -1
|
|
if terminal_state == "present":
|
|
valid_terminal = exact_document(terminal, terminal_contract) and (
|
|
terminal["schemaVersion"] == "agent99_host110_backup_runtime_terminal_receipt_v1"
|
|
and terminal["status"] in {
|
|
"verified", "cleanup_pending", "committed_signal_deferred"
|
|
}
|
|
and terminal["ok"] is (terminal["status"] == "verified")
|
|
and terminal["payloadCommitted"] is True
|
|
and terminal["payloadReceipt"] == str(payload_path)
|
|
and terminal["sourceRevision"] == args.source_revision
|
|
and terminal["sourceHead"] == args.source_head
|
|
and terminal["runId"] == args.run_id
|
|
and terminal["fileCount"] == 18
|
|
and terminal["backupScriptFileCount"] == 17
|
|
and terminal["backupHealthExporterIncluded"] is True
|
|
and terminal["cleanupVerified"] is (
|
|
terminal["stageCleanupVerified"]
|
|
and terminal["rollbackPrestateCleanupVerified"]
|
|
)
|
|
and terminal["independentVerifierVerified"] is True
|
|
and terminal["zeroResidueScope"]
|
|
== "run_owned_destination_temps_and_internal_stage_prestate"
|
|
and terminal["verifierSha256"] == args.verifier_sha256
|
|
and terminal["executorHost"] == "192.168.0.110"
|
|
and terminal["productionServiceRestarted"] is False
|
|
and terminal["secretValuesRead"] is False
|
|
)
|
|
if valid_terminal and terminal["status"] == "verified":
|
|
valid_terminal = (
|
|
terminal["cleanupVerified"] is True
|
|
and terminal["deferredSignal"] is None
|
|
and terminal["terminalExitCode"] == 0
|
|
)
|
|
elif valid_terminal and terminal["status"] == "cleanup_pending":
|
|
valid_terminal = (
|
|
terminal["cleanupVerified"] is False
|
|
and terminal["terminalExitCode"] == 75
|
|
)
|
|
elif valid_terminal:
|
|
signal_codes = {"INT": 130, "HUP": 129, "TERM": 143}
|
|
valid_terminal = (
|
|
terminal["cleanupVerified"] is True
|
|
and terminal["deferredSignal"] in signal_codes
|
|
and terminal["terminalExitCode"]
|
|
== signal_codes[terminal["deferredSignal"]]
|
|
)
|
|
terminal_state = "valid" if valid_terminal else "invalid"
|
|
if valid_terminal:
|
|
terminal_status = terminal["status"]
|
|
terminal_cleanup_verified = terminal["cleanupVerified"]
|
|
terminal_exit_code = terminal["terminalExitCode"]
|
|
|
|
print(json.dumps({
|
|
"schemaVersion": "agent99_host110_backup_transport_receipt_readback_v1",
|
|
"ok": True,
|
|
"sourceRevision": args.source_revision,
|
|
"sourceHead": args.source_head,
|
|
"runId": args.run_id,
|
|
"payloadReceipt": str(payload_path),
|
|
"payloadState": payload_state,
|
|
"terminalReceipt": str(terminal_path),
|
|
"terminalState": terminal_state,
|
|
"terminalStatus": terminal_status,
|
|
"terminalCleanupVerified": terminal_cleanup_verified,
|
|
"terminalExitCode": terminal_exit_code,
|
|
"remoteWritePerformed": False,
|
|
"secretValuesRead": False,
|
|
}, ensure_ascii=True, sort_keys=True, separators=(",", ":")))
|
|
'@
|
|
|
|
function Test-Agent99SafeIdentity {
|
|
param([string]$Value)
|
|
return [bool]($Value -and $Value -match "^[A-Za-z0-9][A-Za-z0-9._-]{0,95}$")
|
|
}
|
|
|
|
function Write-Agent99ImmutableJson {
|
|
param([string]$Path, [object]$Value)
|
|
|
|
if (Test-Path -LiteralPath $Path) { throw "run_identity_evidence_exists" }
|
|
$temporary = "$Path.tmp-$PID-$([Guid]::NewGuid().ToString('N'))"
|
|
$encoding = New-Object System.Text.UTF8Encoding($false)
|
|
try {
|
|
$json = $Value | ConvertTo-Json -Depth 12
|
|
[IO.File]::WriteAllText($temporary, $json + [Environment]::NewLine, $encoding)
|
|
[IO.File]::Move($temporary, $Path)
|
|
} finally {
|
|
Remove-Item -LiteralPath $temporary -Force -ErrorAction SilentlyContinue | Out-Null
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99BoundedProcess {
|
|
param(
|
|
[string]$FilePath,
|
|
[string[]]$Arguments,
|
|
[int]$TimeoutSeconds,
|
|
[string]$WorkingDirectory = "",
|
|
[string]$StandardInputPath = ""
|
|
)
|
|
|
|
$token = [Guid]::NewGuid().ToString("N")
|
|
$stdoutPath = Join-Path $env:TEMP "agent99-host110-backup-$token.out"
|
|
$stderrPath = Join-Path $env:TEMP "agent99-host110-backup-$token.err"
|
|
$stopwatch = [Diagnostics.Stopwatch]::StartNew()
|
|
$process = $null
|
|
try {
|
|
$start = @{
|
|
FilePath = $FilePath
|
|
ArgumentList = $Arguments
|
|
NoNewWindow = $true
|
|
RedirectStandardOutput = $stdoutPath
|
|
RedirectStandardError = $stderrPath
|
|
PassThru = $true
|
|
}
|
|
if ($WorkingDirectory) { $start.WorkingDirectory = $WorkingDirectory }
|
|
if ($StandardInputPath) { $start.RedirectStandardInput = $StandardInputPath }
|
|
$process = Start-Process @start
|
|
$null = $process.Handle
|
|
if (-not $process.WaitForExit($TimeoutSeconds * 1000)) {
|
|
try { & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null } catch {}
|
|
if (-not $process.WaitForExit(2000) -and -not $process.HasExited) {
|
|
$process.Kill()
|
|
$process.WaitForExit(2000) | Out-Null
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
exitCode = -2
|
|
reason = "transport_timeout"
|
|
elapsedMs = $stopwatch.ElapsedMilliseconds
|
|
stdout = ""
|
|
stderrPresent = $false
|
|
}
|
|
}
|
|
$process.WaitForExit() | Out-Null
|
|
$process.Refresh()
|
|
$stdout = if (Test-Path -LiteralPath $stdoutPath) { Get-Content -LiteralPath $stdoutPath -Raw } else { "" }
|
|
$stderr = if (Test-Path -LiteralPath $stderrPath) { Get-Content -LiteralPath $stderrPath -Raw } else { "" }
|
|
if ($stdout.Length -gt 65536 -or $stderr.Length -gt 65536) {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
exitCode = -3
|
|
reason = "transport_output_oversized"
|
|
elapsedMs = $stopwatch.ElapsedMilliseconds
|
|
stdout = ""
|
|
stderrPresent = [bool]$stderr
|
|
}
|
|
}
|
|
return [pscustomobject]@{
|
|
ok = [bool]($process.ExitCode -eq 0)
|
|
exitCode = [int]$process.ExitCode
|
|
reason = if ($process.ExitCode -eq 0) { "completed" } else { "process_failed" }
|
|
elapsedMs = $stopwatch.ElapsedMilliseconds
|
|
stdout = [string]$stdout
|
|
stderrPresent = [bool]$stderr
|
|
}
|
|
} catch {
|
|
return [pscustomobject]@{
|
|
ok = $false
|
|
exitCode = -1
|
|
reason = "transport_exception"
|
|
elapsedMs = $stopwatch.ElapsedMilliseconds
|
|
stdout = ""
|
|
stderrPresent = $false
|
|
errorType = $_.Exception.GetType().Name
|
|
}
|
|
} finally {
|
|
$stopwatch.Stop()
|
|
Remove-Item -LiteralPath $stdoutPath, $stderrPath -Force -ErrorAction SilentlyContinue | Out-Null
|
|
if ($process) { $process.Dispose() }
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99BoundedSsh {
|
|
param([string]$RemoteCommand, [int]$TimeoutSeconds, [string]$StandardInputPath = "")
|
|
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ConnectionAttempts=1",
|
|
"-o", "ServerAliveInterval=10",
|
|
"-o", "ServerAliveCountMax=2",
|
|
"-l", $TargetUser,
|
|
$TargetHost,
|
|
$RemoteCommand
|
|
)
|
|
return Invoke-Agent99BoundedProcess "ssh.exe" $arguments $TimeoutSeconds "" $StandardInputPath
|
|
}
|
|
|
|
function Invoke-Agent99BoundedScp {
|
|
param([string[]]$LocalPaths, [string]$RemoteDirectory, [int]$TimeoutSeconds)
|
|
|
|
$arguments = @(
|
|
"-i", $IdentityFile,
|
|
"-o", "IdentitiesOnly=yes",
|
|
"-o", "BatchMode=yes",
|
|
"-o", "NumberOfPasswordPrompts=0",
|
|
"-o", "StrictHostKeyChecking=yes",
|
|
"-o", "ConnectTimeout=8",
|
|
"-o", "ConnectionAttempts=1"
|
|
)
|
|
$arguments += $LocalPaths
|
|
$arguments += "${TargetUser}@${TargetHost}:${RemoteDirectory}/"
|
|
return Invoke-Agent99BoundedProcess "scp.exe" $arguments $TimeoutSeconds
|
|
}
|
|
|
|
function Invoke-Agent99Git {
|
|
param([string[]]$Arguments, [int]$TimeoutSeconds, [string]$WorkingDirectory = "")
|
|
|
|
$previousPrompt = $env:GIT_TERMINAL_PROMPT
|
|
try {
|
|
$env:GIT_TERMINAL_PROMPT = "0"
|
|
return Invoke-Agent99BoundedProcess $GitPath $Arguments $TimeoutSeconds $WorkingDirectory
|
|
} finally {
|
|
$env:GIT_TERMINAL_PROMPT = $previousPrompt
|
|
}
|
|
}
|
|
|
|
function Assert-Agent99ProcessOk {
|
|
param([object]$Result, [string]$Reason)
|
|
if (-not $Result.ok) { throw $Reason }
|
|
}
|
|
|
|
function New-Agent99SourcePackage {
|
|
param([string]$SourceRoot, [string]$ManifestPath)
|
|
|
|
if (Test-Path -LiteralPath $SourceRoot) { throw "run_identity_source_stage_exists" }
|
|
$clone = Invoke-Agent99Git @(
|
|
"-c", "credential.helper=",
|
|
"-c", "core.autocrlf=false",
|
|
"clone", "--quiet", "--no-tags", "--depth", "200", "--branch", "main",
|
|
$GiteaSourceUrl, $SourceRoot
|
|
) 180 $SourceRootBase
|
|
Assert-Agent99ProcessOk $clone "gitea_source_fetch_failed"
|
|
|
|
$headResult = Invoke-Agent99Git @("rev-parse", "HEAD") 30 $SourceRoot
|
|
Assert-Agent99ProcessOk $headResult "gitea_source_head_failed"
|
|
$sourceHead = ([string]$headResult.stdout).Trim().ToLowerInvariant()
|
|
if ($sourceHead -notmatch "^[0-9a-f]{40}$") { throw "gitea_source_head_invalid" }
|
|
|
|
$commitResult = Invoke-Agent99Git @("cat-file", "-e", "$SourceRevision`^{commit}") 30 $SourceRoot
|
|
Assert-Agent99ProcessOk $commitResult "source_revision_not_fetched"
|
|
$ancestorResult = Invoke-Agent99Git @("merge-base", "--is-ancestor", $SourceRevision, $sourceHead) 30 $SourceRoot
|
|
Assert-Agent99ProcessOk $ancestorResult "source_revision_not_on_gitea_main"
|
|
|
|
$allPaths = @($RuntimeRelativePaths) + @($ExecutorRelativePath, $VerifierRelativePath)
|
|
$checkoutResult = Invoke-Agent99Git (@("-c", "core.autocrlf=false", "checkout", "--quiet", $SourceRevision, "--") + $allPaths) 60 $SourceRoot
|
|
Assert-Agent99ProcessOk $checkoutResult "source_revision_checkout_failed"
|
|
|
|
$rows = @()
|
|
$localPaths = @()
|
|
foreach ($relativePath in $RuntimeRelativePaths) {
|
|
$path = Join-Path $SourceRoot ($relativePath -replace "/", "\")
|
|
if (-not (Test-Path -LiteralPath $path -PathType Leaf)) { throw "runtime_source_file_missing" }
|
|
$rows += [ordered]@{
|
|
name = [IO.Path]::GetFileName($path)
|
|
sha256 = (Get-FileHash -LiteralPath $path -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
}
|
|
$localPaths += $path
|
|
}
|
|
if (($rows.name | Sort-Object -Unique).Count -ne $ExpectedFileCount) { throw "runtime_source_file_set_invalid" }
|
|
|
|
$executorPath = Join-Path $SourceRoot ($ExecutorRelativePath -replace "/", "\")
|
|
$verifierPath = Join-Path $SourceRoot ($VerifierRelativePath -replace "/", "\")
|
|
if (-not (Test-Path -LiteralPath $executorPath -PathType Leaf)) { throw "executor_source_file_missing" }
|
|
if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" }
|
|
$executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant()
|
|
$reconciliationPath = Join-Path $SourceRoot "host110-backup-transport-reconciliation.py"
|
|
$encoding = New-Object System.Text.UTF8Encoding($false)
|
|
[IO.File]::WriteAllText($reconciliationPath, $TransportReconciliationReadback, $encoding)
|
|
$manifest = [ordered]@{
|
|
schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1"
|
|
sourceRevision = $SourceRevision
|
|
sourceHead = $sourceHead
|
|
runId = $RunId
|
|
executorSha256 = $executorDigest
|
|
verifierSha256 = $verifierDigest
|
|
files = $rows
|
|
}
|
|
[IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding)
|
|
$localPaths += $executorPath, $verifierPath, $ManifestPath
|
|
return [pscustomobject]@{
|
|
sourceHead = $sourceHead
|
|
executorSha256 = $executorDigest
|
|
verifierSha256 = $verifierDigest
|
|
verifierPath = $verifierPath
|
|
reconciliationPath = $reconciliationPath
|
|
manifestPath = $ManifestPath
|
|
manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath))
|
|
localPaths = $localPaths
|
|
}
|
|
}
|
|
|
|
function Test-Agent99JsonField {
|
|
param(
|
|
[object]$Object,
|
|
[string]$Name,
|
|
[ValidateSet("String", "Boolean", "Integer", "Number", "Object", "ObjectOrNull", "Null", "NullOrString")]
|
|
[string]$Kind
|
|
)
|
|
|
|
if ($null -eq $Object) { return $false }
|
|
$property = $Object.PSObject.Properties[$Name]
|
|
if ($null -eq $property) { return $false }
|
|
$value = $property.Value
|
|
switch ($Kind) {
|
|
"String" { return [bool]($value -is [string]) }
|
|
"Boolean" { return [bool]($value -is [bool]) }
|
|
"Integer" { return [bool]($value -is [int] -or $value -is [long]) }
|
|
"Number" { return [bool]($value -is [int] -or $value -is [long] -or $value -is [double] -or $value -is [decimal]) }
|
|
"Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) }
|
|
"ObjectOrNull" { return [bool]($null -eq $value -or $value -is [System.Management.Automation.PSCustomObject]) }
|
|
"Null" { return [bool]($null -eq $value) }
|
|
"NullOrString" { return [bool]($null -eq $value -or $value -is [string]) }
|
|
}
|
|
return $false
|
|
}
|
|
|
|
function Assert-Agent99JsonFields {
|
|
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
|
|
|
|
foreach ($name in $Contract.Keys) {
|
|
if (-not (Test-Agent99JsonField $Object $name ([string]$Contract[$name]))) {
|
|
throw $ErrorCode
|
|
}
|
|
}
|
|
}
|
|
|
|
function Assert-Agent99ExactJsonFields {
|
|
param([object]$Object, [hashtable]$Contract, [string]$ErrorCode)
|
|
|
|
Assert-Agent99JsonFields $Object $Contract $ErrorCode
|
|
$properties = @($Object.PSObject.Properties)
|
|
if ($properties.Count -ne $Contract.Count) { throw $ErrorCode }
|
|
foreach ($property in $properties) {
|
|
if (-not $Contract.ContainsKey([string]$property.Name)) { throw $ErrorCode }
|
|
}
|
|
}
|
|
|
|
function Convert-Agent99ExecutorDocument {
|
|
param(
|
|
[object]$Transport,
|
|
[string]$ExpectedMode,
|
|
[switch]$AllowProcessFailure
|
|
)
|
|
|
|
if (-not $Transport.ok) {
|
|
if (
|
|
-not $AllowProcessFailure -or
|
|
[string]$Transport.reason -ne "process_failed" -or
|
|
[int]$Transport.exitCode -le 0 -or
|
|
[string]::IsNullOrWhiteSpace([string]$Transport.stdout)
|
|
) {
|
|
throw "host110_${ExpectedMode}_transport_failed"
|
|
}
|
|
}
|
|
try {
|
|
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
|
} catch {
|
|
throw "host110_${ExpectedMode}_receipt_invalid"
|
|
}
|
|
Assert-Agent99JsonFields $parsed @{
|
|
schemaVersion = "String"
|
|
mode = "String"
|
|
ok = "Boolean"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
runId = "String"
|
|
fileCount = "Integer"
|
|
backupScriptFileCount = "Integer"
|
|
backupHealthExporterIncluded = "Boolean"
|
|
} "host110_${ExpectedMode}_receipt_required_field_failed"
|
|
if (
|
|
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
|
|
[string]$parsed.mode -ne $ExpectedMode.ToLowerInvariant() -or
|
|
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
|
[string]$parsed.sourceHead -notmatch "^[0-9a-f]{40}$" -or
|
|
[string]$parsed.runId -ne $RunId -or
|
|
[int64]$parsed.fileCount -ne $ExpectedFileCount -or
|
|
[int64]$parsed.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$parsed.backupHealthExporterIncluded
|
|
) {
|
|
throw "host110_${ExpectedMode}_receipt_contract_failed"
|
|
}
|
|
return $parsed
|
|
}
|
|
|
|
function Assert-Agent99ApplyResultFields {
|
|
param([object]$Parsed, [string]$VerifierKind)
|
|
|
|
Assert-Agent99ExactJsonFields $Parsed @{
|
|
schemaVersion = "String"
|
|
mode = "String"
|
|
status = "String"
|
|
ok = "Boolean"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
runId = "String"
|
|
fileCount = "Integer"
|
|
backupScriptFileCount = "Integer"
|
|
backupHealthExporterIncluded = "Boolean"
|
|
payloadCommitted = "Boolean"
|
|
rollbackAttempted = "Boolean"
|
|
rollbackPerformed = "Boolean"
|
|
rollbackVerified = "Boolean"
|
|
zeroResidueVerified = "Boolean"
|
|
zeroResidueScope = "String"
|
|
stageCleanupVerified = "Boolean"
|
|
rollbackPrestateCleanupVerified = "Boolean"
|
|
receipt = "String"
|
|
terminalReceipt = "String"
|
|
terminalReceiptPublished = "Boolean"
|
|
terminalExitCode = "Integer"
|
|
deferredSignal = "NullOrString"
|
|
independentVerifierVerified = "Boolean"
|
|
verifier = $VerifierKind
|
|
} "host110_apply_receipt_required_field_failed"
|
|
}
|
|
|
|
function Assert-Agent99VerifierFields {
|
|
param([object]$Verifier)
|
|
|
|
Assert-Agent99ExactJsonFields $Verifier @{
|
|
schemaVersion = "String"
|
|
ok = "Boolean"
|
|
sourceRevision = "String"
|
|
runId = "String"
|
|
fileCount = "Integer"
|
|
backupScriptFileCount = "Integer"
|
|
backupHealthExporterIncluded = "Boolean"
|
|
destination = "String"
|
|
exporterDestination = "String"
|
|
remoteWritePerformed = "Boolean"
|
|
secretValuesRead = "Boolean"
|
|
selfIdentityVerified = "Boolean"
|
|
verifierSha256 = "String"
|
|
} "host110_apply_verifier_required_field_failed"
|
|
}
|
|
|
|
function Get-Agent99ExecutorResult {
|
|
param([object]$Transport, [string]$ExpectedMode, [string]$ExpectedVerifierSha256 = "")
|
|
|
|
$parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode
|
|
if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" }
|
|
if ($ExpectedMode -eq "Apply") {
|
|
Assert-Agent99ApplyResultFields $parsed "Object"
|
|
Assert-Agent99VerifierFields $parsed.verifier
|
|
if (
|
|
[string]$parsed.status -ne "verified" -or
|
|
-not [bool]$parsed.payloadCommitted -or
|
|
[bool]$parsed.rollbackAttempted -or
|
|
[bool]$parsed.rollbackPerformed -or
|
|
[bool]$parsed.rollbackVerified -or
|
|
-not [bool]$parsed.zeroResidueVerified -or
|
|
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
|
|
-not [bool]$parsed.stageCleanupVerified -or
|
|
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
|
|
[int64]$parsed.terminalExitCode -ne 0 -or
|
|
$null -ne $parsed.deferredSignal -or
|
|
-not [bool]$parsed.independentVerifierVerified -or
|
|
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
|
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
|
-not [bool]$parsed.terminalReceiptPublished -or
|
|
[string]$parsed.verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
|
-not [bool]$parsed.verifier.ok -or
|
|
[string]$parsed.verifier.sourceRevision -ne $SourceRevision -or
|
|
[string]$parsed.verifier.runId -ne $RunId -or
|
|
[int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or
|
|
[int64]$parsed.verifier.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$parsed.verifier.backupHealthExporterIncluded -or
|
|
[string]$parsed.verifier.destination -ne "/backup/scripts" -or
|
|
[string]$parsed.verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
|
[bool]$parsed.verifier.remoteWritePerformed -or
|
|
[bool]$parsed.verifier.secretValuesRead -or
|
|
-not [bool]$parsed.verifier.selfIdentityVerified -or
|
|
$ExpectedVerifierSha256 -notmatch "^[0-9a-f]{64}$" -or
|
|
[string]$parsed.verifier.verifierSha256 -ne $ExpectedVerifierSha256
|
|
) {
|
|
throw "host110_apply_independent_verifier_contract_failed"
|
|
}
|
|
}
|
|
return $parsed
|
|
}
|
|
|
|
function Get-Agent99CommittedFailureResult {
|
|
param([object]$Transport)
|
|
|
|
$parsed = Convert-Agent99ExecutorDocument $Transport "Apply" -AllowProcessFailure
|
|
Assert-Agent99ApplyResultFields $parsed "Null"
|
|
if (
|
|
[bool]$parsed.ok -or
|
|
-not [bool]$parsed.payloadCommitted -or
|
|
[bool]$parsed.rollbackAttempted -or
|
|
[bool]$parsed.rollbackPerformed -or
|
|
[bool]$parsed.rollbackVerified -or
|
|
-not [bool]$parsed.zeroResidueVerified -or
|
|
[string]$parsed.zeroResidueScope -ne "run_owned_destination_temps" -or
|
|
-not [bool]$parsed.independentVerifierVerified -or
|
|
[string]$parsed.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
|
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json"
|
|
) {
|
|
throw "host110_apply_committed_failure_contract_failed"
|
|
}
|
|
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
|
|
if ([string]$parsed.status -eq "cleanup_pending") {
|
|
if (
|
|
[int]$Transport.exitCode -ne 75 -or
|
|
[int64]$parsed.terminalExitCode -ne 75 -or
|
|
([bool]$parsed.stageCleanupVerified -and [bool]$parsed.rollbackPrestateCleanupVerified) -or
|
|
($null -ne $parsed.deferredSignal -and -not $signalExitCodes.ContainsKey([string]$parsed.deferredSignal))
|
|
) {
|
|
throw "host110_apply_cleanup_pending_contract_failed"
|
|
}
|
|
if (-not [bool]$parsed.terminalReceiptPublished) {
|
|
throw "host110_apply_cleanup_pending_terminal_receipt_missing"
|
|
}
|
|
} elseif ([string]$parsed.status -eq "committed_signal_deferred") {
|
|
$signalName = [string]$parsed.deferredSignal
|
|
if (
|
|
-not $signalExitCodes.ContainsKey($signalName) -or
|
|
-not [bool]$parsed.stageCleanupVerified -or
|
|
-not [bool]$parsed.rollbackPrestateCleanupVerified -or
|
|
[int]$Transport.exitCode -ne [int]$signalExitCodes[$signalName] -or
|
|
[int64]$parsed.terminalExitCode -ne [int]$signalExitCodes[$signalName]
|
|
) {
|
|
throw "host110_apply_deferred_signal_contract_failed"
|
|
}
|
|
if (-not [bool]$parsed.terminalReceiptPublished) {
|
|
throw "host110_apply_deferred_signal_terminal_receipt_missing"
|
|
}
|
|
} elseif ([string]$parsed.status -eq "terminal_receipt_failed") {
|
|
if (
|
|
[bool]$parsed.terminalReceiptPublished -or
|
|
[int]$Transport.exitCode -ne 76 -or
|
|
[int64]$parsed.terminalExitCode -ne 76
|
|
) {
|
|
throw "host110_apply_terminal_receipt_failure_contract_failed"
|
|
}
|
|
} else {
|
|
throw "host110_apply_committed_failure_status_invalid"
|
|
}
|
|
return $parsed
|
|
}
|
|
|
|
function Get-Agent99VerifierResult {
|
|
param([object]$Transport, [object]$SourcePackage)
|
|
|
|
if (-not $Transport.ok) { throw "host110_verify_transport_failed" }
|
|
try {
|
|
$parsed = [string]$Transport.stdout | ConvertFrom-Json
|
|
} catch {
|
|
throw "host110_verify_receipt_invalid"
|
|
}
|
|
Assert-Agent99VerifierFields $parsed
|
|
if (
|
|
[string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
|
-not [bool]$parsed.ok -or
|
|
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
|
[string]$parsed.runId -ne $RunId -or
|
|
[int]$parsed.fileCount -ne $ExpectedFileCount -or
|
|
[int]$parsed.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$parsed.backupHealthExporterIncluded -or
|
|
[string]$parsed.destination -ne "/backup/scripts" -or
|
|
[string]$parsed.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
|
[bool]$parsed.remoteWritePerformed -or
|
|
[bool]$parsed.secretValuesRead -or
|
|
-not [bool]$parsed.selfIdentityVerified -or
|
|
[string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256
|
|
) {
|
|
throw "host110_verify_receipt_contract_failed"
|
|
}
|
|
return $parsed
|
|
}
|
|
|
|
function Assert-Agent99VerifierEvidence {
|
|
param([object]$Verifier, [string]$ExpectedVerifierSha256 = "")
|
|
|
|
Assert-Agent99VerifierFields $Verifier
|
|
if (
|
|
[string]$Verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or
|
|
-not [bool]$Verifier.ok -or
|
|
[string]$Verifier.sourceRevision -ne $SourceRevision -or
|
|
[string]$Verifier.runId -ne $RunId -or
|
|
[int64]$Verifier.fileCount -ne $ExpectedFileCount -or
|
|
[int64]$Verifier.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$Verifier.backupHealthExporterIncluded -or
|
|
[string]$Verifier.destination -ne "/backup/scripts" -or
|
|
[string]$Verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or
|
|
[bool]$Verifier.remoteWritePerformed -or
|
|
[bool]$Verifier.secretValuesRead -or
|
|
-not [bool]$Verifier.selfIdentityVerified -or
|
|
[string]$Verifier.verifierSha256 -notmatch "^[0-9a-f]{64}$" -or
|
|
($ExpectedVerifierSha256 -and [string]$Verifier.verifierSha256 -ne $ExpectedVerifierSha256)
|
|
) {
|
|
throw "host110_existing_verifier_contract_failed"
|
|
}
|
|
}
|
|
|
|
function Assert-Agent99PreflightEvidence {
|
|
param([object]$Preflight, [string]$ExpectedSourceHead)
|
|
|
|
Assert-Agent99ExactJsonFields $Preflight @{
|
|
schemaVersion = "String"
|
|
ok = "Boolean"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
fileCount = "Integer"
|
|
backupScriptFileCount = "Integer"
|
|
backupHealthExporterIncluded = "Boolean"
|
|
remoteWritePerformed = "Boolean"
|
|
} "host110_existing_preflight_required_field_failed"
|
|
if (
|
|
[string]$Preflight.schemaVersion -ne "agent99_host110_backup_runtime_preflight_v1" -or
|
|
-not [bool]$Preflight.ok -or
|
|
[string]$Preflight.sourceRevision -ne $SourceRevision -or
|
|
[string]$Preflight.sourceHead -ne $ExpectedSourceHead -or
|
|
[int64]$Preflight.fileCount -ne $ExpectedFileCount -or
|
|
[int64]$Preflight.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$Preflight.backupHealthExporterIncluded -or
|
|
[bool]$Preflight.remoteWritePerformed
|
|
) {
|
|
throw "host110_existing_preflight_contract_failed"
|
|
}
|
|
}
|
|
|
|
function Get-Agent99TransportReceiptReadback {
|
|
param([object]$Transport, [object]$SourcePackage)
|
|
|
|
if (-not $Transport.ok) { throw "host110_transport_receipt_readback_failed" }
|
|
try { $parsed = [string]$Transport.stdout | ConvertFrom-Json } catch {
|
|
throw "host110_transport_receipt_readback_invalid"
|
|
}
|
|
Assert-Agent99ExactJsonFields $parsed @{
|
|
schemaVersion = "String"
|
|
ok = "Boolean"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
runId = "String"
|
|
payloadReceipt = "String"
|
|
payloadState = "String"
|
|
terminalReceipt = "String"
|
|
terminalState = "String"
|
|
terminalStatus = "String"
|
|
terminalCleanupVerified = "Boolean"
|
|
terminalExitCode = "Integer"
|
|
remoteWritePerformed = "Boolean"
|
|
secretValuesRead = "Boolean"
|
|
} "host110_transport_receipt_readback_required_field_failed"
|
|
if (
|
|
[string]$parsed.schemaVersion -ne "agent99_host110_backup_transport_receipt_readback_v1" -or
|
|
-not [bool]$parsed.ok -or
|
|
[string]$parsed.sourceRevision -ne $SourceRevision -or
|
|
[string]$parsed.sourceHead -ne [string]$SourcePackage.sourceHead -or
|
|
[string]$parsed.runId -ne $RunId -or
|
|
[string]$parsed.payloadReceipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
|
[string]$parsed.payloadState -notin @("missing", "invalid", "valid") -or
|
|
[string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
|
[string]$parsed.terminalState -notin @("missing", "invalid", "valid") -or
|
|
[bool]$parsed.remoteWritePerformed -or
|
|
[bool]$parsed.secretValuesRead
|
|
) {
|
|
throw "host110_transport_receipt_readback_contract_failed"
|
|
}
|
|
if ([string]$parsed.terminalState -eq "valid") {
|
|
if (
|
|
[string]$parsed.terminalStatus -notin @("verified", "cleanup_pending", "committed_signal_deferred") -or
|
|
([string]$parsed.terminalStatus -eq "verified" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 0)) -or
|
|
([string]$parsed.terminalStatus -eq "cleanup_pending" -and ([bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 75)) -or
|
|
([string]$parsed.terminalStatus -eq "committed_signal_deferred" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -notin @(129, 130, 143)))
|
|
) {
|
|
throw "host110_transport_terminal_readback_contract_failed"
|
|
}
|
|
} elseif (
|
|
[string]$parsed.terminalStatus -ne "" -or
|
|
[bool]$parsed.terminalCleanupVerified -or
|
|
[int64]$parsed.terminalExitCode -ne -1
|
|
) {
|
|
throw "host110_transport_terminal_absence_contract_failed"
|
|
}
|
|
return $parsed
|
|
}
|
|
|
|
function New-Agent99TransportLossReconciliation {
|
|
param(
|
|
[object]$Transport,
|
|
[object]$SourcePackage,
|
|
[object]$ReceiptReadback,
|
|
[object]$VerifierReadback,
|
|
[string]$ReceiptReadbackError = "",
|
|
[string]$VerifierReadbackError = ""
|
|
)
|
|
|
|
$payloadCommitted = [bool]($ReceiptReadback -and [string]$ReceiptReadback.payloadState -eq "valid")
|
|
$destinationVerified = [bool]($VerifierReadback -and [bool]$VerifierReadback.ok)
|
|
$terminalState = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalState } else { "unavailable" }
|
|
$terminalStatus = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalStatus } else { "" }
|
|
$status = "committed_unknown"
|
|
if ($payloadCommitted -and $destinationVerified -and $terminalState -eq "missing") {
|
|
$status = "committed_verified_terminal_missing"
|
|
} elseif ($payloadCommitted -and $destinationVerified -and $terminalState -eq "valid") {
|
|
$status = switch ($terminalStatus) {
|
|
"verified" { "committed_verified_transport_lost" }
|
|
"cleanup_pending" { "committed_cleanup_pending_transport_lost" }
|
|
"committed_signal_deferred" { "committed_signal_deferred_transport_lost" }
|
|
default { "committed_unknown" }
|
|
}
|
|
}
|
|
return [pscustomobject]@{
|
|
schemaVersion = "agent99_host110_backup_transport_reconciliation_v1"
|
|
ok = $false
|
|
status = $status
|
|
sourceRevision = $SourceRevision
|
|
sourceHead = [string]$SourcePackage.sourceHead
|
|
runId = $RunId
|
|
payloadCommitted = $payloadCommitted
|
|
payloadReceiptState = if ($ReceiptReadback) { [string]$ReceiptReadback.payloadState } else { "unavailable" }
|
|
terminalReceiptState = $terminalState
|
|
terminalStatus = $terminalStatus
|
|
destinationVerified = $destinationVerified
|
|
retryApplyAllowed = $false
|
|
transportReason = if ($Transport -and $Transport.PSObject.Properties["reason"]) { [string]$Transport.reason } else { "unknown" }
|
|
transportExitCode = if ($Transport -and $Transport.PSObject.Properties["exitCode"]) { [int]$Transport.exitCode } else { -1 }
|
|
receiptReadback = $ReceiptReadback
|
|
verifier = $VerifierReadback
|
|
receiptReadbackError = $ReceiptReadbackError
|
|
verifierReadbackError = $VerifierReadbackError
|
|
remoteWritePerformed = $false
|
|
secretValuesRead = $false
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99TransportLossReconciliation {
|
|
param([object]$Transport, [object]$SourcePackage)
|
|
|
|
$receiptReadback = $null
|
|
$verifierReadback = $null
|
|
$receiptReadbackError = ""
|
|
$verifierReadbackError = ""
|
|
try {
|
|
$command = "python3 - --status-root /backup/status --source-revision $SourceRevision --source-head $($SourcePackage.sourceHead) --run-id $RunId --verifier-sha256 $($SourcePackage.verifierSha256)"
|
|
$readbackTransport = Invoke-Agent99BoundedSsh $command 60 ([string]$SourcePackage.reconciliationPath)
|
|
$receiptReadback = Get-Agent99TransportReceiptReadback $readbackTransport $SourcePackage
|
|
} catch {
|
|
$receiptReadbackError = [string]$_.Exception.Message
|
|
}
|
|
try {
|
|
$verifierReadback = Invoke-Agent99NoWriteVerifier $SourcePackage
|
|
} catch {
|
|
$verifierReadbackError = [string]$_.Exception.Message
|
|
}
|
|
return New-Agent99TransportLossReconciliation `
|
|
$Transport $SourcePackage $receiptReadback $verifierReadback `
|
|
$receiptReadbackError $verifierReadbackError
|
|
}
|
|
|
|
function Assert-Agent99ExecutorEvidence {
|
|
param([object]$Executor, [string]$ExpectedSourceHead)
|
|
|
|
Assert-Agent99ApplyResultFields $Executor $(if ([bool]$Executor.ok) { "Object" } else { "Null" })
|
|
if (
|
|
[string]$Executor.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or
|
|
[string]$Executor.mode -ne "apply" -or
|
|
[string]$Executor.sourceRevision -ne $SourceRevision -or
|
|
[string]$Executor.sourceHead -ne $ExpectedSourceHead -or
|
|
[string]$Executor.runId -ne $RunId -or
|
|
[int64]$Executor.fileCount -ne $ExpectedFileCount -or
|
|
[int64]$Executor.backupScriptFileCount -ne 17 -or
|
|
-not [bool]$Executor.backupHealthExporterIncluded -or
|
|
-not [bool]$Executor.payloadCommitted -or
|
|
[bool]$Executor.rollbackAttempted -or
|
|
[bool]$Executor.rollbackPerformed -or
|
|
[bool]$Executor.rollbackVerified -or
|
|
-not [bool]$Executor.zeroResidueVerified -or
|
|
[string]$Executor.zeroResidueScope -ne "run_owned_destination_temps" -or
|
|
[string]$Executor.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or
|
|
[string]$Executor.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or
|
|
-not [bool]$Executor.independentVerifierVerified
|
|
) {
|
|
throw "host110_existing_apply_contract_failed"
|
|
}
|
|
if ([bool]$Executor.ok) {
|
|
Assert-Agent99VerifierEvidence $Executor.verifier
|
|
if (
|
|
[string]$Executor.status -ne "verified" -or
|
|
-not [bool]$Executor.stageCleanupVerified -or
|
|
-not [bool]$Executor.rollbackPrestateCleanupVerified -or
|
|
-not [bool]$Executor.terminalReceiptPublished -or
|
|
[int64]$Executor.terminalExitCode -ne 0 -or
|
|
$null -ne $Executor.deferredSignal
|
|
) {
|
|
throw "host110_existing_apply_success_contract_failed"
|
|
}
|
|
return
|
|
}
|
|
|
|
if ($null -ne $Executor.verifier) { throw "host110_existing_apply_failure_verifier_present" }
|
|
$signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 }
|
|
if ([string]$Executor.status -eq "cleanup_pending") {
|
|
if (
|
|
([bool]$Executor.stageCleanupVerified -and [bool]$Executor.rollbackPrestateCleanupVerified) -or
|
|
-not [bool]$Executor.terminalReceiptPublished -or
|
|
[int64]$Executor.terminalExitCode -ne 75
|
|
) { throw "host110_existing_cleanup_pending_contract_failed" }
|
|
} elseif ([string]$Executor.status -eq "committed_signal_deferred") {
|
|
$signalName = [string]$Executor.deferredSignal
|
|
if (
|
|
-not [bool]$Executor.stageCleanupVerified -or
|
|
-not [bool]$Executor.rollbackPrestateCleanupVerified -or
|
|
-not [bool]$Executor.terminalReceiptPublished -or
|
|
-not $signalExitCodes.ContainsKey($signalName) -or
|
|
[int64]$Executor.terminalExitCode -ne [int]$signalExitCodes[$signalName]
|
|
) { throw "host110_existing_committed_signal_contract_failed" }
|
|
} elseif ([string]$Executor.status -eq "terminal_receipt_failed") {
|
|
if (
|
|
[bool]$Executor.terminalReceiptPublished -or
|
|
[int64]$Executor.terminalExitCode -ne 76
|
|
) { throw "host110_existing_terminal_receipt_failure_contract_failed" }
|
|
} else {
|
|
throw "host110_existing_apply_failure_status_invalid"
|
|
}
|
|
}
|
|
|
|
function Assert-Agent99ReconciliationEvidence {
|
|
param([object]$Reconciliation, [string]$ExpectedSourceHead)
|
|
|
|
Assert-Agent99ExactJsonFields $Reconciliation @{
|
|
schemaVersion = "String"
|
|
ok = "Boolean"
|
|
status = "String"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
runId = "String"
|
|
payloadCommitted = "Boolean"
|
|
payloadReceiptState = "String"
|
|
terminalReceiptState = "String"
|
|
terminalStatus = "String"
|
|
destinationVerified = "Boolean"
|
|
retryApplyAllowed = "Boolean"
|
|
transportReason = "String"
|
|
transportExitCode = "Integer"
|
|
receiptReadback = "ObjectOrNull"
|
|
verifier = "ObjectOrNull"
|
|
receiptReadbackError = "String"
|
|
verifierReadbackError = "String"
|
|
remoteWritePerformed = "Boolean"
|
|
secretValuesRead = "Boolean"
|
|
} "host110_existing_reconciliation_required_field_failed"
|
|
if (
|
|
[string]$Reconciliation.schemaVersion -ne "agent99_host110_backup_transport_reconciliation_v1" -or
|
|
[bool]$Reconciliation.ok -or
|
|
[string]$Reconciliation.sourceRevision -ne $SourceRevision -or
|
|
[string]$Reconciliation.sourceHead -ne $ExpectedSourceHead -or
|
|
[string]$Reconciliation.runId -ne $RunId -or
|
|
[string]$Reconciliation.payloadReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or
|
|
[string]$Reconciliation.terminalReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or
|
|
[bool]$Reconciliation.retryApplyAllowed -or
|
|
[bool]$Reconciliation.remoteWritePerformed -or
|
|
[bool]$Reconciliation.secretValuesRead
|
|
) {
|
|
throw "host110_existing_reconciliation_contract_failed"
|
|
}
|
|
if ($Reconciliation.receiptReadback) {
|
|
if ($Reconciliation.receiptReadbackError) { throw "host110_existing_reconciliation_readback_error_conflict" }
|
|
$sourcePackage = [pscustomobject]@{ sourceHead = $ExpectedSourceHead }
|
|
$transport = [pscustomobject]@{
|
|
ok = $true
|
|
stdout = $Reconciliation.receiptReadback | ConvertTo-Json -Compress -Depth 8
|
|
}
|
|
Get-Agent99TransportReceiptReadback $transport $sourcePackage | Out-Null
|
|
} elseif (-not $Reconciliation.receiptReadbackError) {
|
|
throw "host110_existing_reconciliation_readback_error_missing"
|
|
}
|
|
if ($Reconciliation.verifier) {
|
|
if ($Reconciliation.verifierReadbackError) { throw "host110_existing_reconciliation_verifier_error_conflict" }
|
|
Assert-Agent99VerifierEvidence $Reconciliation.verifier
|
|
} elseif (-not $Reconciliation.verifierReadbackError) {
|
|
throw "host110_existing_reconciliation_verifier_error_missing"
|
|
}
|
|
if ([bool]$Reconciliation.payloadCommitted -ne [bool]($Reconciliation.payloadReceiptState -eq "valid")) {
|
|
throw "host110_existing_reconciliation_payload_truth_mismatch"
|
|
}
|
|
if ($Reconciliation.payloadCommitted -and -not $Reconciliation.receiptReadback) {
|
|
throw "host110_existing_reconciliation_payload_readback_missing"
|
|
}
|
|
if ([bool]$Reconciliation.destinationVerified -ne [bool]($null -ne $Reconciliation.verifier)) {
|
|
throw "host110_existing_reconciliation_destination_truth_mismatch"
|
|
}
|
|
$status = [string]$Reconciliation.status
|
|
$knownStatuses = @(
|
|
"committed_verified_terminal_missing",
|
|
"committed_verified_transport_lost",
|
|
"committed_cleanup_pending_transport_lost",
|
|
"committed_signal_deferred_transport_lost",
|
|
"committed_unknown"
|
|
)
|
|
if ($status -notin $knownStatuses) { throw "host110_existing_reconciliation_status_invalid" }
|
|
if ($status -ne "committed_unknown") {
|
|
if (-not $Reconciliation.payloadCommitted -or -not $Reconciliation.destinationVerified) {
|
|
throw "host110_existing_reconciliation_verified_state_invalid"
|
|
}
|
|
}
|
|
if ($status -eq "committed_verified_terminal_missing" -and $Reconciliation.terminalReceiptState -ne "missing") {
|
|
throw "host110_existing_reconciliation_terminal_missing_mismatch"
|
|
}
|
|
$terminalStatusByReconciliation = @{
|
|
committed_verified_transport_lost = "verified"
|
|
committed_cleanup_pending_transport_lost = "cleanup_pending"
|
|
committed_signal_deferred_transport_lost = "committed_signal_deferred"
|
|
}
|
|
if ($terminalStatusByReconciliation.ContainsKey($status)) {
|
|
if (
|
|
$Reconciliation.terminalReceiptState -ne "valid" -or
|
|
[string]$Reconciliation.terminalStatus -ne [string]$terminalStatusByReconciliation[$status]
|
|
) { throw "host110_existing_reconciliation_terminal_state_mismatch" }
|
|
}
|
|
}
|
|
|
|
function Assert-Agent99ExistingBrokerEvidence {
|
|
param([object]$Existing, [string]$ExpectedEvidencePath)
|
|
|
|
Assert-Agent99ExactJsonFields $Existing @{
|
|
schemaVersion = "String"
|
|
ok = "Boolean"
|
|
status = "String"
|
|
mode = "String"
|
|
sourceRevision = "String"
|
|
sourceHead = "String"
|
|
runId = "String"
|
|
targetHost = "String"
|
|
sourceTransport = "String"
|
|
executor = "String"
|
|
verifier = "String"
|
|
decisionProvider = "String"
|
|
criticProvider = "String"
|
|
agentAction = "String"
|
|
check = "ObjectOrNull"
|
|
apply = "ObjectOrNull"
|
|
verify = "ObjectOrNull"
|
|
cleanupVerified = "Boolean"
|
|
errorCode = "String"
|
|
elapsedSeconds = "Number"
|
|
evidence = "String"
|
|
secretValuesRead = "Boolean"
|
|
rawSessionStored = "Boolean"
|
|
} "existing_broker_evidence_required_field_failed"
|
|
if (
|
|
[string]$Existing.mode -ne $Mode -or
|
|
[string]$Existing.sourceRevision -ne $SourceRevision -or
|
|
[string]$Existing.runId -ne $RunId -or
|
|
[string]$Existing.evidence -ne $ExpectedEvidencePath
|
|
) {
|
|
throw "run_identity_conflict"
|
|
}
|
|
if (
|
|
[string]$Existing.schemaVersion -ne "agent99_host110_backup_runtime_broker_receipt_v3" -or
|
|
[string]$Existing.targetHost -ne $TargetHost -or
|
|
[string]$Existing.sourceTransport -ne "windows99_gitea_exact_revision_manifest" -or
|
|
[string]$Existing.executor -ne "host110_backup_runtime_executor" -or
|
|
[string]$Existing.verifier -ne "independent_host110_backup_runtime_python_readback" -or
|
|
[string]$Existing.decisionProvider -ne "deterministic_only" -or
|
|
[string]$Existing.criticProvider -ne "deterministic_only" -or
|
|
[string]$Existing.agentAction -ne $(if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }) -or
|
|
([string]$Existing.sourceHead -ne "" -and [string]$Existing.sourceHead -notmatch "^[0-9a-f]{40}$") -or
|
|
[double]$Existing.elapsedSeconds -lt 0 -or
|
|
[bool]$Existing.secretValuesRead -or
|
|
[bool]$Existing.rawSessionStored
|
|
) {
|
|
throw "existing_broker_evidence_contract_failed"
|
|
}
|
|
if ($Existing.check) { Assert-Agent99PreflightEvidence $Existing.check ([string]$Existing.sourceHead) }
|
|
if ($Existing.verify) { Assert-Agent99VerifierEvidence $Existing.verify }
|
|
if ($Existing.apply) {
|
|
if ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_runtime_executor_v1") {
|
|
Assert-Agent99ExecutorEvidence $Existing.apply ([string]$Existing.sourceHead)
|
|
} elseif ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_transport_reconciliation_v1") {
|
|
Assert-Agent99ReconciliationEvidence $Existing.apply ([string]$Existing.sourceHead)
|
|
} else {
|
|
throw "existing_broker_apply_schema_invalid"
|
|
}
|
|
}
|
|
if (($Existing.apply -or $Existing.verify) -and -not $Existing.check) {
|
|
throw "existing_broker_preflight_evidence_missing"
|
|
}
|
|
if ($Mode -ne "Apply" -and $Existing.apply) { throw "existing_broker_apply_mode_conflict" }
|
|
if ($Mode -eq "Check" -and $Existing.verify) { throw "existing_broker_verify_mode_conflict" }
|
|
|
|
if ([bool]$Existing.ok) {
|
|
if ([string]$Existing.errorCode -ne "" -or -not [bool]$Existing.cleanupVerified) {
|
|
throw "existing_broker_success_terminal_invalid"
|
|
}
|
|
if ($Mode -eq "Apply") {
|
|
if (
|
|
[string]$Existing.status -ne "verified" -or
|
|
-not $Existing.check -or
|
|
-not $Existing.apply -or
|
|
-not [bool]$Existing.apply.ok -or
|
|
-not $Existing.verify -or
|
|
[string]$Existing.apply.verifier.verifierSha256 -ne [string]$Existing.verify.verifierSha256
|
|
) { throw "existing_broker_apply_success_invalid" }
|
|
} elseif ($Mode -eq "Verify") {
|
|
if ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or -not $Existing.verify) {
|
|
throw "existing_broker_verify_success_invalid"
|
|
}
|
|
} elseif ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or $Existing.verify) {
|
|
throw "existing_broker_check_success_invalid"
|
|
}
|
|
return
|
|
}
|
|
|
|
if ([string]$Existing.status -ne "failed" -or [string]::IsNullOrWhiteSpace([string]$Existing.errorCode)) {
|
|
throw "existing_broker_failure_terminal_invalid"
|
|
}
|
|
if ($Existing.verify -and [string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed") {
|
|
throw "existing_broker_failure_verifier_conflict"
|
|
}
|
|
if ([string]$Existing.errorCode -like "host110_apply_*" -and -not $Existing.apply) {
|
|
throw "existing_broker_apply_failure_evidence_missing"
|
|
}
|
|
if ($Existing.apply) {
|
|
if ([bool]$Existing.apply.ok) {
|
|
if (
|
|
[string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed" -or
|
|
-not $Existing.verify -or
|
|
[bool]$Existing.cleanupVerified
|
|
) { throw "existing_broker_postcommit_cleanup_failure_conflict" }
|
|
} elseif ([string]$Existing.errorCode -ne "host110_apply_$([string]$Existing.apply.status)") {
|
|
throw "existing_broker_apply_failure_conflict"
|
|
}
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99ReadOnlyPreflight {
|
|
param([object]$SourcePackage)
|
|
|
|
$command = 'test "$(id -un)" = wooo && hostname -I | tr " " "\n" | grep -qx 192.168.0.110 && test -d /backup/scripts && test ! -L /backup/scripts && test -d /home/wooo/scripts && test ! -L /home/wooo/scripts && command -v timeout >/dev/null && command -v flock >/dev/null && command -v python3 >/dev/null && printf HOST110_BACKUP_PREFLIGHT_OK=1'
|
|
$transport = Invoke-Agent99BoundedSsh $command 45
|
|
if (-not $transport.ok -or ([string]$transport.stdout).Trim() -ne "HOST110_BACKUP_PREFLIGHT_OK=1") {
|
|
throw "host110_read_only_preflight_failed"
|
|
}
|
|
return [pscustomobject]@{
|
|
schemaVersion = "agent99_host110_backup_runtime_preflight_v1"
|
|
ok = $true
|
|
sourceRevision = $SourceRevision
|
|
sourceHead = [string]$SourcePackage.sourceHead
|
|
fileCount = $ExpectedFileCount
|
|
backupScriptFileCount = 17
|
|
backupHealthExporterIncluded = $true
|
|
remoteWritePerformed = $false
|
|
}
|
|
}
|
|
|
|
function Invoke-Agent99NoWriteVerifier {
|
|
param([object]$SourcePackage)
|
|
|
|
$command = "timeout --signal=TERM --kill-after=5s 60s python3 - --manifest-base64 $($SourcePackage.manifestBase64) --destination /backup/scripts --exporter-destination /home/wooo/scripts/backup-health-textfile-exporter.py --source-revision $SourceRevision --run-id $RunId --expected-verifier-sha256 $($SourcePackage.verifierSha256)"
|
|
$transport = Invoke-Agent99BoundedSsh $command 90 ([string]$SourcePackage.verifierPath)
|
|
return Get-Agent99VerifierResult $transport $SourcePackage
|
|
}
|
|
|
|
function Remove-Agent99RemoteStage {
|
|
param([string]$RemoteStage, [bool]$Required)
|
|
|
|
$cleanup = Invoke-Agent99BoundedSsh "rm -rf -- $RemoteStage && test ! -e $RemoteStage" 45
|
|
if ($Required -and -not $cleanup.ok) { throw "remote_source_stage_cleanup_failed" }
|
|
return [bool]$cleanup.ok
|
|
}
|
|
|
|
if ($AgentRoot.TrimEnd("\") -ne $ExpectedAgentRoot) { throw "agent_root_override_forbidden" }
|
|
$SourceRevision = $SourceRevision.ToLowerInvariant()
|
|
if ($SourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
|
if (-not (Test-Agent99SafeIdentity $RunId)) { throw "invalid_run_id" }
|
|
if (-not (Test-Path -LiteralPath $IdentityFile -PathType Leaf)) { throw "agent99_ssh_identity_unavailable" }
|
|
if (-not (Test-Path -LiteralPath $GitPath -PathType Leaf)) { throw "windows_git_unavailable" }
|
|
if (-not (Get-Command ssh.exe -ErrorAction SilentlyContinue)) { throw "windows_openssh_unavailable" }
|
|
if (-not (Get-Command scp.exe -ErrorAction SilentlyContinue)) { throw "windows_openscp_unavailable" }
|
|
|
|
New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-Null
|
|
$evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json"
|
|
if (Test-Path -LiteralPath $evidencePath -PathType Leaf) {
|
|
try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" }
|
|
Assert-Agent99ExistingBrokerEvidence $existing $evidencePath
|
|
$existing | ConvertTo-Json -Compress -Depth 12
|
|
if (-not [bool]$existing.ok) { exit 1 }
|
|
exit 0
|
|
}
|
|
|
|
$sourceRoot = Join-Path $SourceRootBase "host110-backup-source-$RunId"
|
|
$manifestPath = Join-Path $sourceRoot "manifest.json"
|
|
$remoteStage = "/tmp/agent99-host110-backup-runtime-$RunId"
|
|
$startedAt = [DateTime]::UtcNow
|
|
$mutex = [Threading.Mutex]::new($false, $TransportMutexName)
|
|
$acquired = $false
|
|
$remoteStageCreated = $false
|
|
$sourcePackage = $null
|
|
$check = $null
|
|
$apply = $null
|
|
$verify = $null
|
|
$terminalStatus = "failed"
|
|
$errorCode = ""
|
|
$cleanupVerified = $false
|
|
|
|
try {
|
|
try {
|
|
$acquired = [bool]$mutex.WaitOne(30000)
|
|
} catch [Threading.AbandonedMutexException] {
|
|
$acquired = $true
|
|
}
|
|
if (-not $acquired) { throw "host110_backup_runtime_transport_busy" }
|
|
|
|
$sourcePackage = New-Agent99SourcePackage $sourceRoot $manifestPath
|
|
$check = Invoke-Agent99ReadOnlyPreflight $sourcePackage
|
|
|
|
if ($Mode -eq "Apply") {
|
|
$createStage = Invoke-Agent99BoundedSsh "test ! -e $remoteStage && install -d -m 700 $remoteStage" 45
|
|
Assert-Agent99ProcessOk $createStage "remote_source_stage_identity_conflict"
|
|
$remoteStageCreated = $true
|
|
$copyStage = Invoke-Agent99BoundedScp $sourcePackage.localPaths $remoteStage 120
|
|
Assert-Agent99ProcessOk $copyStage "remote_source_stage_copy_failed"
|
|
$stageCheck = Invoke-Agent99BoundedSsh "chmod 700 $remoteStage/host110-backup-runtime-executor.sh $remoteStage/verify-host110-backup-runtime.py && test `$(find $remoteStage -mindepth 1 -maxdepth 1 -type f | wc -l) -eq $ExpectedRemoteStageFileCount" 45
|
|
Assert-Agent99ProcessOk $stageCheck "remote_source_stage_contract_failed"
|
|
|
|
$executorPath = "$remoteStage/host110-backup-runtime-executor.sh"
|
|
$command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage"
|
|
$applyTransport = Invoke-Agent99BoundedSsh $command 330
|
|
if ($applyTransport.ok) {
|
|
$apply = Get-Agent99ExecutorResult $applyTransport "Apply" ([string]$sourcePackage.verifierSha256)
|
|
} else {
|
|
try { $apply = Get-Agent99CommittedFailureResult $applyTransport } catch { $apply = $null }
|
|
if (-not $apply) {
|
|
$apply = Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage
|
|
}
|
|
throw "host110_apply_$([string]$apply.status)"
|
|
}
|
|
$verify = $apply.verifier
|
|
$cleanupVerified = Remove-Agent99RemoteStage $remoteStage $true
|
|
$remoteStageCreated = $false
|
|
$terminalStatus = "verified"
|
|
} elseif ($Mode -eq "Verify") {
|
|
$verify = Invoke-Agent99NoWriteVerifier $sourcePackage
|
|
$terminalStatus = "readback_ok"
|
|
$cleanupVerified = $true
|
|
} else {
|
|
$terminalStatus = "readback_ok"
|
|
$cleanupVerified = $true
|
|
}
|
|
} catch {
|
|
$errorCode = [string]$_.Exception.Message
|
|
} finally {
|
|
if ($remoteStageCreated) {
|
|
try { $cleanupVerified = Remove-Agent99RemoteStage $remoteStage $false } catch {}
|
|
}
|
|
if ($acquired) { try { $mutex.ReleaseMutex() } catch {} }
|
|
$mutex.Dispose()
|
|
}
|
|
|
|
$result = [pscustomobject]@{
|
|
schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3"
|
|
ok = [bool]($terminalStatus -ne "failed")
|
|
status = $terminalStatus
|
|
mode = $Mode
|
|
sourceRevision = $SourceRevision
|
|
sourceHead = if ($sourcePackage) { [string]$sourcePackage.sourceHead } else { "" }
|
|
runId = $RunId
|
|
targetHost = $TargetHost
|
|
sourceTransport = "windows99_gitea_exact_revision_manifest"
|
|
executor = "host110_backup_runtime_executor"
|
|
verifier = "independent_host110_backup_runtime_python_readback"
|
|
decisionProvider = "deterministic_only"
|
|
criticProvider = "deterministic_only"
|
|
agentAction = if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }
|
|
check = $check
|
|
apply = $apply
|
|
verify = $verify
|
|
cleanupVerified = $cleanupVerified
|
|
errorCode = $errorCode
|
|
elapsedSeconds = [math]::Round(([DateTime]::UtcNow - $startedAt).TotalSeconds, 3)
|
|
evidence = $evidencePath
|
|
secretValuesRead = $false
|
|
rawSessionStored = $false
|
|
}
|
|
|
|
try {
|
|
Write-Agent99ImmutableJson $evidencePath $result
|
|
} finally {
|
|
if (Test-Path -LiteralPath $sourceRoot) {
|
|
Remove-Item -LiteralPath $sourceRoot -Recurse -Force -ErrorAction SilentlyContinue
|
|
}
|
|
}
|
|
$result | ConvertTo-Json -Compress -Depth 12
|
|
if (-not $result.ok) { exit 1 }
|