From 2c2dc086b055e2187ac7539f353380bbdbd3de4d Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 19 Jul 2026 02:40:26 +0800 Subject: [PATCH] fix(agent99): fail closed host110 runtime replay --- agent99-host110-backup-runtime-broker.ps1 | 736 +++++++++++++++++- .../AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md | 202 +++++ .../backup/backup-offsite-readiness-gate.sh | 66 +- scripts/backup/common.sh | 85 +- .../backup/gitea-full-backup-restore-drill.sh | 66 +- .../backup/verify-host188-products-archive.py | 39 +- ...agent99-host110-broker-contract-replay.ps1 | 203 ++++- ...t_agent99_host110_backup_runtime_broker.py | 523 ++++++++++++- 8 files changed, 1878 insertions(+), 42 deletions(-) create mode 100644 docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md diff --git a/agent99-host110-backup-runtime-broker.ps1 b/agent99-host110-backup-runtime-broker.ps1 index 83147c373..d01e90dec 100644 --- a/agent99-host110-backup-runtime-broker.ps1 +++ b/agent99-host110-backup-runtime-broker.ps1 @@ -44,6 +44,212 @@ $RuntimeRelativePaths = @( ) $ExecutorRelativePath = "scripts/backup/host110-backup-runtime-executor.sh" $VerifierRelativePath = "scripts/backup/verify-host110-backup-runtime.py" +$TransportReconciliationReadback = @' +#!/usr/bin/env python3 +import argparse +import json +import re +from pathlib import Path + + +def is_string(value): + return type(value) is str + + +def is_boolean(value): + return type(value) is bool + + +def is_integer(value): + return type(value) is int + + +def exact_document(document, contract): + return ( + type(document) is dict + and set(document) == set(contract) + and all(contract[name](document[name]) for name in contract) + ) + + +def read_document(path): + if path.is_symlink(): + return "invalid", None + if not path.exists(): + return "missing", None + if not path.is_file() or path.stat().st_size > 65536: + return "invalid", None + try: + return "present", json.loads(path.read_text(encoding="utf-8")) + except (OSError, UnicodeError, json.JSONDecodeError): + return "invalid", None + + +parser = argparse.ArgumentParser() +parser.add_argument("--status-root", required=True) +parser.add_argument("--source-revision", required=True) +parser.add_argument("--source-head", required=True) +parser.add_argument("--run-id", required=True) +parser.add_argument("--verifier-sha256", required=True) +args = parser.parse_args() + +if not re.fullmatch(r"[0-9a-f]{40}", args.source_revision): + raise SystemExit(64) +if not re.fullmatch(r"[0-9a-f]{40}", args.source_head): + raise SystemExit(64) +if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._-]{0,95}", args.run_id): + raise SystemExit(64) +if not re.fullmatch(r"[0-9a-f]{64}", args.verifier_sha256): + raise SystemExit(64) + +status_root = Path(args.status_root) +if status_root.is_symlink() or not status_root.is_dir(): + raise SystemExit(66) +payload_path = status_root / f"agent99-backup-runtime-deploy-{args.run_id}.json" +terminal_path = status_root / f"agent99-backup-runtime-terminal-{args.run_id}.json" + +payload_contract = { + "schemaVersion": is_string, + "status": is_string, + "sourceRevision": is_string, + "sourceHead": is_string, + "runId": is_string, + "fileCount": is_integer, + "backupScriptFileCount": is_integer, + "backupHealthExporterIncluded": is_boolean, + "rollbackAttempted": is_boolean, + "rollbackPerformed": is_boolean, + "rollbackVerified": is_boolean, + "zeroResidueVerified": is_boolean, + "zeroResidueScope": is_string, + "verifierSha256": is_string, + "executorHost": is_string, + "productionServiceRestarted": is_boolean, + "secretValuesRead": is_boolean, + "writtenAt": is_integer, +} +terminal_contract = { + "schemaVersion": is_string, + "status": is_string, + "ok": is_boolean, + "payloadCommitted": is_boolean, + "payloadReceipt": is_string, + "sourceRevision": is_string, + "sourceHead": is_string, + "runId": is_string, + "fileCount": is_integer, + "backupScriptFileCount": is_integer, + "backupHealthExporterIncluded": is_boolean, + "stageCleanupVerified": is_boolean, + "rollbackPrestateCleanupVerified": is_boolean, + "cleanupVerified": is_boolean, + "independentVerifierVerified": is_boolean, + "zeroResidueScope": is_string, + "deferredSignal": lambda value: value is None or is_string(value), + "terminalExitCode": is_integer, + "verifierSha256": is_string, + "executorHost": is_string, + "productionServiceRestarted": is_boolean, + "secretValuesRead": is_boolean, + "writtenAt": is_integer, +} + +payload_state, payload = read_document(payload_path) +if payload_state == "present": + valid_payload = exact_document(payload, payload_contract) and ( + payload["schemaVersion"] == "agent99_host110_backup_runtime_deploy_receipt_v3" + and payload["status"] == "payload_verified" + and payload["sourceRevision"] == args.source_revision + and payload["sourceHead"] == args.source_head + and payload["runId"] == args.run_id + and payload["fileCount"] == 18 + and payload["backupScriptFileCount"] == 17 + and payload["backupHealthExporterIncluded"] is True + and payload["rollbackAttempted"] is False + and payload["rollbackPerformed"] is False + and payload["rollbackVerified"] is False + and payload["zeroResidueVerified"] is True + and payload["zeroResidueScope"] == "run_owned_destination_temps" + and payload["verifierSha256"] == args.verifier_sha256 + and payload["executorHost"] == "192.168.0.110" + and payload["productionServiceRestarted"] is False + and payload["secretValuesRead"] is False + ) + payload_state = "valid" if valid_payload else "invalid" + +terminal_state, terminal = read_document(terminal_path) +terminal_status = "" +terminal_cleanup_verified = False +terminal_exit_code = -1 +if terminal_state == "present": + valid_terminal = exact_document(terminal, terminal_contract) and ( + terminal["schemaVersion"] == "agent99_host110_backup_runtime_terminal_receipt_v1" + and terminal["status"] in { + "verified", "cleanup_pending", "committed_signal_deferred" + } + and terminal["ok"] is (terminal["status"] == "verified") + and terminal["payloadCommitted"] is True + and terminal["payloadReceipt"] == str(payload_path) + and terminal["sourceRevision"] == args.source_revision + and terminal["sourceHead"] == args.source_head + and terminal["runId"] == args.run_id + and terminal["fileCount"] == 18 + and terminal["backupScriptFileCount"] == 17 + and terminal["backupHealthExporterIncluded"] is True + and terminal["cleanupVerified"] is ( + terminal["stageCleanupVerified"] + and terminal["rollbackPrestateCleanupVerified"] + ) + and terminal["independentVerifierVerified"] is True + and terminal["zeroResidueScope"] + == "run_owned_destination_temps_and_internal_stage_prestate" + and terminal["verifierSha256"] == args.verifier_sha256 + and terminal["executorHost"] == "192.168.0.110" + and terminal["productionServiceRestarted"] is False + and terminal["secretValuesRead"] is False + ) + if valid_terminal and terminal["status"] == "verified": + valid_terminal = ( + terminal["cleanupVerified"] is True + and terminal["deferredSignal"] is None + and terminal["terminalExitCode"] == 0 + ) + elif valid_terminal and terminal["status"] == "cleanup_pending": + valid_terminal = ( + terminal["cleanupVerified"] is False + and terminal["terminalExitCode"] == 75 + ) + elif valid_terminal: + signal_codes = {"INT": 130, "HUP": 129, "TERM": 143} + valid_terminal = ( + terminal["cleanupVerified"] is True + and terminal["deferredSignal"] in signal_codes + and terminal["terminalExitCode"] + == signal_codes[terminal["deferredSignal"]] + ) + terminal_state = "valid" if valid_terminal else "invalid" + if valid_terminal: + terminal_status = terminal["status"] + terminal_cleanup_verified = terminal["cleanupVerified"] + terminal_exit_code = terminal["terminalExitCode"] + +print(json.dumps({ + "schemaVersion": "agent99_host110_backup_transport_receipt_readback_v1", + "ok": True, + "sourceRevision": args.source_revision, + "sourceHead": args.source_head, + "runId": args.run_id, + "payloadReceipt": str(payload_path), + "payloadState": payload_state, + "terminalReceipt": str(terminal_path), + "terminalState": terminal_state, + "terminalStatus": terminal_status, + "terminalCleanupVerified": terminal_cleanup_verified, + "terminalExitCode": terminal_exit_code, + "remoteWritePerformed": False, + "secretValuesRead": False, +}, ensure_ascii=True, sort_keys=True, separators=(",", ":"))) +'@ function Test-Agent99SafeIdentity { param([string]$Value) @@ -245,6 +451,9 @@ function New-Agent99SourcePackage { if (-not (Test-Path -LiteralPath $verifierPath -PathType Leaf)) { throw "verifier_source_file_missing" } $executorDigest = (Get-FileHash -LiteralPath $executorPath -Algorithm SHA256).Hash.ToLowerInvariant() $verifierDigest = (Get-FileHash -LiteralPath $verifierPath -Algorithm SHA256).Hash.ToLowerInvariant() + $reconciliationPath = Join-Path $SourceRoot "host110-backup-transport-reconciliation.py" + $encoding = New-Object System.Text.UTF8Encoding($false) + [IO.File]::WriteAllText($reconciliationPath, $TransportReconciliationReadback, $encoding) $manifest = [ordered]@{ schemaVersion = "agent99_host110_backup_runtime_source_manifest_v1" sourceRevision = $SourceRevision @@ -254,7 +463,6 @@ function New-Agent99SourcePackage { verifierSha256 = $verifierDigest files = $rows } - $encoding = New-Object System.Text.UTF8Encoding($false) [IO.File]::WriteAllText($ManifestPath, ($manifest | ConvertTo-Json -Depth 8 -Compress), $encoding) $localPaths += $executorPath, $verifierPath, $ManifestPath return [pscustomobject]@{ @@ -262,6 +470,7 @@ function New-Agent99SourcePackage { executorSha256 = $executorDigest verifierSha256 = $verifierDigest verifierPath = $verifierPath + reconciliationPath = $reconciliationPath manifestPath = $ManifestPath manifestBase64 = [Convert]::ToBase64String([IO.File]::ReadAllBytes($ManifestPath)) localPaths = $localPaths @@ -272,7 +481,7 @@ function Test-Agent99JsonField { param( [object]$Object, [string]$Name, - [ValidateSet("String", "Boolean", "Integer", "Object", "Null", "NullOrString")] + [ValidateSet("String", "Boolean", "Integer", "Number", "Object", "ObjectOrNull", "Null", "NullOrString")] [string]$Kind ) @@ -284,7 +493,9 @@ function Test-Agent99JsonField { "String" { return [bool]($value -is [string]) } "Boolean" { return [bool]($value -is [bool]) } "Integer" { return [bool]($value -is [int] -or $value -is [long]) } + "Number" { return [bool]($value -is [int] -or $value -is [long] -or $value -is [double] -or $value -is [decimal]) } "Object" { return [bool]($value -is [System.Management.Automation.PSCustomObject]) } + "ObjectOrNull" { return [bool]($null -eq $value -or $value -is [System.Management.Automation.PSCustomObject]) } "Null" { return [bool]($null -eq $value) } "NullOrString" { return [bool]($null -eq $value -or $value -is [string]) } } @@ -301,6 +512,17 @@ function Assert-Agent99JsonFields { } } +function Assert-Agent99ExactJsonFields { + param([object]$Object, [hashtable]$Contract, [string]$ErrorCode) + + Assert-Agent99JsonFields $Object $Contract $ErrorCode + $properties = @($Object.PSObject.Properties) + if ($properties.Count -ne $Contract.Count) { throw $ErrorCode } + foreach ($property in $properties) { + if (-not $Contract.ContainsKey([string]$property.Name)) { throw $ErrorCode } + } +} + function Convert-Agent99ExecutorDocument { param( [object]$Transport, @@ -352,8 +574,17 @@ function Convert-Agent99ExecutorDocument { function Assert-Agent99ApplyResultFields { param([object]$Parsed, [string]$VerifierKind) - Assert-Agent99JsonFields $Parsed @{ + Assert-Agent99ExactJsonFields $Parsed @{ + schemaVersion = "String" + mode = "String" status = "String" + ok = "Boolean" + sourceRevision = "String" + sourceHead = "String" + runId = "String" + fileCount = "Integer" + backupScriptFileCount = "Integer" + backupHealthExporterIncluded = "Boolean" payloadCommitted = "Boolean" rollbackAttempted = "Boolean" rollbackPerformed = "Boolean" @@ -375,7 +606,7 @@ function Assert-Agent99ApplyResultFields { function Assert-Agent99VerifierFields { param([object]$Verifier) - Assert-Agent99JsonFields $Verifier @{ + Assert-Agent99ExactJsonFields $Verifier @{ schemaVersion = "String" ok = "Boolean" sourceRevision = "String" @@ -383,13 +614,17 @@ function Assert-Agent99VerifierFields { fileCount = "Integer" backupScriptFileCount = "Integer" backupHealthExporterIncluded = "Boolean" + destination = "String" + exporterDestination = "String" remoteWritePerformed = "Boolean" + secretValuesRead = "Boolean" selfIdentityVerified = "Boolean" + verifierSha256 = "String" } "host110_apply_verifier_required_field_failed" } function Get-Agent99ExecutorResult { - param([object]$Transport, [string]$ExpectedMode) + param([object]$Transport, [string]$ExpectedMode, [string]$ExpectedVerifierSha256 = "") $parsed = Convert-Agent99ExecutorDocument $Transport $ExpectedMode if (-not [bool]$parsed.ok) { throw "host110_${ExpectedMode}_receipt_contract_failed" } @@ -419,8 +654,13 @@ function Get-Agent99ExecutorResult { [int64]$parsed.verifier.fileCount -ne $ExpectedFileCount -or [int64]$parsed.verifier.backupScriptFileCount -ne 17 -or -not [bool]$parsed.verifier.backupHealthExporterIncluded -or + [string]$parsed.verifier.destination -ne "/backup/scripts" -or + [string]$parsed.verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or [bool]$parsed.verifier.remoteWritePerformed -or - -not [bool]$parsed.verifier.selfIdentityVerified + [bool]$parsed.verifier.secretValuesRead -or + -not [bool]$parsed.verifier.selfIdentityVerified -or + $ExpectedVerifierSha256 -notmatch "^[0-9a-f]{64}$" -or + [string]$parsed.verifier.verifierSha256 -ne $ExpectedVerifierSha256 ) { throw "host110_apply_independent_verifier_contract_failed" } @@ -497,17 +737,7 @@ function Get-Agent99VerifierResult { } catch { throw "host110_verify_receipt_invalid" } - Assert-Agent99JsonFields $parsed @{ - schemaVersion = "String" - ok = "Boolean" - sourceRevision = "String" - runId = "String" - fileCount = "Integer" - backupScriptFileCount = "Integer" - backupHealthExporterIncluded = "Boolean" - remoteWritePerformed = "Boolean" - verifierSha256 = "String" - } "host110_verify_receipt_required_field_failed" + Assert-Agent99VerifierFields $parsed if ( [string]$parsed.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or -not [bool]$parsed.ok -or @@ -516,7 +746,11 @@ function Get-Agent99VerifierResult { [int]$parsed.fileCount -ne $ExpectedFileCount -or [int]$parsed.backupScriptFileCount -ne 17 -or -not [bool]$parsed.backupHealthExporterIncluded -or + [string]$parsed.destination -ne "/backup/scripts" -or + [string]$parsed.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or [bool]$parsed.remoteWritePerformed -or + [bool]$parsed.secretValuesRead -or + -not [bool]$parsed.selfIdentityVerified -or [string]$parsed.verifierSha256 -ne [string]$SourcePackage.verifierSha256 ) { throw "host110_verify_receipt_contract_failed" @@ -524,6 +758,463 @@ function Get-Agent99VerifierResult { return $parsed } +function Assert-Agent99VerifierEvidence { + param([object]$Verifier, [string]$ExpectedVerifierSha256 = "") + + Assert-Agent99VerifierFields $Verifier + if ( + [string]$Verifier.schemaVersion -ne "agent99_host110_backup_runtime_verifier_v1" -or + -not [bool]$Verifier.ok -or + [string]$Verifier.sourceRevision -ne $SourceRevision -or + [string]$Verifier.runId -ne $RunId -or + [int64]$Verifier.fileCount -ne $ExpectedFileCount -or + [int64]$Verifier.backupScriptFileCount -ne 17 -or + -not [bool]$Verifier.backupHealthExporterIncluded -or + [string]$Verifier.destination -ne "/backup/scripts" -or + [string]$Verifier.exporterDestination -ne "/home/wooo/scripts/backup-health-textfile-exporter.py" -or + [bool]$Verifier.remoteWritePerformed -or + [bool]$Verifier.secretValuesRead -or + -not [bool]$Verifier.selfIdentityVerified -or + [string]$Verifier.verifierSha256 -notmatch "^[0-9a-f]{64}$" -or + ($ExpectedVerifierSha256 -and [string]$Verifier.verifierSha256 -ne $ExpectedVerifierSha256) + ) { + throw "host110_existing_verifier_contract_failed" + } +} + +function Assert-Agent99PreflightEvidence { + param([object]$Preflight, [string]$ExpectedSourceHead) + + Assert-Agent99ExactJsonFields $Preflight @{ + schemaVersion = "String" + ok = "Boolean" + sourceRevision = "String" + sourceHead = "String" + fileCount = "Integer" + backupScriptFileCount = "Integer" + backupHealthExporterIncluded = "Boolean" + remoteWritePerformed = "Boolean" + } "host110_existing_preflight_required_field_failed" + if ( + [string]$Preflight.schemaVersion -ne "agent99_host110_backup_runtime_preflight_v1" -or + -not [bool]$Preflight.ok -or + [string]$Preflight.sourceRevision -ne $SourceRevision -or + [string]$Preflight.sourceHead -ne $ExpectedSourceHead -or + [int64]$Preflight.fileCount -ne $ExpectedFileCount -or + [int64]$Preflight.backupScriptFileCount -ne 17 -or + -not [bool]$Preflight.backupHealthExporterIncluded -or + [bool]$Preflight.remoteWritePerformed + ) { + throw "host110_existing_preflight_contract_failed" + } +} + +function Get-Agent99TransportReceiptReadback { + param([object]$Transport, [object]$SourcePackage) + + if (-not $Transport.ok) { throw "host110_transport_receipt_readback_failed" } + try { $parsed = [string]$Transport.stdout | ConvertFrom-Json } catch { + throw "host110_transport_receipt_readback_invalid" + } + Assert-Agent99ExactJsonFields $parsed @{ + schemaVersion = "String" + ok = "Boolean" + sourceRevision = "String" + sourceHead = "String" + runId = "String" + payloadReceipt = "String" + payloadState = "String" + terminalReceipt = "String" + terminalState = "String" + terminalStatus = "String" + terminalCleanupVerified = "Boolean" + terminalExitCode = "Integer" + remoteWritePerformed = "Boolean" + secretValuesRead = "Boolean" + } "host110_transport_receipt_readback_required_field_failed" + if ( + [string]$parsed.schemaVersion -ne "agent99_host110_backup_transport_receipt_readback_v1" -or + -not [bool]$parsed.ok -or + [string]$parsed.sourceRevision -ne $SourceRevision -or + [string]$parsed.sourceHead -ne [string]$SourcePackage.sourceHead -or + [string]$parsed.runId -ne $RunId -or + [string]$parsed.payloadReceipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or + [string]$parsed.payloadState -notin @("missing", "invalid", "valid") -or + [string]$parsed.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or + [string]$parsed.terminalState -notin @("missing", "invalid", "valid") -or + [bool]$parsed.remoteWritePerformed -or + [bool]$parsed.secretValuesRead + ) { + throw "host110_transport_receipt_readback_contract_failed" + } + if ([string]$parsed.terminalState -eq "valid") { + if ( + [string]$parsed.terminalStatus -notin @("verified", "cleanup_pending", "committed_signal_deferred") -or + ([string]$parsed.terminalStatus -eq "verified" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 0)) -or + ([string]$parsed.terminalStatus -eq "cleanup_pending" -and ([bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -ne 75)) -or + ([string]$parsed.terminalStatus -eq "committed_signal_deferred" -and (-not [bool]$parsed.terminalCleanupVerified -or [int64]$parsed.terminalExitCode -notin @(129, 130, 143))) + ) { + throw "host110_transport_terminal_readback_contract_failed" + } + } elseif ( + [string]$parsed.terminalStatus -ne "" -or + [bool]$parsed.terminalCleanupVerified -or + [int64]$parsed.terminalExitCode -ne -1 + ) { + throw "host110_transport_terminal_absence_contract_failed" + } + return $parsed +} + +function New-Agent99TransportLossReconciliation { + param( + [object]$Transport, + [object]$SourcePackage, + [object]$ReceiptReadback, + [object]$VerifierReadback, + [string]$ReceiptReadbackError = "", + [string]$VerifierReadbackError = "" + ) + + $payloadCommitted = [bool]($ReceiptReadback -and [string]$ReceiptReadback.payloadState -eq "valid") + $destinationVerified = [bool]($VerifierReadback -and [bool]$VerifierReadback.ok) + $terminalState = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalState } else { "unavailable" } + $terminalStatus = if ($ReceiptReadback) { [string]$ReceiptReadback.terminalStatus } else { "" } + $status = "committed_unknown" + if ($payloadCommitted -and $destinationVerified -and $terminalState -eq "missing") { + $status = "committed_verified_terminal_missing" + } elseif ($payloadCommitted -and $destinationVerified -and $terminalState -eq "valid") { + $status = switch ($terminalStatus) { + "verified" { "committed_verified_transport_lost" } + "cleanup_pending" { "committed_cleanup_pending_transport_lost" } + "committed_signal_deferred" { "committed_signal_deferred_transport_lost" } + default { "committed_unknown" } + } + } + return [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_transport_reconciliation_v1" + ok = $false + status = $status + sourceRevision = $SourceRevision + sourceHead = [string]$SourcePackage.sourceHead + runId = $RunId + payloadCommitted = $payloadCommitted + payloadReceiptState = if ($ReceiptReadback) { [string]$ReceiptReadback.payloadState } else { "unavailable" } + terminalReceiptState = $terminalState + terminalStatus = $terminalStatus + destinationVerified = $destinationVerified + retryApplyAllowed = $false + transportReason = if ($Transport -and $Transport.PSObject.Properties["reason"]) { [string]$Transport.reason } else { "unknown" } + transportExitCode = if ($Transport -and $Transport.PSObject.Properties["exitCode"]) { [int]$Transport.exitCode } else { -1 } + receiptReadback = $ReceiptReadback + verifier = $VerifierReadback + receiptReadbackError = $ReceiptReadbackError + verifierReadbackError = $VerifierReadbackError + remoteWritePerformed = $false + secretValuesRead = $false + } +} + +function Invoke-Agent99TransportLossReconciliation { + param([object]$Transport, [object]$SourcePackage) + + $receiptReadback = $null + $verifierReadback = $null + $receiptReadbackError = "" + $verifierReadbackError = "" + try { + $command = "python3 - --status-root /backup/status --source-revision $SourceRevision --source-head $($SourcePackage.sourceHead) --run-id $RunId --verifier-sha256 $($SourcePackage.verifierSha256)" + $readbackTransport = Invoke-Agent99BoundedSsh $command 60 ([string]$SourcePackage.reconciliationPath) + $receiptReadback = Get-Agent99TransportReceiptReadback $readbackTransport $SourcePackage + } catch { + $receiptReadbackError = [string]$_.Exception.Message + } + try { + $verifierReadback = Invoke-Agent99NoWriteVerifier $SourcePackage + } catch { + $verifierReadbackError = [string]$_.Exception.Message + } + return New-Agent99TransportLossReconciliation ` + $Transport $SourcePackage $receiptReadback $verifierReadback ` + $receiptReadbackError $verifierReadbackError +} + +function Assert-Agent99ExecutorEvidence { + param([object]$Executor, [string]$ExpectedSourceHead) + + Assert-Agent99ApplyResultFields $Executor $(if ([bool]$Executor.ok) { "Object" } else { "Null" }) + if ( + [string]$Executor.schemaVersion -ne "agent99_host110_backup_runtime_executor_v1" -or + [string]$Executor.mode -ne "apply" -or + [string]$Executor.sourceRevision -ne $SourceRevision -or + [string]$Executor.sourceHead -ne $ExpectedSourceHead -or + [string]$Executor.runId -ne $RunId -or + [int64]$Executor.fileCount -ne $ExpectedFileCount -or + [int64]$Executor.backupScriptFileCount -ne 17 -or + -not [bool]$Executor.backupHealthExporterIncluded -or + -not [bool]$Executor.payloadCommitted -or + [bool]$Executor.rollbackAttempted -or + [bool]$Executor.rollbackPerformed -or + [bool]$Executor.rollbackVerified -or + -not [bool]$Executor.zeroResidueVerified -or + [string]$Executor.zeroResidueScope -ne "run_owned_destination_temps" -or + [string]$Executor.receipt -ne "/backup/status/agent99-backup-runtime-deploy-$RunId.json" -or + [string]$Executor.terminalReceipt -ne "/backup/status/agent99-backup-runtime-terminal-$RunId.json" -or + -not [bool]$Executor.independentVerifierVerified + ) { + throw "host110_existing_apply_contract_failed" + } + if ([bool]$Executor.ok) { + Assert-Agent99VerifierEvidence $Executor.verifier + if ( + [string]$Executor.status -ne "verified" -or + -not [bool]$Executor.stageCleanupVerified -or + -not [bool]$Executor.rollbackPrestateCleanupVerified -or + -not [bool]$Executor.terminalReceiptPublished -or + [int64]$Executor.terminalExitCode -ne 0 -or + $null -ne $Executor.deferredSignal + ) { + throw "host110_existing_apply_success_contract_failed" + } + return + } + + if ($null -ne $Executor.verifier) { throw "host110_existing_apply_failure_verifier_present" } + $signalExitCodes = @{ INT = 130; HUP = 129; TERM = 143 } + if ([string]$Executor.status -eq "cleanup_pending") { + if ( + ([bool]$Executor.stageCleanupVerified -and [bool]$Executor.rollbackPrestateCleanupVerified) -or + -not [bool]$Executor.terminalReceiptPublished -or + [int64]$Executor.terminalExitCode -ne 75 + ) { throw "host110_existing_cleanup_pending_contract_failed" } + } elseif ([string]$Executor.status -eq "committed_signal_deferred") { + $signalName = [string]$Executor.deferredSignal + if ( + -not [bool]$Executor.stageCleanupVerified -or + -not [bool]$Executor.rollbackPrestateCleanupVerified -or + -not [bool]$Executor.terminalReceiptPublished -or + -not $signalExitCodes.ContainsKey($signalName) -or + [int64]$Executor.terminalExitCode -ne [int]$signalExitCodes[$signalName] + ) { throw "host110_existing_committed_signal_contract_failed" } + } elseif ([string]$Executor.status -eq "terminal_receipt_failed") { + if ( + [bool]$Executor.terminalReceiptPublished -or + [int64]$Executor.terminalExitCode -ne 76 + ) { throw "host110_existing_terminal_receipt_failure_contract_failed" } + } else { + throw "host110_existing_apply_failure_status_invalid" + } +} + +function Assert-Agent99ReconciliationEvidence { + param([object]$Reconciliation, [string]$ExpectedSourceHead) + + Assert-Agent99ExactJsonFields $Reconciliation @{ + schemaVersion = "String" + ok = "Boolean" + status = "String" + sourceRevision = "String" + sourceHead = "String" + runId = "String" + payloadCommitted = "Boolean" + payloadReceiptState = "String" + terminalReceiptState = "String" + terminalStatus = "String" + destinationVerified = "Boolean" + retryApplyAllowed = "Boolean" + transportReason = "String" + transportExitCode = "Integer" + receiptReadback = "ObjectOrNull" + verifier = "ObjectOrNull" + receiptReadbackError = "String" + verifierReadbackError = "String" + remoteWritePerformed = "Boolean" + secretValuesRead = "Boolean" + } "host110_existing_reconciliation_required_field_failed" + if ( + [string]$Reconciliation.schemaVersion -ne "agent99_host110_backup_transport_reconciliation_v1" -or + [bool]$Reconciliation.ok -or + [string]$Reconciliation.sourceRevision -ne $SourceRevision -or + [string]$Reconciliation.sourceHead -ne $ExpectedSourceHead -or + [string]$Reconciliation.runId -ne $RunId -or + [string]$Reconciliation.payloadReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or + [string]$Reconciliation.terminalReceiptState -notin @("missing", "invalid", "valid", "unavailable") -or + [bool]$Reconciliation.retryApplyAllowed -or + [bool]$Reconciliation.remoteWritePerformed -or + [bool]$Reconciliation.secretValuesRead + ) { + throw "host110_existing_reconciliation_contract_failed" + } + if ($Reconciliation.receiptReadback) { + if ($Reconciliation.receiptReadbackError) { throw "host110_existing_reconciliation_readback_error_conflict" } + $sourcePackage = [pscustomobject]@{ sourceHead = $ExpectedSourceHead } + $transport = [pscustomobject]@{ + ok = $true + stdout = $Reconciliation.receiptReadback | ConvertTo-Json -Compress -Depth 8 + } + Get-Agent99TransportReceiptReadback $transport $sourcePackage | Out-Null + } elseif (-not $Reconciliation.receiptReadbackError) { + throw "host110_existing_reconciliation_readback_error_missing" + } + if ($Reconciliation.verifier) { + if ($Reconciliation.verifierReadbackError) { throw "host110_existing_reconciliation_verifier_error_conflict" } + Assert-Agent99VerifierEvidence $Reconciliation.verifier + } elseif (-not $Reconciliation.verifierReadbackError) { + throw "host110_existing_reconciliation_verifier_error_missing" + } + if ([bool]$Reconciliation.payloadCommitted -ne [bool]($Reconciliation.payloadReceiptState -eq "valid")) { + throw "host110_existing_reconciliation_payload_truth_mismatch" + } + if ($Reconciliation.payloadCommitted -and -not $Reconciliation.receiptReadback) { + throw "host110_existing_reconciliation_payload_readback_missing" + } + if ([bool]$Reconciliation.destinationVerified -ne [bool]($null -ne $Reconciliation.verifier)) { + throw "host110_existing_reconciliation_destination_truth_mismatch" + } + $status = [string]$Reconciliation.status + $knownStatuses = @( + "committed_verified_terminal_missing", + "committed_verified_transport_lost", + "committed_cleanup_pending_transport_lost", + "committed_signal_deferred_transport_lost", + "committed_unknown" + ) + if ($status -notin $knownStatuses) { throw "host110_existing_reconciliation_status_invalid" } + if ($status -ne "committed_unknown") { + if (-not $Reconciliation.payloadCommitted -or -not $Reconciliation.destinationVerified) { + throw "host110_existing_reconciliation_verified_state_invalid" + } + } + if ($status -eq "committed_verified_terminal_missing" -and $Reconciliation.terminalReceiptState -ne "missing") { + throw "host110_existing_reconciliation_terminal_missing_mismatch" + } + $terminalStatusByReconciliation = @{ + committed_verified_transport_lost = "verified" + committed_cleanup_pending_transport_lost = "cleanup_pending" + committed_signal_deferred_transport_lost = "committed_signal_deferred" + } + if ($terminalStatusByReconciliation.ContainsKey($status)) { + if ( + $Reconciliation.terminalReceiptState -ne "valid" -or + [string]$Reconciliation.terminalStatus -ne [string]$terminalStatusByReconciliation[$status] + ) { throw "host110_existing_reconciliation_terminal_state_mismatch" } + } +} + +function Assert-Agent99ExistingBrokerEvidence { + param([object]$Existing, [string]$ExpectedEvidencePath) + + Assert-Agent99ExactJsonFields $Existing @{ + schemaVersion = "String" + ok = "Boolean" + status = "String" + mode = "String" + sourceRevision = "String" + sourceHead = "String" + runId = "String" + targetHost = "String" + sourceTransport = "String" + executor = "String" + verifier = "String" + decisionProvider = "String" + criticProvider = "String" + agentAction = "String" + check = "ObjectOrNull" + apply = "ObjectOrNull" + verify = "ObjectOrNull" + cleanupVerified = "Boolean" + errorCode = "String" + elapsedSeconds = "Number" + evidence = "String" + secretValuesRead = "Boolean" + rawSessionStored = "Boolean" + } "existing_broker_evidence_required_field_failed" + if ( + [string]$Existing.mode -ne $Mode -or + [string]$Existing.sourceRevision -ne $SourceRevision -or + [string]$Existing.runId -ne $RunId -or + [string]$Existing.evidence -ne $ExpectedEvidencePath + ) { + throw "run_identity_conflict" + } + if ( + [string]$Existing.schemaVersion -ne "agent99_host110_backup_runtime_broker_receipt_v3" -or + [string]$Existing.targetHost -ne $TargetHost -or + [string]$Existing.sourceTransport -ne "windows99_gitea_exact_revision_manifest" -or + [string]$Existing.executor -ne "host110_backup_runtime_executor" -or + [string]$Existing.verifier -ne "independent_host110_backup_runtime_python_readback" -or + [string]$Existing.decisionProvider -ne "deterministic_only" -or + [string]$Existing.criticProvider -ne "deterministic_only" -or + [string]$Existing.agentAction -ne $(if ($Mode -eq "Apply") { "controlled_apply" } else { "readback" }) -or + ([string]$Existing.sourceHead -ne "" -and [string]$Existing.sourceHead -notmatch "^[0-9a-f]{40}$") -or + [double]$Existing.elapsedSeconds -lt 0 -or + [bool]$Existing.secretValuesRead -or + [bool]$Existing.rawSessionStored + ) { + throw "existing_broker_evidence_contract_failed" + } + if ($Existing.check) { Assert-Agent99PreflightEvidence $Existing.check ([string]$Existing.sourceHead) } + if ($Existing.verify) { Assert-Agent99VerifierEvidence $Existing.verify } + if ($Existing.apply) { + if ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_runtime_executor_v1") { + Assert-Agent99ExecutorEvidence $Existing.apply ([string]$Existing.sourceHead) + } elseif ([string]$Existing.apply.schemaVersion -eq "agent99_host110_backup_transport_reconciliation_v1") { + Assert-Agent99ReconciliationEvidence $Existing.apply ([string]$Existing.sourceHead) + } else { + throw "existing_broker_apply_schema_invalid" + } + } + if (($Existing.apply -or $Existing.verify) -and -not $Existing.check) { + throw "existing_broker_preflight_evidence_missing" + } + if ($Mode -ne "Apply" -and $Existing.apply) { throw "existing_broker_apply_mode_conflict" } + if ($Mode -eq "Check" -and $Existing.verify) { throw "existing_broker_verify_mode_conflict" } + + if ([bool]$Existing.ok) { + if ([string]$Existing.errorCode -ne "" -or -not [bool]$Existing.cleanupVerified) { + throw "existing_broker_success_terminal_invalid" + } + if ($Mode -eq "Apply") { + if ( + [string]$Existing.status -ne "verified" -or + -not $Existing.check -or + -not $Existing.apply -or + -not [bool]$Existing.apply.ok -or + -not $Existing.verify -or + [string]$Existing.apply.verifier.verifierSha256 -ne [string]$Existing.verify.verifierSha256 + ) { throw "existing_broker_apply_success_invalid" } + } elseif ($Mode -eq "Verify") { + if ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or -not $Existing.verify) { + throw "existing_broker_verify_success_invalid" + } + } elseif ([string]$Existing.status -ne "readback_ok" -or -not $Existing.check -or $Existing.apply -or $Existing.verify) { + throw "existing_broker_check_success_invalid" + } + return + } + + if ([string]$Existing.status -ne "failed" -or [string]::IsNullOrWhiteSpace([string]$Existing.errorCode)) { + throw "existing_broker_failure_terminal_invalid" + } + if ($Existing.verify -and [string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed") { + throw "existing_broker_failure_verifier_conflict" + } + if ([string]$Existing.errorCode -like "host110_apply_*" -and -not $Existing.apply) { + throw "existing_broker_apply_failure_evidence_missing" + } + if ($Existing.apply) { + if ([bool]$Existing.apply.ok) { + if ( + [string]$Existing.errorCode -ne "remote_source_stage_cleanup_failed" -or + -not $Existing.verify -or + [bool]$Existing.cleanupVerified + ) { throw "existing_broker_postcommit_cleanup_failure_conflict" } + } elseif ([string]$Existing.errorCode -ne "host110_apply_$([string]$Existing.apply.status)") { + throw "existing_broker_apply_failure_conflict" + } + } +} + function Invoke-Agent99ReadOnlyPreflight { param([object]$SourcePackage) @@ -573,9 +1264,7 @@ New-Item -ItemType Directory -Force -Path $EvidenceRoot, $SourceRootBase | Out-N $evidencePath = Join-Path $EvidenceRoot "agent99-host110-backup-runtime-$RunId.json" if (Test-Path -LiteralPath $evidencePath -PathType Leaf) { try { $existing = Get-Content -LiteralPath $evidencePath -Raw | ConvertFrom-Json } catch { throw "run_identity_evidence_invalid" } - if ([string]$existing.sourceRevision -ne $SourceRevision -or [string]$existing.mode -ne $Mode) { - throw "run_identity_conflict" - } + Assert-Agent99ExistingBrokerEvidence $existing $evidencePath $existing | ConvertTo-Json -Compress -Depth 12 if (-not [bool]$existing.ok) { exit 1 } exit 0 @@ -620,9 +1309,12 @@ try { $command = "timeout --signal=TERM --kill-after=10s 300s $executorPath --mode apply --source-revision $SourceRevision --run-id $RunId --source-stage $remoteStage" $applyTransport = Invoke-Agent99BoundedSsh $command 330 if ($applyTransport.ok) { - $apply = Get-Agent99ExecutorResult $applyTransport "Apply" + $apply = Get-Agent99ExecutorResult $applyTransport "Apply" ([string]$sourcePackage.verifierSha256) } else { - $apply = Get-Agent99CommittedFailureResult $applyTransport + try { $apply = Get-Agent99CommittedFailureResult $applyTransport } catch { $apply = $null } + if (-not $apply) { + $apply = Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage + } throw "host110_apply_$([string]$apply.status)" } $verify = $apply.verifier diff --git a/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md b/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md new file mode 100644 index 000000000..b74a1b58c --- /dev/null +++ b/docs/proposals/AWOOOI-AG99-H110-BACKUP-RUNTIME-V11.md @@ -0,0 +1,202 @@ +# AWOOOI Agent99 Host110 Backup Runtime V11 + +Status: proposed. Central review and explicit owner approval are required before +any production apply. This candidate must not be pushed to `main`, sent through +CD, or applied to Windows99 or Host110 while it remains proposed. + +This artifact supersedes V10. Every V10 identity below is void for approval and +remains audit evidence only: + +- branch/ref: `codex/host110-backup-runtime-v10-20260718` +- candidate/revision: `d1797a1c8766722ae00ac14e20fed00f4e3f3a5b` +- artifact SHA-256: `eb48661bbecc4e169111879bb18dfab63be29878d5f5c3651d9917f81f195d28` +- exact diff SHA-256: `f7efb98d5f91fa6899713b692c8bb015fcb6ae6dba4291cc6ae429290072964e` +- every other V10 proposal, content, revision, review, and derived hash + +No V10 hash, ref, review, receipt, or test grants production execution +authority. + +## Exact Scope + +- Agent99 runtime bundle: 19 files, guarded by one producer/consumer parity + test. +- Host110 backup payload: 18 ordered files, comprising 17 backup scripts and + `backup-health-textfile-exporter.py`. +- Host110 remote stage: 21 unique basenames, comprising the ordered 18-file + payload, fixed executor, independent verifier, and `manifest.json`. +- Fixed target: `wooo@192.168.0.110`, dispatched only by the Windows99 Agent99 + typed broker from an exact Gitea revision and digest-bound manifest. +- V11 changes the Windows99 broker, three shell shared-lock entrypoints, the + Host188 archive verifier's shared-lock acquisition, and their no-write replay + tests. +- `scripts/backup/backup-gitea.sh` has zero diff from the production base. No + Gitea primary stop/start or backup-container creation is part of this apply. +- The restore drill and offsite gate gain only canonical shared-lock behavior. + Applying this candidate replaces those scripts but does not invoke a drill, + run a backup, or start an offsite sync. +- Prior proposal documents remain tracked audit artifacts and are not runtime + authority. + +## V10 Defects Closed + +### Persisted Broker Evidence Is Exact And Fail-Closed + +Before reusing a run-bound broker receipt, the broker now validates exact key +sets, required native JSON types, schema version, run ID, source revision, mode, +target, evidence path binding, status, error, cleanup, and nested Apply/Verify +contracts. Missing, extra, wrong-type, wrong-mode, wrong-run, conflicting, or +malformed evidence fails closed before any replay decision. + +A non-empty string such as `"false"` can no longer be cast to a truthy Boolean. +An existing failure remains a failure and cannot be upgraded to success merely +because it is parseable or shares a source revision. + +### Mandatory Post-Commit Transport Reconciliation + +If Apply loses SSH transport or times out, the broker performs a read-only, +exact-RunId receipt readback under `/backup/status` and an independent +destination verifier readback. It never blindly retries Apply and never +collapses a post-commit state into a generic precommit failure. + +The reconciliation states are explicit and non-success: + +- `committed_verified_terminal_missing` +- `committed_verified_transport_lost` +- `committed_cleanup_pending_transport_lost` +- `committed_signal_deferred_transport_lost` +- `committed_unknown` + +Every state records `retryApplyAllowed=false`. Receipt schema, native types, +source/run/head identity, payload paths and counts, verifier identity, cleanup, +signal, and terminal semantics are validated before classification. The +readback performs no remote write and reads no secret value. + +### Canonical Shared Lock And Nested Re-entry + +Production backup entrypoints canonicalize absolute, relative, and symlink +invocation paths before deciding whether the Host110 runtime lock applies. +Unresolvable production identity fails closed. + +For nested calls, an inherited FD 197 is accepted only when its canonical +target is the exact runtime lock. Linux uses `/proc/$$/fd/197`; the bounded +non-Linux fallback resolves the descriptor path with `F_GETPATH`. The entrypoint +then requests the shared lock on the same descriptor. It does not close and +reopen a valid inherited descriptor, so there is no release/reacquire gap. + +If FD 197 is absent, the entrypoint opens the lock and acquires the shared side. +If FD 197 exists but points elsewhere or cannot be resolved, execution fails +closed. `BACKUP_RUNTIME_SHARED_LOCK_HELD` and all other external environment +values grant no lock authority. + +The Host188 archive verifier follows the same contract: exact inherited FD 197 +is duplicated from the same open file description, wrong identity fails +closed, and a forged environment value cannot bypass an active exclusive lock. + +## Apply Effects + +- Windows99 Agent99 runtime promotion writes the 19 runtime files plus bounded + manifest and receipt state. +- Host110 Apply replaces the ordered 18 payload paths through one exclusive, + digest-bound filesystem transaction and writes payload/terminal receipts + below `/backup/status`. +- The deployed common library, restore drill, offsite gate, and Host188 archive + verifier gain the shared-lock behavior described above. Apply does not invoke + their backup, drill, archive, or sync behavior. +- Apply does not stop/start Gitea, create a backup container, call Google Drive, + prune snapshots, restart a VM or service, change a database/network/firewall, + or read/change a credential or secret. + +## Reproducible Validation + +Expanded focused and compatibility suite: + +```sh +pytest -q \ + scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \ + scripts/backup/tests/test_verify_offsite_full_sync_runtime.py \ + scripts/backup/tests/test_verify_host188_products_archive.py \ + scripts/backup/tests/test_backup_host188_products_contract.py \ + scripts/ops/tests/test_agent99_signoz_metadata_executor_runtime_entrypoint.py \ + scripts/ops/tests/test_db_bounded_executor_runtime_entrypoint.py \ + scripts/reboot-recovery/tests/test_agent99_live_preflight_decision.py \ + scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py \ + scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +``` + +Current result: `120 passed, 4 skipped`. + +The shared-lock subset includes real `fcntl` contention for absolute, +relative, symlink, forged-environment, inherited-exact-FD, inherited-wrong-FD, +and Host188 archive paths. Current result: `18 passed, 31 deselected`. + +Static, syntax, and repository checks: + +```sh +ruff check \ + scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py \ + scripts/backup/verify-host188-products-archive.py +bash -n \ + scripts/backup/common.sh \ + scripts/backup/gitea-full-backup-restore-drill.sh \ + scripts/backup/backup-offsite-readiness-gate.sh \ + scripts/backup/host110-backup-runtime-executor.sh +git diff --check +PATH=/Users/ogt/.pyenv/shims:$PATH bash scripts/ops/ansible-validate.sh +``` + +`ansible-validate.sh` parses +`ops/reboot-recovery/full-stack-backup-baseline.yml` as YAML metadata. It runs +`ansible-playbook --syntax-check` only for actual playbooks. Current checks are +green; `ansible-lint` is not installed and is reported as skipped. + +Windows99 no-write PowerShell validation consists of: + +1. parsing the complete candidate broker with Windows PowerShell; +2. loading the candidate broker's actual function AST in the contract replay; +3. replaying valid success, committed non-success, existing evidence rejection, + terminal-missing reconciliation, and committed-unknown cases. + +Current readback is parser `errorCount=0`, plus 38 existing-evidence +rejections, 38 missing-field rejections, 6 type rejections, +`terminalMissingReconciled=true`, `committedUnknownPreserved=true`, +`remoteWritePerformed=false`, and `secretValuesRead=false`. + +## Exact Identity And Diff Serialization + +The central-review intake must bind all of the following independently: + +- live Gitea feature ref `codex/host110-backup-runtime-v11-20260718`; +- exact candidate Git SHA; +- this tracked artifact path and exact artifact-bytes SHA-256; +- exact binary diff SHA-256, path count, and insertion/deletion counts; +- the 19/18/21 count contracts; +- `scripts/backup/backup-gitea.sh` zero diff; +- the Windows99 no-write replay result. + +Exact diff bytes are serialized with: + +```sh +git diff --binary --full-index \ + e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3.. | + shasum -a 256 +``` + +No alternative diff serialization or untracked artifact bytes may be used for +approval identity. + +## Rollback + +- Source rollback reverts the exact V11 diff to + `e7e3bcf8a5a8a19c1253d73f51d9dca248af0aa3` without force-pushing. +- Agent99 rollback restores prior runtime files and manifest and verifies prior + hashes independently. +- Before payload commit, Host110 rollback restores prior payload content, + SHA-256, uid, gid, and mode under the exclusive lock and proves no run-owned + destination temporary remains. +- After payload commit, transport loss cannot trigger a blind Apply retry. + Exact receipts and independent destination readback determine whether the + state is committed or unknown; every uncertain state remains non-success. +- Source rollback restores prior lock behavior. It does not execute a backup, + restore drill, offsite sync, container lifecycle, or service restart. +- Any identity, schema, type, receipt, reconciliation, lock, cleanup, or + residue mismatch fails closed and cannot produce broker `verified`. diff --git a/scripts/backup/backup-offsite-readiness-gate.sh b/scripts/backup/backup-offsite-readiness-gate.sh index bec98e8ed..1733bceae 100755 --- a/scripts/backup/backup-offsite-readiness-gate.sh +++ b/scripts/backup/backup-offsite-readiness-gate.sh @@ -9,14 +9,74 @@ set -euo pipefail -if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then +resolve_backup_runtime_script_path() { + local raw_path="$1" candidate="" resolved="" resolved_directory="" + [ -n "${raw_path}" ] || return 1 + case "${raw_path}" in /*) candidate="${raw_path}" ;; *) candidate="${PWD}/${raw_path}" ;; esac + if command -v realpath >/dev/null 2>&1; then + resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi + fi + if command -v readlink >/dev/null 2>&1; then + resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi + fi + [ ! -L "${candidate}" ] || return 1 + resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1 + [ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1 + printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")" +} + +backup_runtime_inherited_fd_status() { + local expected_path="$1" fd_path="" fd_target="" + if [ -d "/proc/$$/fd" ]; then + fd_path="/proc/$$/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + fd_target="$(resolve_backup_runtime_script_path "${fd_path}")" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 + return 0 + fi + fd_path="/dev/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + command -v python3 >/dev/null 2>&1 || return 2 + fd_target="$(python3 - <<'PY' +import fcntl +import os + +try: + target = os.readlink("/dev/fd/197") +except OSError: + value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024) + target = os.fsdecode(value.split(b"\0", 1)[0]) +print(os.path.realpath(target)) +PY + )" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 +} + +runtime_script_path="$(resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}")" || { + echo "backup runtime gate unavailable: script identity unresolved" >&2 + exit 69 +} +if [[ "${runtime_script_path}" == /backup/scripts/* ]]; then command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; } [ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; } (umask 077; : >> /tmp/agent99-host110-backup-runtime.lock) [ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; } - exec 197>>/tmp/agent99-host110-backup-runtime.lock + runtime_lock_path="$(resolve_backup_runtime_script_path /tmp/agent99-host110-backup-runtime.lock)" \ + || { echo "backup runtime gate unavailable: lock identity unresolved" >&2; exit 69; } + if backup_runtime_inherited_fd_status "${runtime_lock_path}"; then + : + else + inherited_fd_status=$? + if [ "${inherited_fd_status}" -eq 1 ]; then + exec 197>>/tmp/agent99-host110-backup-runtime.lock + else + echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2 + exit 69 + fi + fi flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; } - export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 fi BACKUP_BASE="${BACKUP_BASE:-/backup}" diff --git a/scripts/backup/common.sh b/scripts/backup/common.sh index 955d03498..f23a34e6b 100644 --- a/scripts/backup/common.sh +++ b/scripts/backup/common.sh @@ -3,14 +3,78 @@ # Production backup entrypoints share this stable gate. Agent99 takes the # exclusive side while promoting a complete, digest-bound runtime bundle. +resolve_backup_runtime_source_path() { + local raw_path="$1" + local candidate="" + local resolved="" + local resolved_directory="" + [ -n "${raw_path}" ] || return 1 + case "${raw_path}" in + /*) candidate="${raw_path}" ;; + *) candidate="${PWD}/${raw_path}" ;; + esac + if command -v realpath >/dev/null 2>&1; then + resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then + printf '%s\n' "${resolved}" + return 0 + fi + fi + if command -v readlink >/dev/null 2>&1; then + resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then + printf '%s\n' "${resolved}" + return 0 + fi + fi + [ ! -L "${candidate}" ] || return 1 + resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1 + [ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1 + printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")" +} + +backup_runtime_inherited_fd_status() { + local expected_path="$1" + local fd_path="" + local fd_target="" + if [ -d "/proc/$$/fd" ]; then + fd_path="/proc/$$/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + fd_target="$(resolve_backup_runtime_source_path "${fd_path}")" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 + return 0 + fi + fd_path="/dev/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + command -v python3 >/dev/null 2>&1 || return 2 + fd_target="$(python3 - <<'PY' +import fcntl +import os + +try: + target = os.readlink("/dev/fd/197") +except OSError: + value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024) + target = os.fsdecode(value.split(b"\0", 1)[0]) +print(os.path.realpath(target)) +PY + )" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 +} + acquire_backup_runtime_shared_lock() { - local caller_path="${BASH_SOURCE[1]:-${BASH_SOURCE[0]}}" + local caller_path="" local lock_path="/tmp/agent99-host110-backup-runtime.lock" + local lock_canonical="" + local inherited_fd_status=0 + caller_path="$(resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}")" || { + echo "backup runtime gate unavailable: caller identity unresolved" >&2 + return 69 + } case "${caller_path}" in /backup/scripts/*) ;; *) return 0 ;; esac - [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0 command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable: flock missing" >&2 return 69 @@ -21,12 +85,25 @@ acquire_backup_runtime_shared_lock() { } (umask 077; : >> "${lock_path}") || return 69 [ -f "${lock_path}" ] && [ -O "${lock_path}" ] || return 69 - exec 197>>"${lock_path}" + lock_canonical="$(resolve_backup_runtime_source_path "${lock_path}")" || { + echo "backup runtime gate unavailable: lock identity unresolved" >&2 + return 69 + } + if backup_runtime_inherited_fd_status "${lock_canonical}"; then + : + else + inherited_fd_status=$? + if [ "${inherited_fd_status}" -eq 1 ]; then + exec 197>>"${lock_path}" + else + echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2 + return 69 + fi + fi flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2 return 75 } - export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 } acquire_backup_runtime_shared_lock || { diff --git a/scripts/backup/gitea-full-backup-restore-drill.sh b/scripts/backup/gitea-full-backup-restore-drill.sh index 13415e1c6..730570661 100755 --- a/scripts/backup/gitea-full-backup-restore-drill.sh +++ b/scripts/backup/gitea-full-backup-restore-drill.sh @@ -5,14 +5,74 @@ set -euo pipefail -if [[ "${BASH_SOURCE[0]}" == /backup/scripts/* ]] && [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]; then +resolve_backup_runtime_script_path() { + local raw_path="$1" candidate="" resolved="" resolved_directory="" + [ -n "${raw_path}" ] || return 1 + case "${raw_path}" in /*) candidate="${raw_path}" ;; *) candidate="${PWD}/${raw_path}" ;; esac + if command -v realpath >/dev/null 2>&1; then + resolved="$(realpath -e -- "${candidate}" 2>/dev/null || realpath -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi + fi + if command -v readlink >/dev/null 2>&1; then + resolved="$(readlink -f -- "${candidate}" 2>/dev/null)" || resolved="" + if [ -n "${resolved}" ]; then printf '%s\n' "${resolved}"; return 0; fi + fi + [ ! -L "${candidate}" ] || return 1 + resolved_directory="$(cd -P -- "$(dirname -- "${candidate}")" 2>/dev/null && pwd -P)" || return 1 + [ -e "${resolved_directory}/$(basename -- "${candidate}")" ] || return 1 + printf '%s/%s\n' "${resolved_directory}" "$(basename -- "${candidate}")" +} + +backup_runtime_inherited_fd_status() { + local expected_path="$1" fd_path="" fd_target="" + if [ -d "/proc/$$/fd" ]; then + fd_path="/proc/$$/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + fd_target="$(resolve_backup_runtime_script_path "${fd_path}")" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 + return 0 + fi + fd_path="/dev/fd/197" + [ -e "${fd_path}" ] || [ -L "${fd_path}" ] || return 1 + command -v python3 >/dev/null 2>&1 || return 2 + fd_target="$(python3 - <<'PY' +import fcntl +import os + +try: + target = os.readlink("/dev/fd/197") +except OSError: + value = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024) + target = os.fsdecode(value.split(b"\0", 1)[0]) +print(os.path.realpath(target)) +PY + )" || return 2 + [ "${fd_target}" = "${expected_path}" ] || return 2 +} + +runtime_script_path="$(resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}")" || { + echo "backup runtime gate unavailable: script identity unresolved" >&2 + exit 69 +} +if [[ "${runtime_script_path}" == /backup/scripts/* ]]; then command -v flock >/dev/null 2>&1 || { echo "backup runtime gate unavailable" >&2; exit 69; } [ ! -L /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate unsafe" >&2; exit 69; } (umask 077; : >> /tmp/agent99-host110-backup-runtime.lock) [ -f /tmp/agent99-host110-backup-runtime.lock ] && [ -O /tmp/agent99-host110-backup-runtime.lock ] || { echo "backup runtime gate owner invalid" >&2; exit 69; } - exec 197>>/tmp/agent99-host110-backup-runtime.lock + runtime_lock_path="$(resolve_backup_runtime_script_path /tmp/agent99-host110-backup-runtime.lock)" \ + || { echo "backup runtime gate unavailable: lock identity unresolved" >&2; exit 69; } + if backup_runtime_inherited_fd_status "${runtime_lock_path}"; then + : + else + inherited_fd_status=$? + if [ "${inherited_fd_status}" -eq 1 ]; then + exec 197>>/tmp/agent99-host110-backup-runtime.lock + else + echo "backup runtime gate unsafe: inherited fd 197 identity mismatch" >&2 + exit 69 + fi + fi flock -s -w 60 197 || { echo "backup runtime deployment is active; retry later" >&2; exit 75; } - export BACKUP_RUNTIME_SHARED_LOCK_HELD=1 fi DUMP_ZIP="${AIOPS_GITEA_FULL_BACKUP_DRILL_DUMP_ZIP:-}" diff --git a/scripts/backup/verify-host188-products-archive.py b/scripts/backup/verify-host188-products-archive.py index 79c4ed3b6..bfa1cd3c6 100644 --- a/scripts/backup/verify-host188-products-archive.py +++ b/scripts/backup/verify-host188-products-archive.py @@ -27,11 +27,46 @@ RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock") def acquire_runtime_lock(): - if Path(__file__).resolve().parent != Path("/backup/scripts") or os.environ.get("BACKUP_RUNTIME_SHARED_LOCK_HELD") == "1": + if Path(__file__).resolve().parent != Path("/backup/scripts"): return None if RUNTIME_LOCK.is_symlink(): raise RuntimeError("backup_runtime_gate_unsafe") - handle = RUNTIME_LOCK.open("a+") + inherited_fd_path: Path | None = None + proc_fd_root = Path(f"/proc/{os.getpid()}/fd") + if proc_fd_root.is_dir(): + candidate = proc_fd_root / "197" + if candidate.exists() or candidate.is_symlink(): + inherited_fd_path = candidate + else: + candidate = Path("/dev/fd/197") + if candidate.exists() or candidate.is_symlink(): + inherited_fd_path = candidate + + if inherited_fd_path is None: + handle = RUNTIME_LOCK.open("a+") + else: + if not RUNTIME_LOCK.is_file(): + raise RuntimeError("backup_runtime_gate_unavailable") + if proc_fd_root.is_dir(): + try: + inherited_target = inherited_fd_path.resolve(strict=True) + expected_target = RUNTIME_LOCK.resolve(strict=True) + except OSError as exc: + raise RuntimeError("backup_runtime_inherited_fd_unresolved") from exc + else: + try: + raw_target = fcntl.fcntl(197, getattr(fcntl, "F_GETPATH", 50), b"\0" * 1024) + inherited_target = Path(os.fsdecode(raw_target.split(b"\0", 1)[0])).resolve(strict=True) + expected_target = RUNTIME_LOCK.resolve(strict=True) + except OSError as exc: + raise RuntimeError("backup_runtime_inherited_fd_unresolved") from exc + if inherited_target != expected_target: + raise RuntimeError("backup_runtime_inherited_fd_mismatch") + try: + handle = os.fdopen(os.dup(197), "a+") + except OSError as exc: + raise RuntimeError("backup_runtime_inherited_fd_unavailable") from exc + if RUNTIME_LOCK.stat().st_uid != os.getuid(): handle.close() raise RuntimeError("backup_runtime_gate_owner_invalid") diff --git a/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 b/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 index 96d5d85ab..0122b360e 100644 --- a/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 +++ b/scripts/reboot-recovery/tests/agent99-host110-broker-contract-replay.ps1 @@ -35,11 +35,19 @@ Write-ReplayTrace "define_functions" $functionNames = @( "Test-Agent99JsonField", "Assert-Agent99JsonFields", + "Assert-Agent99ExactJsonFields", "Convert-Agent99ExecutorDocument", "Assert-Agent99ApplyResultFields", "Assert-Agent99VerifierFields", "Get-Agent99ExecutorResult", - "Get-Agent99CommittedFailureResult" + "Get-Agent99CommittedFailureResult", + "Assert-Agent99VerifierEvidence", + "Assert-Agent99PreflightEvidence", + "Get-Agent99TransportReceiptReadback", + "New-Agent99TransportLossReconciliation", + "Assert-Agent99ExecutorEvidence", + "Assert-Agent99ReconciliationEvidence", + "Assert-Agent99ExistingBrokerEvidence" ) foreach ($functionName in $functionNames) { $node = $ast.Find( @@ -56,8 +64,11 @@ foreach ($functionName in $functionNames) { $SourceRevision = "a" * 40 $RunId = "windows99-contract-replay" +$Mode = "Apply" +$TargetHost = "192.168.0.110" $ExpectedFileCount = 18 $sourceHead = "b" * 40 +$VerifierSha256 = "c" * 64 $payloadReceipt = "/backup/status/agent99-backup-runtime-deploy-$RunId.json" $terminalReceipt = "/backup/status/agent99-backup-runtime-terminal-$RunId.json" @@ -95,8 +106,12 @@ $verifier = [pscustomobject]@{ fileCount = 18 backupScriptFileCount = 17 backupHealthExporterIncluded = $true + destination = "/backup/scripts" + exporterDestination = "/home/wooo/scripts/backup-health-textfile-exporter.py" remoteWritePerformed = $false + secretValuesRead = $false selfIdentityVerified = $true + verifierSha256 = $VerifierSha256 } $success = [pscustomobject]@{ schemaVersion = "agent99_host110_backup_runtime_executor_v1" @@ -127,7 +142,7 @@ $success = [pscustomobject]@{ } Write-ReplayTrace "accept_success" -Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" | Out-Null +Get-Agent99ExecutorResult (New-ReplayTransport $success 0) "Apply" $VerifierSha256 | Out-Null $topLevelRequired = @( "schemaVersion", "mode", "status", "ok", "sourceRevision", "sourceHead", "runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded", @@ -139,21 +154,22 @@ $topLevelRequired = @( $verifierRequired = @( "schemaVersion", "ok", "sourceRevision", "runId", "fileCount", "backupScriptFileCount", "backupHealthExporterIncluded", "remoteWritePerformed", - "selfIdentityVerified" + "secretValuesRead", "selfIdentityVerified", "destination", "exporterDestination", + "verifierSha256" ) $missingFieldRejections = 0 Write-ReplayTrace "reject_missing_top" foreach ($field in $topLevelRequired) { $mutated = Copy-ReplayDocument $success $mutated.PSObject.Properties.Remove($field) - Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_$field" + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_$field" $missingFieldRejections++ } Write-ReplayTrace "reject_missing_verifier" foreach ($field in $verifierRequired) { $mutated = Copy-ReplayDocument $success $mutated.verifier.PSObject.Properties.Remove($field) - Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "missing_verifier_$field" + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "missing_verifier_$field" $missingFieldRejections++ } @@ -169,12 +185,12 @@ Write-ReplayTrace "reject_types" foreach ($mutation in $typeMutations) { $mutated = Copy-ReplayDocument $success $mutated.($mutation.field) = $mutation.value - Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_$($mutation.field)" + Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_$($mutation.field)" $typeRejections++ } $mutated = Copy-ReplayDocument $success $mutated.verifier.remoteWritePerformed = "false" -Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" } "type_verifier_remoteWritePerformed" +Assert-Rejected { Get-Agent99ExecutorResult (New-ReplayTransport $mutated 0) "Apply" $VerifierSha256 } "type_verifier_remoteWritePerformed" $typeRejections++ $cleanupPending = Copy-ReplayDocument $success @@ -204,6 +220,174 @@ $terminalReceiptFailed.verifier = $null Write-ReplayTrace "accept_terminal_receipt_failed" Get-Agent99CommittedFailureResult (New-ReplayTransport $terminalReceiptFailed 76) | Out-Null +$EvidencePath = "C:\Wooo\Agent99\evidence\host110-backup-runtime\agent99-host110-backup-runtime-$RunId.json" +$preflight = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_preflight_v1" + ok = $true + sourceRevision = $SourceRevision + sourceHead = $sourceHead + fileCount = 18 + backupScriptFileCount = 17 + backupHealthExporterIncluded = $true + remoteWritePerformed = $false +} +$brokerSuccess = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_runtime_broker_receipt_v3" + ok = $true + status = "verified" + mode = "Apply" + sourceRevision = $SourceRevision + sourceHead = $sourceHead + runId = $RunId + targetHost = $TargetHost + sourceTransport = "windows99_gitea_exact_revision_manifest" + executor = "host110_backup_runtime_executor" + verifier = "independent_host110_backup_runtime_python_readback" + decisionProvider = "deterministic_only" + criticProvider = "deterministic_only" + agentAction = "controlled_apply" + check = $preflight + apply = $success + verify = $verifier + cleanupVerified = $true + errorCode = "" + elapsedSeconds = 1.25 + evidence = $EvidencePath + secretValuesRead = $false + rawSessionStored = $false +} +Write-ReplayTrace "accept_existing_success" +Assert-Agent99ExistingBrokerEvidence $brokerSuccess $EvidencePath + +$brokerRequired = @( + "schemaVersion", "ok", "status", "mode", "sourceRevision", "sourceHead", + "runId", "targetHost", "sourceTransport", "executor", "verifier", + "decisionProvider", "criticProvider", "agentAction", "check", "apply", + "verify", "cleanupVerified", "errorCode", "elapsedSeconds", "evidence", + "secretValuesRead", "rawSessionStored" +) +$existingEvidenceRejections = 0 +Write-ReplayTrace "reject_existing_missing" +foreach ($field in $brokerRequired) { + $mutated = Copy-ReplayDocument $brokerSuccess + $mutated.PSObject.Properties.Remove($field) + Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_missing_$field" + $existingEvidenceRejections++ +} +$existingMutations = @( + @{ name = "ok_string_false"; field = "ok"; value = "false" }, + @{ name = "wrong_mode"; field = "mode"; value = "Verify" }, + @{ name = "wrong_run_id"; field = "runId"; value = "other-run" }, + @{ name = "wrong_status"; field = "status"; value = "readback_ok" }, + @{ name = "wrong_cleanup"; field = "cleanupVerified"; value = $false }, + @{ name = "cleanup_string"; field = "cleanupVerified"; value = "true" }, + @{ name = "elapsed_string"; field = "elapsedSeconds"; value = "1.25" } +) +Write-ReplayTrace "reject_existing_conflicts" +foreach ($mutation in $existingMutations) { + $mutated = Copy-ReplayDocument $brokerSuccess + $mutated.($mutation.field) = $mutation.value + Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_$($mutation.name)" + $existingEvidenceRejections++ +} +$mutated = Copy-ReplayDocument $brokerSuccess +$mutated.apply.runId = "other-run" +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_apply_run_conflict" +$existingEvidenceRejections++ +$mutated = Copy-ReplayDocument $brokerSuccess +$mutated.verify.PSObject.Properties.Remove("selfIdentityVerified") +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_verify_nested_missing" +$existingEvidenceRejections++ +$mutated = Copy-ReplayDocument $brokerSuccess +$mutated | Add-Member -NotePropertyName unexpected -NotePropertyValue $true +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_top" +$existingEvidenceRejections++ +$mutated = Copy-ReplayDocument $brokerSuccess +$mutated.apply | Add-Member -NotePropertyName unexpected -NotePropertyValue $true +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $mutated $EvidencePath } "existing_extra_apply" +$existingEvidenceRejections++ + +$brokerFailure = Copy-ReplayDocument $brokerSuccess +$brokerFailure.ok = $false +$brokerFailure.status = "failed" +$brokerFailure.cleanupVerified = $false +$brokerFailure.errorCode = "host110_apply_cleanup_pending" +$brokerFailure.apply = $cleanupPending +$brokerFailure.verify = $null +Write-ReplayTrace "accept_existing_failure" +Assert-Agent99ExistingBrokerEvidence $brokerFailure $EvidencePath +$failureWithoutPreflight = Copy-ReplayDocument $brokerFailure +$failureWithoutPreflight.check = $null +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithoutPreflight $EvidencePath } "failure_preflight_missing" +$existingEvidenceRejections++ +$failureWithImpossibleVerifier = Copy-ReplayDocument $brokerFailure +$failureWithImpossibleVerifier.verify = $verifier +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $failureWithImpossibleVerifier $EvidencePath } "failure_verifier_conflict" +$existingEvidenceRejections++ +$postcommitCleanupFailure = Copy-ReplayDocument $brokerSuccess +$postcommitCleanupFailure.ok = $false +$postcommitCleanupFailure.status = "failed" +$postcommitCleanupFailure.cleanupVerified = $false +$postcommitCleanupFailure.errorCode = "remote_source_stage_cleanup_failed" +Assert-Agent99ExistingBrokerEvidence $postcommitCleanupFailure $EvidencePath +$malformedFailure = Copy-ReplayDocument $brokerFailure +$malformedFailure.ok = "false" +Assert-Rejected { Assert-Agent99ExistingBrokerEvidence $malformedFailure $EvidencePath } "malformed_failure_not_upgraded" +$existingEvidenceRejections++ + +$sourcePackage = [pscustomobject]@{ sourceHead = $sourceHead } +$transportLoss = [pscustomobject]@{ reason = "transport_timeout"; exitCode = -2 } +$receiptReadback = [pscustomobject]@{ + schemaVersion = "agent99_host110_backup_transport_receipt_readback_v1" + ok = $true + sourceRevision = $SourceRevision + sourceHead = $sourceHead + runId = $RunId + payloadReceipt = $payloadReceipt + payloadState = "valid" + terminalReceipt = $terminalReceipt + terminalState = "missing" + terminalStatus = "" + terminalCleanupVerified = $false + terminalExitCode = -1 + remoteWritePerformed = $false + secretValuesRead = $false +} +Write-ReplayTrace "reconcile_terminal_missing" +$terminalMissing = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $receiptReadback $verifier +if ( + $terminalMissing.status -ne "committed_verified_terminal_missing" -or + -not $terminalMissing.payloadCommitted -or + -not $terminalMissing.destinationVerified -or + $terminalMissing.retryApplyAllowed +) { throw "transport_terminal_missing_reconciliation_failed" } +Assert-Agent99ReconciliationEvidence $terminalMissing $sourceHead + +$terminalPresentReadback = Copy-ReplayDocument $receiptReadback +$terminalPresentReadback.terminalState = "valid" +$terminalPresentReadback.terminalStatus = "verified" +$terminalPresentReadback.terminalCleanupVerified = $true +$terminalPresentReadback.terminalExitCode = 0 +$transportRecovered = New-Agent99TransportLossReconciliation $transportLoss $sourcePackage $terminalPresentReadback $verifier +if ($transportRecovered.status -ne "committed_verified_transport_lost" -or $transportRecovered.retryApplyAllowed) { + throw "transport_verified_reconciliation_failed" +} +Assert-Agent99ReconciliationEvidence $transportRecovered $sourceHead +$conflictingTransport = Copy-ReplayDocument $transportRecovered +$conflictingTransport.terminalStatus = "cleanup_pending" +Assert-Rejected { Assert-Agent99ReconciliationEvidence $conflictingTransport $sourceHead } "transport_terminal_status_conflict" +$existingEvidenceRejections++ + +$committedUnknown = New-Agent99TransportLossReconciliation ` + $transportLoss $sourcePackage $null $null "receipt_readback_unavailable" "verifier_unavailable" +if ( + $committedUnknown.status -ne "committed_unknown" -or + $committedUnknown.payloadCommitted -or + $committedUnknown.destinationVerified -or + $committedUnknown.retryApplyAllowed +) { throw "transport_unknown_reconciliation_failed" } +Assert-Agent99ReconciliationEvidence $committedUnknown $sourceHead + Write-ReplayTrace "complete" [pscustomobject]@{ schemaVersion = "agent99_host110_broker_contract_replay_v1" @@ -213,6 +397,11 @@ Write-ReplayTrace "complete" cleanupPendingPreserved = $true committedSignalPreserved = $true terminalReceiptFailurePreserved = $true + existingSuccessAccepted = $true + existingFailurePreserved = $true + existingEvidenceRejections = $existingEvidenceRejections + terminalMissingReconciled = $true + committedUnknownPreserved = $true missingFieldRejections = $missingFieldRejections typeRejections = $typeRejections remoteWritePerformed = $false diff --git a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py index b025a3556..7bd52708b 100644 --- a/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py +++ b/scripts/reboot-recovery/tests/test_agent99_host110_backup_runtime_broker.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import fcntl import grp import hashlib import importlib.util @@ -94,6 +95,88 @@ def _write_executable(path: Path, source: str) -> None: path.chmod(0o755) +def _write_fcntl_flock(path: Path) -> None: + _write_executable( + path, + """#!/usr/bin/env python3 +import fcntl +import sys +import time + +args = sys.argv[1:] +operation = fcntl.LOCK_SH if "-s" in args else fcntl.LOCK_EX +wait_seconds = float(args[args.index("-w") + 1]) if "-w" in args else 0.0 +descriptor = int(args[-1]) +deadline = time.monotonic() + wait_seconds +while True: + try: + fcntl.flock(descriptor, operation | fcntl.LOCK_NB) + raise SystemExit(0) + except BlockingIOError: + if time.monotonic() >= deadline: + raise SystemExit(1) + time.sleep(0.02) +""", + ) + + +def _build_shared_lock_gate_fixture( + tmp_path: Path, + entrypoint: str, + *, + reject_fd_reopen: bool = False, + body_sleep_seconds: float = 0.0, +) -> tuple[Path, Path, Path, dict[str, str]]: + runtime_root = tmp_path / "runtime-scripts" + runtime_root.mkdir() + lock_path = tmp_path / "runtime.lock" + fake_bin = tmp_path / "fake-bin" + fake_bin.mkdir() + _write_fcntl_flock(fake_bin / "flock") + + source_map = { + "common": ROOT / "scripts/backup/common.sh", + "restore": ROOT / "scripts/backup/gitea-full-backup-restore-drill.sh", + "offsite": ROOT / "scripts/backup/backup-offsite-readiness-gate.sh", + } + source = source_map[entrypoint].read_text(encoding="utf-8") + source = source.replace("/backup/scripts", str(runtime_root)) + source = source.replace("/tmp/agent99-host110-backup-runtime.lock", str(lock_path)) + source = source.replace("flock -s -w 60", "flock -s -w 1") + if reject_fd_reopen: + source, replacement_count = re.subn( + r"(?m)^(\s*)exec 197>>.*$", + r"\1printf 'FD_REOPENED=1\\n' >&2; exit 98", + source, + ) + assert replacement_count == 1 + + body = "printf 'BODY_REACHED=1\\n'\n" + if body_sleep_seconds: + body += f"sleep {body_sleep_seconds}\n" + if entrypoint == "common": + common_path = runtime_root / "common.sh" + gate_source = source.split("# WOOO AIOps - 備份共用函式庫", 1)[0] + _write_executable(common_path, gate_source) + script_path = runtime_root / "backup-gitea.sh" + _write_executable( + script_path, + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + f"source {str(common_path)!r}\n" + + body, + ) + else: + product_start = "DUMP_ZIP=" if entrypoint == "restore" else "BACKUP_BASE=" + gate_source = source.split(product_start, 1)[0] + script_path = runtime_root / source_map[entrypoint].name + _write_executable(script_path, gate_source + body) + + environment = os.environ.copy() + environment["PATH"] = f"{fake_bin}:{environment['PATH']}" + return script_path, runtime_root, lock_path, environment + + def _build_executor_harness(tmp_path: Path, run_id: str) -> dict[str, object]: destination = tmp_path / "backup-scripts" exporter_root = tmp_path / "exporter" @@ -337,6 +420,10 @@ def test_windows99_broker_fetches_exact_gitea_revision_and_is_bounded() -> None: assert '$property = $Object.PSObject.Properties[$Name]' in source assert 'if ($null -eq $property) { return $false }' in source assert 'Get-Agent99CommittedFailureResult $applyTransport' in source + assert 'Invoke-Agent99TransportLossReconciliation $applyTransport $sourcePackage' in source + assert 'status = "committed_unknown"' in source + assert '"committed_verified_terminal_missing"' in source + assert 'retryApplyAllowed = $false' in source assert 'throw "host110_apply_$([string]$apply.status)"' in source @@ -405,11 +492,19 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w for function_name in ( "Test-Agent99JsonField", "Assert-Agent99JsonFields", + "Assert-Agent99ExactJsonFields", "Convert-Agent99ExecutorDocument", "Assert-Agent99ApplyResultFields", "Assert-Agent99VerifierFields", "Get-Agent99ExecutorResult", "Get-Agent99CommittedFailureResult", + "Assert-Agent99VerifierEvidence", + "Assert-Agent99PreflightEvidence", + "Get-Agent99TransportReceiptReadback", + "New-Agent99TransportLossReconciliation", + "Assert-Agent99ExecutorEvidence", + "Assert-Agent99ReconciliationEvidence", + "Assert-Agent99ExistingBrokerEvidence", ): assert f'"{function_name}"' in source assert "$mutated.PSObject.Properties.Remove($field)" in source @@ -417,12 +512,188 @@ def test_windows99_contract_replay_uses_actual_broker_functions_without_remote_w assert 'status = "cleanup_pending"' in source assert 'status = "committed_signal_deferred"' in source assert 'status = "terminal_receipt_failed"' in source + assert 'ok = "false"' in source + assert 'status -ne "committed_verified_terminal_missing"' in source + assert 'status -ne "committed_unknown"' in source + assert "existingEvidenceRejections" in source + assert "malformed_failure_not_upgraded" in source assert "remoteWritePerformed = $false" in source assert "secretValuesRead = $false" in source for forbidden in ("New-Item", "Set-Content", "WriteAllText", "ssh.exe", "scp.exe"): assert forbidden not in source +def test_existing_broker_evidence_replay_is_exact_typed_and_nested_fail_closed() -> None: + source = BROKER.read_text(encoding="utf-8") + exact_guard = _powershell_function(source, "Assert-Agent99ExactJsonFields") + existing = _powershell_function(source, "Assert-Agent99ExistingBrokerEvidence") + reconciliation = _powershell_function(source, "Assert-Agent99ReconciliationEvidence") + + assert "$properties.Count -ne $Contract.Count" in exact_guard + assert "$Contract.ContainsKey([string]$property.Name)" in exact_guard + for field in ( + "schemaVersion", + "ok", + "status", + "mode", + "sourceRevision", + "sourceHead", + "runId", + "targetHost", + "sourceTransport", + "executor", + "verifier", + "decisionProvider", + "criticProvider", + "agentAction", + "check", + "apply", + "verify", + "cleanupVerified", + "errorCode", + "elapsedSeconds", + "evidence", + "secretValuesRead", + "rawSessionStored", + ): + assert re.search(rf"^\s+{field} = \"", existing, re.MULTILINE), field + assert "Assert-Agent99PreflightEvidence $Existing.check" in existing + assert "Assert-Agent99ExecutorEvidence $Existing.apply" in existing + assert "Assert-Agent99ReconciliationEvidence $Existing.apply" in existing + assert "Assert-Agent99VerifierEvidence $Existing.verify" in existing + assert 'throw "existing_broker_apply_failure_evidence_missing"' in existing + assert 'retryApplyAllowed = "Boolean"' in reconciliation + assert "host110_existing_reconciliation_payload_truth_mismatch" in reconciliation + + +def test_transport_loss_reconciliation_is_read_only_run_bound_and_never_retries_apply() -> None: + source = BROKER.read_text(encoding="utf-8") + reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0] + reconcile = _powershell_function(source, "Invoke-Agent99TransportLossReconciliation") + classify = _powershell_function(source, "New-Agent99TransportLossReconciliation") + + for expected in ( + "agent99-backup-runtime-deploy-{args.run_id}.json", + "agent99-backup-runtime-terminal-{args.run_id}.json", + 'set(document) == set(contract)', + 'payload["status"] == "payload_verified"', + 'terminal["payloadReceipt"] == str(payload_path)', + '"remoteWritePerformed": False', + '"secretValuesRead": False', + ): + assert expected in reader + assert "Invoke-Agent99BoundedSsh $command 60" in reconcile + assert "Invoke-Agent99NoWriteVerifier $SourcePackage" in reconcile + assert 'status = "committed_unknown"' in classify + assert '"committed_verified_terminal_missing"' in classify + assert '"committed_verified_transport_lost"' in classify + assert "retryApplyAllowed = $false" in classify + for forbidden in ("rm ", "mv ", "install ", "scp", "--mode apply"): + assert forbidden not in reconcile + + +def test_transport_receipt_readback_rejects_extra_and_wrong_typed_documents(tmp_path: Path) -> None: + source = BROKER.read_text(encoding="utf-8") + reader = source.split("$TransportReconciliationReadback = @'", 1)[1].split("\n'@", 1)[0] + status_root = tmp_path / "status" + status_root.mkdir() + run_id = "transport-loss-replay" + source_revision = "a" * 40 + source_head = "b" * 40 + verifier_sha256 = "c" * 64 + payload_path = status_root / f"agent99-backup-runtime-deploy-{run_id}.json" + terminal_path = status_root / f"agent99-backup-runtime-terminal-{run_id}.json" + payload = { + "schemaVersion": "agent99_host110_backup_runtime_deploy_receipt_v3", + "status": "payload_verified", + "sourceRevision": source_revision, + "sourceHead": source_head, + "runId": run_id, + "fileCount": 18, + "backupScriptFileCount": 17, + "backupHealthExporterIncluded": True, + "rollbackAttempted": False, + "rollbackPerformed": False, + "rollbackVerified": False, + "zeroResidueVerified": True, + "zeroResidueScope": "run_owned_destination_temps", + "verifierSha256": verifier_sha256, + "executorHost": "192.168.0.110", + "productionServiceRestarted": False, + "secretValuesRead": False, + "writtenAt": 1, + } + terminal = { + "schemaVersion": "agent99_host110_backup_runtime_terminal_receipt_v1", + "status": "verified", + "ok": True, + "payloadCommitted": True, + "payloadReceipt": str(payload_path), + "sourceRevision": source_revision, + "sourceHead": source_head, + "runId": run_id, + "fileCount": 18, + "backupScriptFileCount": 17, + "backupHealthExporterIncluded": True, + "stageCleanupVerified": True, + "rollbackPrestateCleanupVerified": True, + "cleanupVerified": True, + "independentVerifierVerified": True, + "zeroResidueScope": "run_owned_destination_temps_and_internal_stage_prestate", + "deferredSignal": None, + "terminalExitCode": 0, + "verifierSha256": verifier_sha256, + "executorHost": "192.168.0.110", + "productionServiceRestarted": False, + "secretValuesRead": False, + "writtenAt": 2, + } + + def run_readback() -> dict[str, object]: + result = subprocess.run( + [ + sys.executable, + "-", + "--status-root", + str(status_root), + "--source-revision", + source_revision, + "--source-head", + source_head, + "--run-id", + run_id, + "--verifier-sha256", + verifier_sha256, + ], + input=reader, + check=False, + capture_output=True, + text=True, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) + + payload_path.write_text(json.dumps(payload), encoding="utf-8") + terminal_path.write_text(json.dumps(terminal), encoding="utf-8") + valid = run_readback() + assert valid["payloadState"] == "valid" + assert valid["terminalState"] == "valid" + assert valid["terminalStatus"] == "verified" + + payload["unexpected"] = True + payload_path.write_text(json.dumps(payload), encoding="utf-8") + assert run_readback()["payloadState"] == "invalid" + payload.pop("unexpected") + payload["writtenAt"] = "1" + payload_path.write_text(json.dumps(payload), encoding="utf-8") + assert run_readback()["payloadState"] == "invalid" + payload["writtenAt"] = 1 + payload_path.write_text(json.dumps(payload), encoding="utf-8") + terminal["ok"] = "false" + terminal_path.write_text(json.dumps(terminal), encoding="utf-8") + assert run_readback()["terminalState"] == "invalid" + + def test_broker_scp_basenames_match_executor_source_stage_contract() -> None: broker = BROKER.read_text(encoding="utf-8") executor = EXECUTOR.read_text(encoding="utf-8") @@ -854,8 +1125,258 @@ def test_backup_runtime_entrypoints_share_the_deployment_gate() -> None: for source in (common, restore, offsite): assert "/tmp/agent99-host110-backup-runtime.lock" in source assert "flock -s -w 60" in source + assert "realpath -e" in source + assert "readlink -f" in source + assert "identity unresolved" in source + assert "/proc/$$/fd/197" in source + assert "/dev/fd/197" in source + assert "inherited fd 197 identity mismatch" in source + assert source.index("backup_runtime_inherited_fd_status") < source.index("exec 197>>") + assert 'resolve_backup_runtime_source_path "${BASH_SOURCE[1]:-${BASH_SOURCE[0]:-}}"' in common + assert '${PWD}/${raw_path}' in common + assert '[ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ] || return 0' not in common + for source in (restore, offsite): + assert 'resolve_backup_runtime_script_path "${BASH_SOURCE[0]:-}"' in source + assert '&& [ "${BACKUP_RUNTIME_SHARED_LOCK_HELD:-0}" != "1" ]' not in source assert "fcntl.LOCK_SH | fcntl.LOCK_NB" in archive - assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" in archive + assert "BACKUP_RUNTIME_SHARED_LOCK_HELD" not in archive + assert 'Path(f"/proc/{os.getpid()}/fd")' in archive + assert 'Path("/dev/fd/197")' in archive + assert "os.dup(197)" in archive + + +def test_common_shared_lock_preserves_resolved_nonproduction_fixture_behavior(tmp_path: Path) -> None: + gate_source = (ROOT / "scripts/backup/common.sh").read_text(encoding="utf-8").split( + "# WOOO AIOps - 備份共用函式庫", + 1, + )[0] + gate_fixture = tmp_path / "common-gate.sh" + _write_executable(gate_fixture, gate_source) + wrapper = tmp_path / "fixture-entrypoint.sh" + _write_executable( + wrapper, + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + f"source {str(gate_fixture)!r}\n" + "printf 'NONPRODUCTION_FIXTURE_OK=1\\n'\n", + ) + + result = subprocess.run( + [str(wrapper)], + check=False, + capture_output=True, + text=True, + ) + + assert result.returncode == 0, result.stderr + assert "NONPRODUCTION_FIXTURE_OK=1" in result.stdout + + +@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"]) +@pytest.mark.parametrize("invocation", ["absolute", "relative", "symlink"]) +def test_backup_runtime_shared_lock_blocks_absolute_relative_and_symlink_invocation( + tmp_path: Path, + entrypoint: str, + invocation: str, +) -> None: + script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture( + tmp_path, + entrypoint, + ) + + command_path = str(script_path) + cwd = runtime_root + if invocation == "relative": + command_path = f"./{script_path.name}" + elif invocation == "symlink": + link_path = tmp_path / f"{entrypoint}-entrypoint-link" + link_path.symlink_to(script_path) + command_path = str(link_path) + + environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1" + lock_handle = lock_path.open("a+") + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + result = subprocess.run( + [str(command_path)], + cwd=cwd, + check=False, + capture_output=True, + text=True, + env=environment, + timeout=3, + ) + finally: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) + lock_handle.close() + + assert result.returncode == 75 + assert "backup runtime deployment is active" in result.stderr + assert "BODY_REACHED=1" not in result.stdout + + +@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"]) +def test_backup_runtime_shared_lock_reuses_exact_inherited_fd_without_reopen( + tmp_path: Path, + entrypoint: str, +) -> None: + script_path, runtime_root, lock_path, environment = _build_shared_lock_gate_fixture( + tmp_path, + entrypoint, + reject_fd_reopen=True, + body_sleep_seconds=0.5, + ) + wrapper = tmp_path / "run-with-inherited-lock.sh" + _write_executable( + wrapper, + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + f"exec 197>>{str(lock_path)!r}\n" + "flock -s -w 1 197\n" + "export BACKUP_RUNTIME_SHARED_LOCK_HELD=forged_external_value\n" + f"exec {str(script_path)!r}\n", + ) + + process = subprocess.Popen( + [str(wrapper)], + cwd=runtime_root, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=environment, + ) + assert process.stdout is not None + assert process.stdout.readline().strip() == "BODY_REACHED=1" + contender = lock_path.open("a+") + try: + with pytest.raises(BlockingIOError): + fcntl.flock(contender.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + finally: + contender.close() + stdout, stderr = process.communicate(timeout=3) + + assert process.returncode == 0, stderr + assert "FD_REOPENED=1" not in stderr + assert stdout == "" + + +@pytest.mark.parametrize("entrypoint", ["common", "restore", "offsite"]) +def test_backup_runtime_shared_lock_rejects_inherited_fd_for_another_file( + tmp_path: Path, + entrypoint: str, +) -> None: + script_path, runtime_root, _, environment = _build_shared_lock_gate_fixture(tmp_path, entrypoint) + wrong_lock = tmp_path / "wrong-runtime.lock" + wrapper = tmp_path / "run-with-wrong-inherited-lock.sh" + _write_executable( + wrapper, + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + f"exec 197>>{str(wrong_lock)!r}\n" + "export BACKUP_RUNTIME_SHARED_LOCK_HELD=1\n" + f"exec {str(script_path)!r}\n", + ) + + result = subprocess.run( + [str(wrapper)], + cwd=runtime_root, + check=False, + capture_output=True, + text=True, + env=environment, + timeout=3, + ) + + assert result.returncode == 69 + assert "inherited fd 197 identity mismatch" in result.stderr + assert "BODY_REACHED=1" not in result.stdout + + +def test_host188_archive_lock_ignores_forged_env_and_validates_inherited_fd(tmp_path: Path) -> None: + runtime_root = tmp_path / "runtime-scripts" + runtime_root.mkdir() + lock_path = tmp_path / "runtime.lock" + archive_path = runtime_root / "verify-host188-products-archive.py" + archive_source = (ROOT / "scripts/backup/verify-host188-products-archive.py").read_text(encoding="utf-8") + archive_source = archive_source.replace('Path("/backup/scripts")', f"Path({str(runtime_root)!r})") + archive_source = archive_source.replace( + 'RUNTIME_LOCK = Path("/tmp/agent99-host110-backup-runtime.lock")', + f"RUNTIME_LOCK = Path({str(lock_path)!r})", + ) + archive_source = archive_source.replace("time.monotonic() + 60", "time.monotonic() + 0.2") + _write_executable(archive_path, archive_source) + runner = tmp_path / "archive-lock-replay.py" + runner.write_text( + """import fcntl +import importlib.util +import os +import pathlib +import sys + +archive_path = pathlib.Path(sys.argv[1]) +lock_path = pathlib.Path(sys.argv[2]) +mode = sys.argv[3] +spec = importlib.util.spec_from_file_location("host188_archive_lock_replay", archive_path) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +if mode in {"exact", "wrong"}: + selected = lock_path if mode == "exact" else lock_path.with_name("wrong-runtime.lock") + selected.touch() + inherited = selected.open("a+") + fcntl.flock(inherited.fileno(), fcntl.LOCK_SH | fcntl.LOCK_NB) + os.dup2(inherited.fileno(), 197) +try: + handle = module.acquire_runtime_lock() +except RuntimeError as exc: + print(str(exc)) + raise SystemExit(69) +else: + print("LOCK_OK=1") + if handle is not None: + handle.close() +""", + encoding="utf-8", + ) + + environment = os.environ.copy() + environment["BACKUP_RUNTIME_SHARED_LOCK_HELD"] = "1" + lock_handle = lock_path.open("a+") + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + try: + forged = subprocess.run( + [sys.executable, str(runner), str(archive_path), str(lock_path), "forged"], + check=False, + capture_output=True, + text=True, + env=environment, + timeout=3, + ) + finally: + fcntl.flock(lock_handle.fileno(), fcntl.LOCK_UN) + lock_handle.close() + exact = subprocess.run( + [sys.executable, str(runner), str(archive_path), str(lock_path), "exact"], + check=False, + capture_output=True, + text=True, + env=environment, + timeout=3, + ) + wrong = subprocess.run( + [sys.executable, str(runner), str(archive_path), str(lock_path), "wrong"], + check=False, + capture_output=True, + text=True, + env=environment, + timeout=3, + ) + + assert forged.returncode == 69 + assert "backup_runtime_deployment_active" in forged.stdout + assert exact.returncode == 0, exact.stdout + exact.stderr + assert "LOCK_OK=1" in exact.stdout + assert wrong.returncode == 69 + assert "backup_runtime_inherited_fd_mismatch" in wrong.stdout def test_independent_verifier_accepts_exact_bundle_without_writing(tmp_path, monkeypatch, capsys) -> None: