fix(agent99): use ephemeral apply transport

2026-07-14 21:31:34 +08:00
parent 5f4f1ab865
commit df88473f8b
3 changed files with 355 additions and 4 deletions
--- a/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
+++ b/scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1
@@ -1,7 +1,8 @@
 [CmdletBinding()]
 param(
  [Parameter(Mandatory = $true)]
-  [string]$EnvelopeJson
+  [string]$EnvelopeJson,
+  [switch]$TransportPayloadDeletedBeforeReceiver
 )

 $ErrorActionPreference = "Stop"
@@ -48,6 +49,9 @@ $script:ReceiverRunId = ""
 $script:ReceiverWorkItemId = ""
 $script:ReceiverSourceRevision = ""
 $script:ReceiverManifestDigest = ""
+$script:ReceiverTransportPayloadMode = "unknown"
+$script:ReceiverTransientTransportPayloadStored = $false
+$script:ReceiverTransportPayloadDeletedBeforeReceiver = $false

 function Get-AgentSha256Bytes {
  param([byte[]]$Bytes)
@@ -593,6 +597,11 @@ function Write-AgentRemoteDeployReceipt {
  param([string]$Path, [object]$Receipt)

  if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
+  $Receipt | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
+  $Receipt | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
+  $Receipt | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
+  $Receipt | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
+  $Receipt | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
  $temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
  try {
    $json = $Receipt | ConvertTo-Json -Depth 12
@@ -617,6 +626,11 @@ function Write-AgentRemoteDeployReceipt {
 function Write-AgentResultAndExit {
  param([object]$Result, [int]$ExitCode)

+  $Result | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
+  $Result | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
+  $Result | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
+  $Result | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
+  $Result | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
  $Result | ConvertTo-Json -Compress -Depth 12
  exit $ExitCode
 }
@@ -629,6 +643,25 @@ try {
  if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" }
  $mode = [string]$envelope.mode
  if ($mode -notin @("check", "apply")) { throw "invalid_mode" }
+  $transportPayloadMode = [string]$envelope.transportPayloadMode
+  $transportPayloadContainsSecrets = [bool]$envelope.transportPayloadContainsSecrets
+  $expectedTransportPayloadMode = if ($mode -eq "apply") {
+    "ephemeral_file_delete_before_receiver"
+  } else {
+    "stdin_sentinel"
+  }
+  if ($transportPayloadMode -ne $expectedTransportPayloadMode) { throw "invalid_transport_payload_mode" }
+  if ($transportPayloadContainsSecrets) { throw "transport_payload_secret_contract_failed" }
+  $script:ReceiverTransportPayloadMode = $transportPayloadMode
+  $script:ReceiverTransientTransportPayloadStored = [bool]($mode -eq "apply")
+  if ($mode -eq "apply") {
+    if (-not $TransportPayloadDeletedBeforeReceiver) {
+      throw "transport_payload_delete_before_receiver_unverified"
+    }
+    $script:ReceiverTransportPayloadDeletedBeforeReceiver = $true
+  } elseif ($TransportPayloadDeletedBeforeReceiver) {
+    throw "unexpected_transport_payload_delete_receipt"
+  }
  $sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
  if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
  if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }
--- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh
+++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh
@@ -147,7 +147,7 @@ if [[ "${MODE}" == "apply" ]]; then
  [[ "${WORK_ITEM_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_work_item_id_missing_or_invalid"
 fi

-for dependency in git python3 ssh mktemp hostname timeout; do
+for dependency in git python3 ssh scp mktemp hostname timeout; do
  command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}"
 done

@@ -260,6 +260,12 @@ envelope = {
    "sourceHostAlias": "110",
    "targetHostAlias": "99",
    "transport": "110_to_99_ssh_publickey",
+    "transportPayloadMode": (
+        "ephemeral_file_delete_before_receiver"
+        if mode == "apply"
+        else "stdin_sentinel"
+    ),
+    "transportPayloadContainsSecrets": False,
    "mode": mode,
    "traceId": trace_id,
    "runId": run_id,
@@ -287,17 +293,25 @@ python3 - \
 from __future__ import annotations

 import base64
+import json
 import sys
 from pathlib import Path

 receiver_path, envelope_path, output_path = map(Path, sys.argv[1:4])
 receiver_base64 = base64.b64encode(receiver_path.read_bytes()).decode("ascii")
 envelope_base64 = base64.b64encode(envelope_path.read_bytes()).decode("ascii")
+envelope_mode = json.loads(envelope_path.read_text(encoding="utf-8"))["mode"]
+transport_delete_switch = (
+    " -TransportPayloadDeletedBeforeReceiver"
+    if envelope_mode == "apply"
+    else ""
+)
 bootstrap = (
    "$ErrorActionPreference = 'Stop'\r\n"
    f"$receiverText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{receiver_base64}'))\r\n"
    f"$envelopeText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{envelope_base64}'))\r\n"
-    "& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText\r\n"
+    "& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText"
+    f"{transport_delete_switch}\r\n"
 )
 output_path.write_text(bootstrap, encoding="utf-8")
 PY
@@ -334,6 +348,21 @@ output_path.write_text(
 )
 PY

+TRANSPORT_TOKEN="$(python3 - "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
+from __future__ import annotations
+
+import hashlib
+import sys
+from pathlib import Path
+
+print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:20])
+PY
+)"
+readonly TRANSPORT_TOKEN
+readonly REMOTE_TRANSPORT_DIR_WINDOWS='C:\Wooo\Agent99\deploy\transport-inbox'
+readonly REMOTE_TRANSPORT_PATH_WINDOWS="${REMOTE_TRANSPORT_DIR_WINDOWS}\\agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
+readonly REMOTE_TRANSPORT_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
+
 SSH_OPTIONS=(
  -T
  -o BatchMode=yes
@@ -355,6 +384,27 @@ if [[ -n "${IDENTITY_FILE}" ]]; then
  SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
 fi

+SCP_OPTIONS=(
+  -q
+  -o BatchMode=yes
+  -o PreferredAuthentications=publickey
+  -o PubkeyAuthentication=yes
+  -o PasswordAuthentication=no
+  -o KbdInteractiveAuthentication=no
+  -o NumberOfPasswordPrompts=0
+  -o StrictHostKeyChecking=yes
+  -o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}"
+  -o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}"
+  -o ConnectionAttempts=1
+  -o ServerAliveInterval=5
+  -o ServerAliveCountMax=2
+  -o LogLevel=ERROR
+  -o IdentitiesOnly=yes
+)
+if [[ -n "${IDENTITY_FILE}" ]]; then
+  SCP_OPTIONS+=( -i "${IDENTITY_FILE}" )
+fi
+
 readonly REMOTE_COMMAND="$(python3 - <<'PY'
 from __future__ import annotations

@@ -400,9 +450,164 @@ print(
 )
 PY
 )"
+
+readonly REMOTE_PREPARE_COMMAND="$(python3 - \
+  "${REMOTE_TRANSPORT_DIR_WINDOWS}" \
+  "${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
+from __future__ import annotations
+
+import base64
+import sys
+
+directory = sys.argv[1].replace("'", "''")
+path = sys.argv[2].replace("'", "''")
+script = rf'''$ErrorActionPreference = "Stop"
+$directory = '{directory}'
+$payloadPath = '{path}'
+New-Item -ItemType Directory -Force -Path $directory | Out-Null
+if (Test-Path -LiteralPath $payloadPath) {{
+  Remove-Item -LiteralPath $payloadPath -Force
+}}
+if (Test-Path -LiteralPath $payloadPath) {{
+  throw "remote_transport_stale_payload_cleanup_failed"
+}}'''
+encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
+print(
+    "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
+    f"-EncodedCommand {encoded}"
+)
+PY
+)"
+
+readonly REMOTE_FILE_COMMAND="$(python3 - \
+  "${REMOTE_TRANSPORT_PATH_WINDOWS}" \
+  "${REMOTE_STDIN_SENTINEL}" <<'PY'
+from __future__ import annotations
+
+import base64
+import sys
+
+path = sys.argv[1].replace("'", "''")
+sentinel = sys.argv[2].replace("'", "''")
+decoder = rf'''$ErrorActionPreference = "Stop"
+$payloadPath = '{path}'
+$sentinel = '{sentinel}'
+$allLines = @()
+try {{
+  $payloadFile = Get-Item -LiteralPath $payloadPath -ErrorAction Stop
+  if ($payloadFile.Length -le 0 -or $payloadFile.Length -gt 1048576) {{
+    throw "remote_bootstrap_file_size_invalid"
+  }}
+  $allLines = @([IO.File]::ReadAllLines($payloadPath, [Text.Encoding]::ASCII))
+}} finally {{
+  if (Test-Path -LiteralPath $payloadPath) {{
+    Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue
+  }}
+}}
+if (Test-Path -LiteralPath $payloadPath) {{
+  throw "remote_bootstrap_file_delete_failed"
+}}
+if ($allLines.Count -lt 2 -or $allLines[-1] -cne $sentinel) {{
+  throw "remote_bootstrap_sentinel_missing"
+}}
+$payloadLines = @($allLines[0..($allLines.Count - 2)])
+$charCount = [int](($payloadLines | Measure-Object -Property Length -Sum).Sum) + (2 * $payloadLines.Count)
+if ($payloadLines.Count -gt 8 -or $charCount -gt 1048576) {{
+  throw "remote_bootstrap_input_limit_exceeded"
+}}
+$payload = [string]::Join("", $payloadLines)
+$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
+$inputStream = New-Object IO.MemoryStream(,$compressed)
+$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
+$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
+try {{
+  $inputText = $reader.ReadToEnd()
+}} finally {{
+  $reader.Dispose()
+  $gzip.Dispose()
+  $inputStream.Dispose()
+}}
+& ([ScriptBlock]::Create($inputText))'''
+encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii")
+print(
+    "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
+    f"-EncodedCommand {encoded}"
+)
+PY
+)"
+
+readonly REMOTE_CLEANUP_COMMAND="$(python3 - \
+  "${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
+from __future__ import annotations
+
+import base64
+import sys
+
+path = sys.argv[1].replace("'", "''")
+script = rf'''$ErrorActionPreference = "Stop"
+$payloadPath = '{path}'
+if (Test-Path -LiteralPath $payloadPath) {{
+  Remove-Item -LiteralPath $payloadPath -Force
+}}
+if (Test-Path -LiteralPath $payloadPath) {{
+  throw "remote_transport_payload_cleanup_failed"
+}}'''
+encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
+print(
+    "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
+    f"-EncodedCommand {encoded}"
+)
+PY
+)"
+
+cleanup_remote_payload() {
+  timeout --signal=TERM --kill-after=5 30s \
+    ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_CLEANUP_COMMAND}" </dev/null \
+    >/dev/null 2>&1
+}
+
+if [[ "${MODE}" == "check" ]]; then
+  set +e
+  timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
+    ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
+  status=$?
+  set -e
+  exit "${status}"
+fi
+
+set +e
+timeout --signal=TERM --kill-after=5 30s \
+  ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_PREPARE_COMMAND}" </dev/null
+prepare_status=$?
+set -e
+if [[ "${prepare_status}" -ne 0 ]]; then
+  printf 'agent99_transport_error=ephemeral_payload_prepare_failed\n' >&2
+  exit "${prepare_status}"
+fi
+
+set +e
+timeout --signal=TERM --kill-after=10 120s \
+  scp "${SCP_OPTIONS[@]}" "${BOOTSTRAP_PAYLOAD_PATH}" \
+  "${TARGET}:${REMOTE_TRANSPORT_PATH_SCP}"
+copy_status=$?
+set -e
+if [[ "${copy_status}" -ne 0 ]]; then
+  cleanup_remote_payload || true
+  printf 'agent99_transport_error=ephemeral_payload_copy_failed\n' >&2
+  exit "${copy_status}"
+fi
+
 set +e
 timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
-  ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
+  ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_FILE_COMMAND}" </dev/null
 status=$?
+cleanup_remote_payload
+cleanup_status=$?
 set -e
+if [[ "${cleanup_status}" -ne 0 ]]; then
+  printf 'agent99_transport_error=ephemeral_payload_cleanup_verifier_failed\n' >&2
+  if [[ "${status}" -eq 0 ]]; then
+    exit 74
+  fi
+fi
 exit "${status}"
--- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
+++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py
@@ -89,6 +89,10 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None:
    assert "source_host_is_not_110" in source
    assert '"sourceHostAlias": "110"' in source
    assert '"transport": "110_to_99_ssh_publickey"' in source
+    assert '"ephemeral_file_delete_before_receiver"' in source
+    assert '"transportPayloadContainsSecrets": False' in source
+    assert 'scp "${SCP_OPTIONS[@]}"' in source
+    assert "REMOTE_CLEANUP_COMMAND" in source
    assert "-o BatchMode=yes" in source
    assert "-o PreferredAuthentications=publickey" in source
    assert "-o PubkeyAuthentication=yes" in source
@@ -264,6 +268,11 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
    assert "tokenValueRead = $false" in source
    assert "environmentSecretRead = $false" in source
    assert "rawEnvelopeStored = $false" in source
+    assert "transportPayloadMode" in source
+    assert "transientTransportPayloadStored" in source
+    assert "transientTransportPayloadDeletedBeforeReceiver" in source
+    assert "transportPayloadContainsSecrets" in source
+    assert "TransportPayloadDeletedBeforeReceiver" in source
    assert "$env:" not in source
    assert "GetEnvironmentVariable" not in source
    assert (
@@ -653,3 +662,107 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
    assert str(identity) in args
    payload_bytes = int(bytes_path.read_text(encoding="utf-8"))
    assert 50_000 < payload_bytes < 1_048_576
+
+
+def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin(
+    tmp_path: Path,
+) -> None:
+    fake_bin = tmp_path / "bin"
+    fake_bin.mkdir()
+    ssh_count_path = tmp_path / "ssh-count.txt"
+    ssh_stdin_path = tmp_path / "ssh-stdin-bytes.txt"
+    scp_args_path = tmp_path / "scp-args.txt"
+    known_hosts = tmp_path / "known_hosts"
+    identity = tmp_path / "existing_identity"
+    known_hosts.write_text("fixed-host-key-placeholder\n", encoding="utf-8")
+    identity.write_text("identity-content-is-not-read-by-wrapper\n", encoding="utf-8")
+
+    fake_hostname = fake_bin / "hostname"
+    fake_hostname.write_text(
+        "#!/usr/bin/env bash\n"
+        "[[ \"${1:-}\" == '-I' ]] || exit 65\n"
+        "printf '%s\\n' '192.168.0.110 127.0.0.1'\n",
+        encoding="utf-8",
+    )
+    fake_ssh = fake_bin / "ssh"
+    fake_ssh.write_text(
+        "#!/usr/bin/env bash\n"
+        f"count=$(cat {shlex.quote(str(ssh_count_path))} 2>/dev/null || printf '0')\n"
+        "count=$((count + 1))\n"
+        f"printf '%s' \"$count\" >{shlex.quote(str(ssh_count_path))}\n"
+        f"wc -c | tr -d ' ' >>{shlex.quote(str(ssh_stdin_path))}\n"
+        "if [[ \"$count\" -eq 2 ]]; then\n"
+        "  printf '%s\\n' '{\"status\":\"deployed_verified\"}'\n"
+        "fi\n",
+        encoding="utf-8",
+    )
+    fake_scp = fake_bin / "scp"
+    fake_scp.write_text(
+        "#!/usr/bin/env bash\n"
+        f"printf '%s\\n' \"$@\" >{shlex.quote(str(scp_args_path))}\n",
+        encoding="utf-8",
+    )
+    fake_timeout = fake_bin / "timeout"
+    fake_timeout.write_text(
+        "#!/usr/bin/env bash\n"
+        "shift 3\n"
+        "exec \"$@\"\n",
+        encoding="utf-8",
+    )
+    real_git = shutil.which("git")
+    assert real_git is not None
+    fake_git = fake_bin / "git"
+    fake_git.write_text(
+        "#!/usr/bin/env bash\n"
+        "case \" $* \" in\n"
+        "  *\" remote get-url origin \"*) "
+        "printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n"
+        "  *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
+        "  *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
+        "printf '%040d\\n' 0; exit 0 ;;\n"
+        "  *\" cat-file -e \"*) exit 0 ;;\n"
+        "  *\" diff --quiet \"*) exit 0 ;;\n"
+        "esac\n"
+        f"exec {shlex.quote(real_git)} \"$@\"\n",
+        encoding="utf-8",
+    )
+    for executable in (
+        fake_hostname,
+        fake_ssh,
+        fake_scp,
+        fake_timeout,
+        fake_git,
+    ):
+        executable.chmod(0o700)
+
+    environment = os.environ.copy()
+    environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
+    result = subprocess.run(
+        [
+            "bash",
+            str(SENDER),
+            "--apply",
+            "--trace-id",
+            "trace-fixture",
+            "--run-id",
+            "run-fixture",
+            "--work-item-id",
+            "AIA-P0-006",
+            "--known-hosts-file",
+            str(known_hosts),
+            "--identity-file",
+            str(identity),
+        ],
+        check=False,
+        capture_output=True,
+        text=True,
+        env=environment,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert '"status":"deployed_verified"' in result.stdout
+    assert ssh_count_path.read_text(encoding="utf-8") == "3"
+    assert ssh_stdin_path.read_text(encoding="utf-8").splitlines() == ["0", "0", "0"]
+    scp_args = scp_args_path.read_text(encoding="utf-8")
+    assert "Administrator@192.168.0.99:C:/Wooo/Agent99/deploy/transport-inbox/" in scp_args
+    assert "agent99-bootstrap-" in scp_args