Merge remote-tracking branch 'origin/main' into codex/p0-obs-001-20260714

This commit is contained in:
ogt
2026-07-14 21:35:50 +08:00
24 changed files with 886 additions and 73 deletions

View File

@@ -15,13 +15,58 @@ source "$(dirname "$0")/common.sh"
SERVICE="awoooi"
AWOOOI_HOST="192.168.0.188"
AWOOOI_DB_USER="awoooi"
AWOOOI_DB_PASS="awoooi_prod_2026"
AWOOOI_DB_PASS="${AWOOOI_DB_PASS:-}"
AWOOOI_DB_HOST="localhost"
AWOOOI_DB_PORT="5432"
K3S_DB_USER="postgres"
LOCAL_REPO="${BACKUP_BASE}/awoooi"
DUMP_DIR="/tmp/awoooi-backup-$$"
quote_remote() {
printf "%q" "$1"
}
pgpass_escape() {
local value="$1"
value="${value//\\/\\\\}"
value="${value//:/\\:}"
printf '%s' "${value}"
}
pgpass_line() {
local database="$1"
printf '%s:%s:%s:%s:%s\n' \
"$(pgpass_escape "${AWOOOI_DB_HOST}")" \
"$(pgpass_escape "${AWOOOI_DB_PORT}")" \
"$(pgpass_escape "${database}")" \
"$(pgpass_escape "${AWOOOI_DB_USER}")" \
"$(pgpass_escape "${AWOOOI_DB_PASS}")"
}
remote_pgpass_wrapper() {
local command="$1"
printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}"
}
run_remote_pg_dump() {
local database="$1"
local output_file="$2"
local stderr_file="${output_file}.stderr"
local command
command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")"
if pgpass_line "${database}" \
| ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \
>"${output_file}" 2>"${stderr_file}"; then
rm -f "${stderr_file}"
return 0
fi
tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true
rm -f "${stderr_file}"
return 1
}
# 保留策略覆寫(比其他服務更長)
KEEP_DAILY=14 # 14 天每日
KEEP_WEEKLY=8 # 8 週每週
@@ -31,6 +76,12 @@ main() {
local start_time=$(date +%s)
local failed=0
if [ -z "${AWOOOI_DB_PASS}" ]; then
log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback"
notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable"
exit 1
fi
log_info "========== 開始 AWOOOI 資料庫備份 =========="
mkdir -p "${DUMP_DIR}"
@@ -38,9 +89,8 @@ main() {
log_info "Dump awoooi_prod..."
local timestamp=$(date "+%Y%m%d_%H%M%S")
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
awoooi_prod" > "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" 2>&1; then
if run_remote_pg_dump \
"awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then
local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1)
log_success "awoooi_prod dump 完成 (${size})"
else
@@ -50,9 +100,8 @@ main() {
# Step 2: awoooi_dev dump
log_info "Dump awoooi_dev..."
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
awoooi_dev 2>/dev/null" > "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" 2>/dev/null; then
if run_remote_pg_dump \
"awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then
local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1)
log_success "awoooi_dev dump 完成 (${size})"
else
@@ -61,9 +110,8 @@ main() {
# Step 3: k3s_datastore dumpKine 後端)
log_info "Dump k3s_datastore..."
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
k3s_datastore 2>/dev/null" > "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" 2>/dev/null; then
if run_remote_pg_dump \
"k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then
local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1)
log_success "k3s_datastore dump 完成 (${size})"
else

View File

@@ -0,0 +1,25 @@
from __future__ import annotations
import re
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi.sh"
def test_daily_backup_has_no_hardcoded_database_password() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assignment = re.search(r"^AWOOOI_DB_PASS=(.+)$", text, re.MULTILINE)
assert assignment is not None
assert assignment.group(1) == '"${AWOOOI_DB_PASS:-}"'
assert "PGPASSWORD='${AWOOOI_DB_PASS}'" not in text
assert 'PGPASSFILE="$pgpass"' in text
assert "pg_dump --no-password" in text
def test_daily_backup_fails_closed_without_runtime_credential() -> None:
text = SCRIPT.read_text(encoding="utf-8")
assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text
assert "refusing an unauthenticated or hardcoded fallback" in text

View File

@@ -21,6 +21,7 @@ function Get-AgentLivePreflightDecision {
[int]$SelfHealthPerformanceIssueCount,
[int]$StaleAlertExecutionCount,
[int]$AlertIncomingCount,
[int]$StaleAlertIncomingCount,
[int]$PendingCommandCount
)
@@ -75,8 +76,10 @@ function Get-AgentLivePreflightDecision {
}
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
$blockingReasons += "pending_work_before_reboot"
} elseif ($AlertIncomingCount -gt 0) {
$warningReasons += "fresh_alert_pending_next_inbox_tick"
}
return [pscustomobject]@{
@@ -428,6 +431,7 @@ $decision = Get-AgentLivePreflightDecision `
-SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount `
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
-AlertIncomingCount $alertIncoming.count `
-StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes `
-PendingCommandCount $commandQueue.count
$blockingReasons = @($decision.blockingReasons)
$warningReasons = @($decision.warningReasons)
@@ -459,6 +463,7 @@ $result = [pscustomobject]@{
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount
alertIncomingCount = [int]$alertIncoming.count
alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes
alertRunningCount = [int]$alertRunning.count
pendingCommandCount = [int]$commandQueue.count
relayListenerCount = $relayListeners.Count
@@ -480,6 +485,7 @@ Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
Write-Output "ALERT_INCOMING_STALE_COUNT=$($result.summary.alertIncomingStaleCount)"
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"

View File

@@ -1,7 +1,8 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)]
[string]$EnvelopeJson
[string]$EnvelopeJson,
[switch]$TransportPayloadDeletedBeforeReceiver
)
$ErrorActionPreference = "Stop"
@@ -48,6 +49,9 @@ $script:ReceiverRunId = ""
$script:ReceiverWorkItemId = ""
$script:ReceiverSourceRevision = ""
$script:ReceiverManifestDigest = ""
$script:ReceiverTransportPayloadMode = "unknown"
$script:ReceiverTransientTransportPayloadStored = $false
$script:ReceiverTransportPayloadDeletedBeforeReceiver = $false
function Get-AgentSha256Bytes {
param([byte[]]$Bytes)
@@ -284,7 +288,7 @@ function Invoke-AgentLivePreflight {
[int]$summary.requiredEvidenceParseFailureCount -eq 0 -and
[int]$summary.requiredEvidenceContentFailureCount -eq 0 -and
[int]$summary.relayListenerCount -gt 0 -and
[int]$summary.alertIncomingCount -eq 0 -and
[int]$summary.alertIncomingStaleCount -eq 0 -and
[int]$summary.alertRunningCount -eq 0 -and
[int]$summary.pendingCommandCount -eq 0 -and
$runtimePlaneBlockingReasons.Count -eq 0
@@ -317,6 +321,7 @@ function Invoke-AgentLivePreflight {
} | Where-Object { $_ } | Sort-Object -Unique)
} else { @() }
alertIncomingCount = if ($summary) { [int]$summary.alertIncomingCount } else { -1 }
alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 }
alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 }
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
preflightGreen = [bool]($summary -and $summary.preflightGreen)
@@ -592,6 +597,11 @@ function Write-AgentRemoteDeployReceipt {
param([string]$Path, [object]$Receipt)
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
$Receipt | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
$Receipt | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
$Receipt | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
$Receipt | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
$Receipt | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
try {
$json = $Receipt | ConvertTo-Json -Depth 12
@@ -616,6 +626,11 @@ function Write-AgentRemoteDeployReceipt {
function Write-AgentResultAndExit {
param([object]$Result, [int]$ExitCode)
$Result | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
$Result | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
$Result | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
$Result | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
$Result | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
$Result | ConvertTo-Json -Compress -Depth 12
exit $ExitCode
}
@@ -628,6 +643,25 @@ try {
if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" }
$mode = [string]$envelope.mode
if ($mode -notin @("check", "apply")) { throw "invalid_mode" }
$transportPayloadMode = [string]$envelope.transportPayloadMode
$transportPayloadContainsSecrets = [bool]$envelope.transportPayloadContainsSecrets
$expectedTransportPayloadMode = if ($mode -eq "apply") {
"ephemeral_file_delete_before_receiver"
} else {
"stdin_sentinel"
}
if ($transportPayloadMode -ne $expectedTransportPayloadMode) { throw "invalid_transport_payload_mode" }
if ($transportPayloadContainsSecrets) { throw "transport_payload_secret_contract_failed" }
$script:ReceiverTransportPayloadMode = $transportPayloadMode
$script:ReceiverTransientTransportPayloadStored = [bool]($mode -eq "apply")
if ($mode -eq "apply") {
if (-not $TransportPayloadDeletedBeforeReceiver) {
throw "transport_payload_delete_before_receiver_unverified"
}
$script:ReceiverTransportPayloadDeletedBeforeReceiver = $true
} elseif ($TransportPayloadDeletedBeforeReceiver) {
throw "unexpected_transport_payload_delete_receipt"
}
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }

View File

@@ -11,6 +11,7 @@ readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git
readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1"
readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
readonly REMOTE_STDIN_SENTINEL="__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__"
RUNTIME_FILES=(
"agent99-bootstrap.ps1"
@@ -146,7 +147,7 @@ if [[ "${MODE}" == "apply" ]]; then
[[ "${WORK_ITEM_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_work_item_id_missing_or_invalid"
fi
for dependency in git python3 ssh mktemp hostname timeout; do
for dependency in git python3 ssh scp mktemp hostname timeout; do
command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}"
done
@@ -259,6 +260,12 @@ envelope = {
"sourceHostAlias": "110",
"targetHostAlias": "99",
"transport": "110_to_99_ssh_publickey",
"transportPayloadMode": (
"ephemeral_file_delete_before_receiver"
if mode == "apply"
else "stdin_sentinel"
),
"transportPayloadContainsSecrets": False,
"mode": mode,
"traceId": trace_id,
"runId": run_id,
@@ -286,24 +293,33 @@ python3 - \
from __future__ import annotations
import base64
import json
import sys
from pathlib import Path
receiver_path, envelope_path, output_path = map(Path, sys.argv[1:4])
receiver_base64 = base64.b64encode(receiver_path.read_bytes()).decode("ascii")
envelope_base64 = base64.b64encode(envelope_path.read_bytes()).decode("ascii")
envelope_mode = json.loads(envelope_path.read_text(encoding="utf-8"))["mode"]
transport_delete_switch = (
" -TransportPayloadDeletedBeforeReceiver"
if envelope_mode == "apply"
else ""
)
bootstrap = (
"$ErrorActionPreference = 'Stop'\r\n"
f"$receiverText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{receiver_base64}'))\r\n"
f"$envelopeText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{envelope_base64}'))\r\n"
"& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText\r\n"
"& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText"
f"{transport_delete_switch}\r\n"
)
output_path.write_text(bootstrap, encoding="utf-8")
PY
python3 - \
"${BOOTSTRAP_PATH}" \
"${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
"${BOOTSTRAP_PAYLOAD_PATH}" \
"${REMOTE_STDIN_SENTINEL}" <<'PY'
from __future__ import annotations
import base64
@@ -312,17 +328,41 @@ import sys
from pathlib import Path
bootstrap_path, output_path = map(Path, sys.argv[1:3])
sentinel = sys.argv[3]
compressed = gzip.compress(
bootstrap_path.read_bytes(),
compresslevel=9,
mtime=0,
)
payload = base64.b64encode(compressed).decode("ascii")
line_width = 65536
payload_lines = [
payload[offset : offset + line_width]
for offset in range(0, len(payload), line_width)
]
if not payload_lines or len(payload_lines) > 8:
raise SystemExit("remote_bootstrap_payload_line_contract_failed")
output_path.write_text(
base64.b64encode(compressed).decode("ascii"),
"\n".join(payload_lines) + "\n" + sentinel + "\n",
encoding="ascii",
)
PY
TRANSPORT_TOKEN="$(python3 - "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
from __future__ import annotations
import hashlib
import sys
from pathlib import Path
print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:20])
PY
)"
readonly TRANSPORT_TOKEN
readonly REMOTE_TRANSPORT_DIR_WINDOWS='C:\Wooo\Agent99\deploy\transport-inbox'
readonly REMOTE_TRANSPORT_PATH_WINDOWS="${REMOTE_TRANSPORT_DIR_WINDOWS}\\agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
readonly REMOTE_TRANSPORT_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
SSH_OPTIONS=(
-T
-o BatchMode=yes
@@ -344,12 +384,53 @@ if [[ -n "${IDENTITY_FILE}" ]]; then
SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
fi
SCP_OPTIONS=(
-q
-o BatchMode=yes
-o PreferredAuthentications=publickey
-o PubkeyAuthentication=yes
-o PasswordAuthentication=no
-o KbdInteractiveAuthentication=no
-o NumberOfPasswordPrompts=0
-o StrictHostKeyChecking=yes
-o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}"
-o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}"
-o ConnectionAttempts=1
-o ServerAliveInterval=5
-o ServerAliveCountMax=2
-o LogLevel=ERROR
-o IdentitiesOnly=yes
)
if [[ -n "${IDENTITY_FILE}" ]]; then
SCP_OPTIONS+=( -i "${IDENTITY_FILE}" )
fi
readonly REMOTE_COMMAND="$(python3 - <<'PY'
from __future__ import annotations
import base64
decoder = r'''$payload = [Console]::In.ReadToEnd()
decoder = r'''$sentinel = "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__"
$lines = New-Object System.Collections.Generic.List[string]
$lineCount = 0
$charCount = 0
$terminated = $false
while (($line = [Console]::In.ReadLine()) -ne $null) {
if ($line -ceq $sentinel) {
$terminated = $true
break
}
$lineCount += 1
$charCount += $line.Length + 2
if ($lineCount -gt 8 -or $charCount -gt 1048576) {
throw "remote_bootstrap_input_limit_exceeded"
}
[void]$lines.Add($line)
}
if (-not $terminated) {
throw "remote_bootstrap_sentinel_missing"
}
$payload = [string]::Join("", $lines)
$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
$inputStream = New-Object IO.MemoryStream(,$compressed)
$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
@@ -369,9 +450,164 @@ print(
)
PY
)"
readonly REMOTE_PREPARE_COMMAND="$(python3 - \
"${REMOTE_TRANSPORT_DIR_WINDOWS}" \
"${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
from __future__ import annotations
import base64
import sys
directory = sys.argv[1].replace("'", "''")
path = sys.argv[2].replace("'", "''")
script = rf'''$ErrorActionPreference = "Stop"
$directory = '{directory}'
$payloadPath = '{path}'
New-Item -ItemType Directory -Force -Path $directory | Out-Null
if (Test-Path -LiteralPath $payloadPath) {{
Remove-Item -LiteralPath $payloadPath -Force
}}
if (Test-Path -LiteralPath $payloadPath) {{
throw "remote_transport_stale_payload_cleanup_failed"
}}'''
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
)"
readonly REMOTE_FILE_COMMAND="$(python3 - \
"${REMOTE_TRANSPORT_PATH_WINDOWS}" \
"${REMOTE_STDIN_SENTINEL}" <<'PY'
from __future__ import annotations
import base64
import sys
path = sys.argv[1].replace("'", "''")
sentinel = sys.argv[2].replace("'", "''")
decoder = rf'''$ErrorActionPreference = "Stop"
$payloadPath = '{path}'
$sentinel = '{sentinel}'
$allLines = @()
try {{
$payloadFile = Get-Item -LiteralPath $payloadPath -ErrorAction Stop
if ($payloadFile.Length -le 0 -or $payloadFile.Length -gt 1048576) {{
throw "remote_bootstrap_file_size_invalid"
}}
$allLines = @([IO.File]::ReadAllLines($payloadPath, [Text.Encoding]::ASCII))
}} finally {{
if (Test-Path -LiteralPath $payloadPath) {{
Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue
}}
}}
if (Test-Path -LiteralPath $payloadPath) {{
throw "remote_bootstrap_file_delete_failed"
}}
if ($allLines.Count -lt 2 -or $allLines[-1] -cne $sentinel) {{
throw "remote_bootstrap_sentinel_missing"
}}
$payloadLines = @($allLines[0..($allLines.Count - 2)])
$charCount = [int](($payloadLines | Measure-Object -Property Length -Sum).Sum) + (2 * $payloadLines.Count)
if ($payloadLines.Count -gt 8 -or $charCount -gt 1048576) {{
throw "remote_bootstrap_input_limit_exceeded"
}}
$payload = [string]::Join("", $payloadLines)
$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
$inputStream = New-Object IO.MemoryStream(,$compressed)
$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
try {{
$inputText = $reader.ReadToEnd()
}} finally {{
$reader.Dispose()
$gzip.Dispose()
$inputStream.Dispose()
}}
& ([ScriptBlock]::Create($inputText))'''
encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
)"
readonly REMOTE_CLEANUP_COMMAND="$(python3 - \
"${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
from __future__ import annotations
import base64
import sys
path = sys.argv[1].replace("'", "''")
script = rf'''$ErrorActionPreference = "Stop"
$payloadPath = '{path}'
if (Test-Path -LiteralPath $payloadPath) {{
Remove-Item -LiteralPath $payloadPath -Force
}}
if (Test-Path -LiteralPath $payloadPath) {{
throw "remote_transport_payload_cleanup_failed"
}}'''
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
print(
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
f"-EncodedCommand {encoded}"
)
PY
)"
cleanup_remote_payload() {
timeout --signal=TERM --kill-after=5 30s \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_CLEANUP_COMMAND}" </dev/null \
>/dev/null 2>&1
}
if [[ "${MODE}" == "check" ]]; then
set +e
timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
status=$?
set -e
exit "${status}"
fi
set +e
timeout --signal=TERM --kill-after=5 30s \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_PREPARE_COMMAND}" </dev/null
prepare_status=$?
set -e
if [[ "${prepare_status}" -ne 0 ]]; then
printf 'agent99_transport_error=ephemeral_payload_prepare_failed\n' >&2
exit "${prepare_status}"
fi
set +e
timeout --signal=TERM --kill-after=10 120s \
scp "${SCP_OPTIONS[@]}" "${BOOTSTRAP_PAYLOAD_PATH}" \
"${TARGET}:${REMOTE_TRANSPORT_PATH_SCP}"
copy_status=$?
set -e
if [[ "${copy_status}" -ne 0 ]]; then
cleanup_remote_payload || true
printf 'agent99_transport_error=ephemeral_payload_copy_failed\n' >&2
exit "${copy_status}"
fi
set +e
timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_FILE_COMMAND}" </dev/null
status=$?
cleanup_remote_payload
cleanup_status=$?
set -e
if [[ "${cleanup_status}" -ne 0 ]]; then
printf 'agent99_transport_error=ephemeral_payload_cleanup_verifier_failed\n' >&2
if [[ "${status}" -eq 0 ]]; then
exit 74
fi
fi
exit "${status}"

View File

@@ -41,6 +41,7 @@ def _base_case() -> dict[str, Any]:
"selfHealthPerformanceIssueCount": 0,
"staleAlertExecutionCount": 0,
"alertIncomingCount": 0,
"staleAlertIncomingCount": 0,
"pendingCommandCount": 0,
}
@@ -82,6 +83,7 @@ $parameters = @{{
SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
AlertIncomingCount = [int]$case.alertIncomingCount
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
PendingCommandCount = [int]$case.pendingCommandCount
}}
Get-AgentLivePreflightDecision @parameters |
@@ -106,6 +108,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
assert '$warningReasons += "self_health_warning"' in decision
assert '$warningReasons += "performance_warning"' in decision
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
assert '"critical" { $blockingReasons += "self_health_critical" }' in decision
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
@@ -172,6 +175,27 @@ def test_decision_behavior_when_powershell_is_available() -> None:
("sre_alert_relay_not_listening",),
(),
),
(
_with_overrides(alertIncomingCount=1),
True,
(),
("fresh_alert_pending_next_inbox_tick",),
),
(
_with_overrides(
alertIncomingCount=1,
staleAlertIncomingCount=1,
),
False,
("pending_work_before_reboot",),
(),
),
(
_with_overrides(pendingCommandCount=1),
False,
("pending_work_before_reboot",),
(),
),
)
for case, expected_eligible, expected_blockers, expected_warnings in cases:

View File

@@ -89,6 +89,10 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None:
assert "source_host_is_not_110" in source
assert '"sourceHostAlias": "110"' in source
assert '"transport": "110_to_99_ssh_publickey"' in source
assert '"ephemeral_file_delete_before_receiver"' in source
assert '"transportPayloadContainsSecrets": False' in source
assert 'scp "${SCP_OPTIONS[@]}"' in source
assert "REMOTE_CLEANUP_COMMAND" in source
assert "-o BatchMode=yes" in source
assert "-o PreferredAuthentications=publickey" in source
assert "-o PubkeyAuthentication=yes" in source
@@ -148,7 +152,18 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data()
assert "IO.Compression.GzipStream" in source
assert 'decoder.encode("utf-16le")' in source
assert "-EncodedCommand" in source
assert "[Console]::In.ReadToEnd()" in source
assert "[Console]::In.ReadToEnd()" not in source
assert "$reader.ReadToEnd()" in source
assert "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__" in source
assert "[Console]::In.ReadLine()" in source
assert "remote_bootstrap_input_limit_exceeded" in source
assert "remote_bootstrap_sentinel_missing" in source
assert "line_width = 65536" in source
assert "len(payload_lines) > 8" in source
assert '"\\n".join(payload_lines)' in source
assert "remote_bootstrap_payload_line_contract_failed" in source
assert "$lineCount -gt 8" in source
assert "$charCount -gt 1048576" in source
assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source
assert "rawEnvelope" not in source
assert "sshpass" not in source
@@ -253,6 +268,11 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
assert "tokenValueRead = $false" in source
assert "environmentSecretRead = $false" in source
assert "rawEnvelopeStored = $false" in source
assert "transportPayloadMode" in source
assert "transientTransportPayloadStored" in source
assert "transientTransportPayloadDeletedBeforeReceiver" in source
assert "transportPayloadContainsSecrets" in source
assert "TransportPayloadDeletedBeforeReceiver" in source
assert "$env:" not in source
assert "GetEnvironmentVariable" not in source
assert (
@@ -502,7 +522,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
assert '$_ -ne "self_health_not_ok"' in preflight
assert '[int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount' in preflight
assert '[int]$summary.requiredEvidenceFailureCount -eq 0' in preflight
assert '[int]$summary.alertIncomingCount -eq 0' in preflight
assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
assert '[int]$summary.alertRunningCount -eq 0' in preflight
assert '[int]$summary.pendingCommandCount -eq 0' in preflight
assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
@@ -640,4 +661,108 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
assert "Administrator@192.168.0.99" in args
assert str(identity) in args
payload_bytes = int(bytes_path.read_text(encoding="utf-8"))
assert 50_000 < payload_bytes < 500_000
assert 50_000 < payload_bytes < 1_048_576
def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin(
tmp_path: Path,
) -> None:
fake_bin = tmp_path / "bin"
fake_bin.mkdir()
ssh_count_path = tmp_path / "ssh-count.txt"
ssh_stdin_path = tmp_path / "ssh-stdin-bytes.txt"
scp_args_path = tmp_path / "scp-args.txt"
known_hosts = tmp_path / "known_hosts"
identity = tmp_path / "existing_identity"
known_hosts.write_text("fixed-host-key-placeholder\n", encoding="utf-8")
identity.write_text("identity-content-is-not-read-by-wrapper\n", encoding="utf-8")
fake_hostname = fake_bin / "hostname"
fake_hostname.write_text(
"#!/usr/bin/env bash\n"
"[[ \"${1:-}\" == '-I' ]] || exit 65\n"
"printf '%s\\n' '192.168.0.110 127.0.0.1'\n",
encoding="utf-8",
)
fake_ssh = fake_bin / "ssh"
fake_ssh.write_text(
"#!/usr/bin/env bash\n"
f"count=$(cat {shlex.quote(str(ssh_count_path))} 2>/dev/null || printf '0')\n"
"count=$((count + 1))\n"
f"printf '%s' \"$count\" >{shlex.quote(str(ssh_count_path))}\n"
f"wc -c | tr -d ' ' >>{shlex.quote(str(ssh_stdin_path))}\n"
"if [[ \"$count\" -eq 2 ]]; then\n"
" printf '%s\\n' '{\"status\":\"deployed_verified\"}'\n"
"fi\n",
encoding="utf-8",
)
fake_scp = fake_bin / "scp"
fake_scp.write_text(
"#!/usr/bin/env bash\n"
f"printf '%s\\n' \"$@\" >{shlex.quote(str(scp_args_path))}\n",
encoding="utf-8",
)
fake_timeout = fake_bin / "timeout"
fake_timeout.write_text(
"#!/usr/bin/env bash\n"
"shift 3\n"
"exec \"$@\"\n",
encoding="utf-8",
)
real_git = shutil.which("git")
assert real_git is not None
fake_git = fake_bin / "git"
fake_git.write_text(
"#!/usr/bin/env bash\n"
"case \" $* \" in\n"
" *\" remote get-url origin \"*) "
"printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n"
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
"printf '%040d\\n' 0; exit 0 ;;\n"
" *\" cat-file -e \"*) exit 0 ;;\n"
" *\" diff --quiet \"*) exit 0 ;;\n"
"esac\n"
f"exec {shlex.quote(real_git)} \"$@\"\n",
encoding="utf-8",
)
for executable in (
fake_hostname,
fake_ssh,
fake_scp,
fake_timeout,
fake_git,
):
executable.chmod(0o700)
environment = os.environ.copy()
environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
result = subprocess.run(
[
"bash",
str(SENDER),
"--apply",
"--trace-id",
"trace-fixture",
"--run-id",
"run-fixture",
"--work-item-id",
"AIA-P0-006",
"--known-hosts-file",
str(known_hosts),
"--identity-file",
str(identity),
],
check=False,
capture_output=True,
text=True,
env=environment,
)
assert result.returncode == 0, result.stderr
assert '"status":"deployed_verified"' in result.stdout
assert ssh_count_path.read_text(encoding="utf-8") == "3"
assert ssh_stdin_path.read_text(encoding="utf-8").splitlines() == ["0", "0", "0"]
scp_args = scp_args_path.read_text(encoding="utf-8")
assert "Administrator@192.168.0.99:C:/Wooo/Agent99/deploy/transport-inbox/" in scp_args
assert "agent99-bootstrap-" in scp_args