Merge remote-tracking branch 'origin/main' into codex/p0-obs-001-20260714
This commit is contained in:
@@ -15,13 +15,58 @@ source "$(dirname "$0")/common.sh"
|
||||
SERVICE="awoooi"
|
||||
AWOOOI_HOST="192.168.0.188"
|
||||
AWOOOI_DB_USER="awoooi"
|
||||
AWOOOI_DB_PASS="awoooi_prod_2026"
|
||||
AWOOOI_DB_PASS="${AWOOOI_DB_PASS:-}"
|
||||
AWOOOI_DB_HOST="localhost"
|
||||
AWOOOI_DB_PORT="5432"
|
||||
K3S_DB_USER="postgres"
|
||||
LOCAL_REPO="${BACKUP_BASE}/awoooi"
|
||||
DUMP_DIR="/tmp/awoooi-backup-$$"
|
||||
|
||||
quote_remote() {
|
||||
printf "%q" "$1"
|
||||
}
|
||||
|
||||
pgpass_escape() {
|
||||
local value="$1"
|
||||
value="${value//\\/\\\\}"
|
||||
value="${value//:/\\:}"
|
||||
printf '%s' "${value}"
|
||||
}
|
||||
|
||||
pgpass_line() {
|
||||
local database="$1"
|
||||
printf '%s:%s:%s:%s:%s\n' \
|
||||
"$(pgpass_escape "${AWOOOI_DB_HOST}")" \
|
||||
"$(pgpass_escape "${AWOOOI_DB_PORT}")" \
|
||||
"$(pgpass_escape "${database}")" \
|
||||
"$(pgpass_escape "${AWOOOI_DB_USER}")" \
|
||||
"$(pgpass_escape "${AWOOOI_DB_PASS}")"
|
||||
}
|
||||
|
||||
remote_pgpass_wrapper() {
|
||||
local command="$1"
|
||||
printf 'umask 077; pgpass=$(mktemp "${TMPDIR:-/tmp}/awoooi-pgpass.XXXXXX") || exit 1; cleanup() { rm -f "$pgpass"; }; trap cleanup EXIT HUP INT TERM; cat > "$pgpass"; PGPASSFILE="$pgpass" %s' "${command}"
|
||||
}
|
||||
|
||||
run_remote_pg_dump() {
|
||||
local database="$1"
|
||||
local output_file="$2"
|
||||
local stderr_file="${output_file}.stderr"
|
||||
local command
|
||||
|
||||
command="pg_dump --no-password -U $(quote_remote "${AWOOOI_DB_USER}") -h $(quote_remote "${AWOOOI_DB_HOST}") -p $(quote_remote "${AWOOOI_DB_PORT}") $(quote_remote "${database}")"
|
||||
if pgpass_line "${database}" \
|
||||
| ssh "ollama@${AWOOOI_HOST}" "$(remote_pgpass_wrapper "${command}")" \
|
||||
>"${output_file}" 2>"${stderr_file}"; then
|
||||
rm -f "${stderr_file}"
|
||||
return 0
|
||||
fi
|
||||
|
||||
tail -40 "${stderr_file}" | sed -E 's/(password=)[^ ]+/\1REDACTED/g' >&2 || true
|
||||
rm -f "${stderr_file}"
|
||||
return 1
|
||||
}
|
||||
|
||||
# 保留策略覆寫(比其他服務更長)
|
||||
KEEP_DAILY=14 # 14 天每日
|
||||
KEEP_WEEKLY=8 # 8 週每週
|
||||
@@ -31,6 +76,12 @@ main() {
|
||||
local start_time=$(date +%s)
|
||||
local failed=0
|
||||
|
||||
if [ -z "${AWOOOI_DB_PASS}" ]; then
|
||||
log_error "AWOOOI_DB_PASS is unavailable; refusing an unauthenticated or hardcoded fallback"
|
||||
notify_clawbot "failed" "${SERVICE}" "AWOOOI DB backup credential unavailable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_info "========== 開始 AWOOOI 資料庫備份 =========="
|
||||
mkdir -p "${DUMP_DIR}"
|
||||
|
||||
@@ -38,9 +89,8 @@ main() {
|
||||
log_info "Dump awoooi_prod..."
|
||||
local timestamp=$(date "+%Y%m%d_%H%M%S")
|
||||
|
||||
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
|
||||
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
|
||||
awoooi_prod" > "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" 2>&1; then
|
||||
if run_remote_pg_dump \
|
||||
"awoooi_prod" "${DUMP_DIR}/awoooi_prod_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/awoooi_prod_${timestamp}.sql" | cut -f1)
|
||||
log_success "awoooi_prod dump 完成 (${size})"
|
||||
else
|
||||
@@ -50,9 +100,8 @@ main() {
|
||||
|
||||
# Step 2: awoooi_dev dump
|
||||
log_info "Dump awoooi_dev..."
|
||||
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
|
||||
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
|
||||
awoooi_dev 2>/dev/null" > "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" 2>/dev/null; then
|
||||
if run_remote_pg_dump \
|
||||
"awoooi_dev" "${DUMP_DIR}/awoooi_dev_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/awoooi_dev_${timestamp}.sql" | cut -f1)
|
||||
log_success "awoooi_dev dump 完成 (${size})"
|
||||
else
|
||||
@@ -61,9 +110,8 @@ main() {
|
||||
|
||||
# Step 3: k3s_datastore dump(Kine 後端)
|
||||
log_info "Dump k3s_datastore..."
|
||||
if ssh ollama@${AWOOOI_HOST} "PGPASSWORD='${AWOOOI_DB_PASS}' pg_dump \
|
||||
-U ${AWOOOI_DB_USER} -h ${AWOOOI_DB_HOST} -p ${AWOOOI_DB_PORT} \
|
||||
k3s_datastore 2>/dev/null" > "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" 2>/dev/null; then
|
||||
if run_remote_pg_dump \
|
||||
"k3s_datastore" "${DUMP_DIR}/k3s_datastore_${timestamp}.sql"; then
|
||||
local size=$(du -h "${DUMP_DIR}/k3s_datastore_${timestamp}.sql" | cut -f1)
|
||||
log_success "k3s_datastore dump 完成 (${size})"
|
||||
else
|
||||
|
||||
25
scripts/backup/tests/test_backup_awoooi_secret_contract.py
Normal file
25
scripts/backup/tests/test_backup_awoooi_secret_contract.py
Normal file
@@ -0,0 +1,25 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
SCRIPT = ROOT / "scripts" / "backup" / "backup-awoooi.sh"
|
||||
|
||||
|
||||
def test_daily_backup_has_no_hardcoded_database_password() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
assignment = re.search(r"^AWOOOI_DB_PASS=(.+)$", text, re.MULTILINE)
|
||||
|
||||
assert assignment is not None
|
||||
assert assignment.group(1) == '"${AWOOOI_DB_PASS:-}"'
|
||||
assert "PGPASSWORD='${AWOOOI_DB_PASS}'" not in text
|
||||
assert 'PGPASSFILE="$pgpass"' in text
|
||||
assert "pg_dump --no-password" in text
|
||||
|
||||
|
||||
def test_daily_backup_fails_closed_without_runtime_credential() -> None:
|
||||
text = SCRIPT.read_text(encoding="utf-8")
|
||||
|
||||
assert 'if [ -z "${AWOOOI_DB_PASS}" ]; then' in text
|
||||
assert "refusing an unauthenticated or hardcoded fallback" in text
|
||||
@@ -21,6 +21,7 @@ function Get-AgentLivePreflightDecision {
|
||||
[int]$SelfHealthPerformanceIssueCount,
|
||||
[int]$StaleAlertExecutionCount,
|
||||
[int]$AlertIncomingCount,
|
||||
[int]$StaleAlertIncomingCount,
|
||||
[int]$PendingCommandCount
|
||||
)
|
||||
|
||||
@@ -75,8 +76,10 @@ function Get-AgentLivePreflightDecision {
|
||||
}
|
||||
|
||||
if ($StaleAlertExecutionCount -gt 0) { $blockingReasons += "stale_alert_execution" }
|
||||
if ($AlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
|
||||
if ($StaleAlertIncomingCount -gt 0 -or $PendingCommandCount -gt 0) {
|
||||
$blockingReasons += "pending_work_before_reboot"
|
||||
} elseif ($AlertIncomingCount -gt 0) {
|
||||
$warningReasons += "fresh_alert_pending_next_inbox_tick"
|
||||
}
|
||||
|
||||
return [pscustomobject]@{
|
||||
@@ -428,6 +431,7 @@ $decision = Get-AgentLivePreflightDecision `
|
||||
-SelfHealthPerformanceIssueCount $selfHealthPerformanceIssueCount `
|
||||
-StaleAlertExecutionCount $alertRunning.staleOverFiveMinutes `
|
||||
-AlertIncomingCount $alertIncoming.count `
|
||||
-StaleAlertIncomingCount $alertIncoming.staleOverFiveMinutes `
|
||||
-PendingCommandCount $commandQueue.count
|
||||
$blockingReasons = @($decision.blockingReasons)
|
||||
$warningReasons = @($decision.warningReasons)
|
||||
@@ -459,6 +463,7 @@ $result = [pscustomobject]@{
|
||||
selfHealthPerformanceSeverity = $selfHealthPerformanceSeverity
|
||||
selfHealthPerformanceIssueCount = $selfHealthPerformanceIssueCount
|
||||
alertIncomingCount = [int]$alertIncoming.count
|
||||
alertIncomingStaleCount = [int]$alertIncoming.staleOverFiveMinutes
|
||||
alertRunningCount = [int]$alertRunning.count
|
||||
pendingCommandCount = [int]$commandQueue.count
|
||||
relayListenerCount = $relayListeners.Count
|
||||
@@ -480,6 +485,7 @@ Write-Output "RUNTIME_MATCHED=$([int]$manifest.runtimeMatched)"
|
||||
Write-Output "TASKS_HEALTHY=$($result.summary.healthyTaskCount)/$($result.summary.requiredTaskCount)"
|
||||
Write-Output "RELAY_LISTENER_COUNT=$($result.summary.relayListenerCount)"
|
||||
Write-Output "ALERT_INCOMING_COUNT=$($result.summary.alertIncomingCount)"
|
||||
Write-Output "ALERT_INCOMING_STALE_COUNT=$($result.summary.alertIncomingStaleCount)"
|
||||
Write-Output "COMMAND_PENDING_COUNT=$($result.summary.pendingCommandCount)"
|
||||
Write-Output "SELF_HEALTH_SEVERITY=$($result.summary.selfHealthSeverity)"
|
||||
Write-Output "BLOCKING_REASONS=$($blockingReasons -join ',')"
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
[CmdletBinding()]
|
||||
param(
|
||||
[Parameter(Mandatory = $true)]
|
||||
[string]$EnvelopeJson
|
||||
[string]$EnvelopeJson,
|
||||
[switch]$TransportPayloadDeletedBeforeReceiver
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
@@ -48,6 +49,9 @@ $script:ReceiverRunId = ""
|
||||
$script:ReceiverWorkItemId = ""
|
||||
$script:ReceiverSourceRevision = ""
|
||||
$script:ReceiverManifestDigest = ""
|
||||
$script:ReceiverTransportPayloadMode = "unknown"
|
||||
$script:ReceiverTransientTransportPayloadStored = $false
|
||||
$script:ReceiverTransportPayloadDeletedBeforeReceiver = $false
|
||||
|
||||
function Get-AgentSha256Bytes {
|
||||
param([byte[]]$Bytes)
|
||||
@@ -284,7 +288,7 @@ function Invoke-AgentLivePreflight {
|
||||
[int]$summary.requiredEvidenceParseFailureCount -eq 0 -and
|
||||
[int]$summary.requiredEvidenceContentFailureCount -eq 0 -and
|
||||
[int]$summary.relayListenerCount -gt 0 -and
|
||||
[int]$summary.alertIncomingCount -eq 0 -and
|
||||
[int]$summary.alertIncomingStaleCount -eq 0 -and
|
||||
[int]$summary.alertRunningCount -eq 0 -and
|
||||
[int]$summary.pendingCommandCount -eq 0 -and
|
||||
$runtimePlaneBlockingReasons.Count -eq 0
|
||||
@@ -317,6 +321,7 @@ function Invoke-AgentLivePreflight {
|
||||
} | Where-Object { $_ } | Sort-Object -Unique)
|
||||
} else { @() }
|
||||
alertIncomingCount = if ($summary) { [int]$summary.alertIncomingCount } else { -1 }
|
||||
alertIncomingStaleCount = if ($summary) { [int]$summary.alertIncomingStaleCount } else { -1 }
|
||||
alertRunningCount = if ($summary) { [int]$summary.alertRunningCount } else { -1 }
|
||||
pendingCommandCount = if ($summary) { [int]$summary.pendingCommandCount } else { -1 }
|
||||
preflightGreen = [bool]($summary -and $summary.preflightGreen)
|
||||
@@ -592,6 +597,11 @@ function Write-AgentRemoteDeployReceipt {
|
||||
param([string]$Path, [object]$Receipt)
|
||||
|
||||
if (Test-Path -LiteralPath $Path) { throw "immutable_receipt_path_already_exists" }
|
||||
$Receipt | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
|
||||
$Receipt | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
|
||||
$Receipt | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
|
||||
$Receipt | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
|
||||
$Receipt | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
|
||||
$temporary = "$Path.tmp.$PID.$([Guid]::NewGuid().ToString('N'))"
|
||||
try {
|
||||
$json = $Receipt | ConvertTo-Json -Depth 12
|
||||
@@ -616,6 +626,11 @@ function Write-AgentRemoteDeployReceipt {
|
||||
function Write-AgentResultAndExit {
|
||||
param([object]$Result, [int]$ExitCode)
|
||||
|
||||
$Result | Add-Member -NotePropertyName transportPayloadMode -NotePropertyValue $script:ReceiverTransportPayloadMode -Force
|
||||
$Result | Add-Member -NotePropertyName transientTransportPayloadStored -NotePropertyValue $script:ReceiverTransientTransportPayloadStored -Force
|
||||
$Result | Add-Member -NotePropertyName transientTransportPayloadDeletedBeforeReceiver -NotePropertyValue $script:ReceiverTransportPayloadDeletedBeforeReceiver -Force
|
||||
$Result | Add-Member -NotePropertyName transportPayloadContainsSecrets -NotePropertyValue $false -Force
|
||||
$Result | Add-Member -NotePropertyName rawEnvelopeStored -NotePropertyValue $false -Force
|
||||
$Result | ConvertTo-Json -Compress -Depth 12
|
||||
exit $ExitCode
|
||||
}
|
||||
@@ -628,6 +643,25 @@ try {
|
||||
if ([string]$envelope.transport -ne "110_to_99_ssh_publickey") { throw "invalid_transport" }
|
||||
$mode = [string]$envelope.mode
|
||||
if ($mode -notin @("check", "apply")) { throw "invalid_mode" }
|
||||
$transportPayloadMode = [string]$envelope.transportPayloadMode
|
||||
$transportPayloadContainsSecrets = [bool]$envelope.transportPayloadContainsSecrets
|
||||
$expectedTransportPayloadMode = if ($mode -eq "apply") {
|
||||
"ephemeral_file_delete_before_receiver"
|
||||
} else {
|
||||
"stdin_sentinel"
|
||||
}
|
||||
if ($transportPayloadMode -ne $expectedTransportPayloadMode) { throw "invalid_transport_payload_mode" }
|
||||
if ($transportPayloadContainsSecrets) { throw "transport_payload_secret_contract_failed" }
|
||||
$script:ReceiverTransportPayloadMode = $transportPayloadMode
|
||||
$script:ReceiverTransientTransportPayloadStored = [bool]($mode -eq "apply")
|
||||
if ($mode -eq "apply") {
|
||||
if (-not $TransportPayloadDeletedBeforeReceiver) {
|
||||
throw "transport_payload_delete_before_receiver_unverified"
|
||||
}
|
||||
$script:ReceiverTransportPayloadDeletedBeforeReceiver = $true
|
||||
} elseif ($TransportPayloadDeletedBeforeReceiver) {
|
||||
throw "unexpected_transport_payload_delete_receipt"
|
||||
}
|
||||
$sourceRevision = ([string]$envelope.sourceRevision).ToLowerInvariant()
|
||||
if ($sourceRevision -notmatch "^[0-9a-f]{40}$") { throw "invalid_source_revision" }
|
||||
if ([int]$envelope.expectedRuntimeFileCount -ne 14) { throw "invalid_expected_runtime_file_count" }
|
||||
|
||||
@@ -11,6 +11,7 @@ readonly GITEA_ORIGIN_SSH_INTERNAL="ssh://git@192.168.0.110:2222/wooo/awoooi.git
|
||||
readonly RECEIVER_RELATIVE="scripts/reboot-recovery/agent99-remote-atomic-deploy-receiver.ps1"
|
||||
readonly PREFLIGHT_RELATIVE="scripts/reboot-recovery/agent99-live-preflight.ps1"
|
||||
readonly WRAPPER_RELATIVE="scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh"
|
||||
readonly REMOTE_STDIN_SENTINEL="__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__"
|
||||
|
||||
RUNTIME_FILES=(
|
||||
"agent99-bootstrap.ps1"
|
||||
@@ -146,7 +147,7 @@ if [[ "${MODE}" == "apply" ]]; then
|
||||
[[ "${WORK_ITEM_ID}" =~ ${SAFE_ID_PATTERN} ]] || fail "apply_work_item_id_missing_or_invalid"
|
||||
fi
|
||||
|
||||
for dependency in git python3 ssh mktemp hostname timeout; do
|
||||
for dependency in git python3 ssh scp mktemp hostname timeout; do
|
||||
command -v "${dependency}" >/dev/null 2>&1 || fail "required_command_missing_${dependency}"
|
||||
done
|
||||
|
||||
@@ -259,6 +260,12 @@ envelope = {
|
||||
"sourceHostAlias": "110",
|
||||
"targetHostAlias": "99",
|
||||
"transport": "110_to_99_ssh_publickey",
|
||||
"transportPayloadMode": (
|
||||
"ephemeral_file_delete_before_receiver"
|
||||
if mode == "apply"
|
||||
else "stdin_sentinel"
|
||||
),
|
||||
"transportPayloadContainsSecrets": False,
|
||||
"mode": mode,
|
||||
"traceId": trace_id,
|
||||
"runId": run_id,
|
||||
@@ -286,24 +293,33 @@ python3 - \
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
receiver_path, envelope_path, output_path = map(Path, sys.argv[1:4])
|
||||
receiver_base64 = base64.b64encode(receiver_path.read_bytes()).decode("ascii")
|
||||
envelope_base64 = base64.b64encode(envelope_path.read_bytes()).decode("ascii")
|
||||
envelope_mode = json.loads(envelope_path.read_text(encoding="utf-8"))["mode"]
|
||||
transport_delete_switch = (
|
||||
" -TransportPayloadDeletedBeforeReceiver"
|
||||
if envelope_mode == "apply"
|
||||
else ""
|
||||
)
|
||||
bootstrap = (
|
||||
"$ErrorActionPreference = 'Stop'\r\n"
|
||||
f"$receiverText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{receiver_base64}'))\r\n"
|
||||
f"$envelopeText = [Text.Encoding]::UTF8.GetString([Convert]::FromBase64String('{envelope_base64}'))\r\n"
|
||||
"& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText\r\n"
|
||||
"& ([ScriptBlock]::Create($receiverText)) -EnvelopeJson $envelopeText"
|
||||
f"{transport_delete_switch}\r\n"
|
||||
)
|
||||
output_path.write_text(bootstrap, encoding="utf-8")
|
||||
PY
|
||||
|
||||
python3 - \
|
||||
"${BOOTSTRAP_PATH}" \
|
||||
"${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
|
||||
"${BOOTSTRAP_PAYLOAD_PATH}" \
|
||||
"${REMOTE_STDIN_SENTINEL}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
@@ -312,17 +328,41 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
bootstrap_path, output_path = map(Path, sys.argv[1:3])
|
||||
sentinel = sys.argv[3]
|
||||
compressed = gzip.compress(
|
||||
bootstrap_path.read_bytes(),
|
||||
compresslevel=9,
|
||||
mtime=0,
|
||||
)
|
||||
payload = base64.b64encode(compressed).decode("ascii")
|
||||
line_width = 65536
|
||||
payload_lines = [
|
||||
payload[offset : offset + line_width]
|
||||
for offset in range(0, len(payload), line_width)
|
||||
]
|
||||
if not payload_lines or len(payload_lines) > 8:
|
||||
raise SystemExit("remote_bootstrap_payload_line_contract_failed")
|
||||
output_path.write_text(
|
||||
base64.b64encode(compressed).decode("ascii"),
|
||||
"\n".join(payload_lines) + "\n" + sentinel + "\n",
|
||||
encoding="ascii",
|
||||
)
|
||||
PY
|
||||
|
||||
TRANSPORT_TOKEN="$(python3 - "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
print(hashlib.sha256(Path(sys.argv[1]).read_bytes()).hexdigest()[:20])
|
||||
PY
|
||||
)"
|
||||
readonly TRANSPORT_TOKEN
|
||||
readonly REMOTE_TRANSPORT_DIR_WINDOWS='C:\Wooo\Agent99\deploy\transport-inbox'
|
||||
readonly REMOTE_TRANSPORT_PATH_WINDOWS="${REMOTE_TRANSPORT_DIR_WINDOWS}\\agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
|
||||
readonly REMOTE_TRANSPORT_PATH_SCP="C:/Wooo/Agent99/deploy/transport-inbox/agent99-bootstrap-${TRANSPORT_TOKEN}.b64"
|
||||
|
||||
SSH_OPTIONS=(
|
||||
-T
|
||||
-o BatchMode=yes
|
||||
@@ -344,12 +384,53 @@ if [[ -n "${IDENTITY_FILE}" ]]; then
|
||||
SSH_OPTIONS+=( -i "${IDENTITY_FILE}" )
|
||||
fi
|
||||
|
||||
SCP_OPTIONS=(
|
||||
-q
|
||||
-o BatchMode=yes
|
||||
-o PreferredAuthentications=publickey
|
||||
-o PubkeyAuthentication=yes
|
||||
-o PasswordAuthentication=no
|
||||
-o KbdInteractiveAuthentication=no
|
||||
-o NumberOfPasswordPrompts=0
|
||||
-o StrictHostKeyChecking=yes
|
||||
-o "UserKnownHostsFile=${KNOWN_HOSTS_FILE}"
|
||||
-o "ConnectTimeout=${CONNECT_TIMEOUT_SECONDS}"
|
||||
-o ConnectionAttempts=1
|
||||
-o ServerAliveInterval=5
|
||||
-o ServerAliveCountMax=2
|
||||
-o LogLevel=ERROR
|
||||
-o IdentitiesOnly=yes
|
||||
)
|
||||
if [[ -n "${IDENTITY_FILE}" ]]; then
|
||||
SCP_OPTIONS+=( -i "${IDENTITY_FILE}" )
|
||||
fi
|
||||
|
||||
readonly REMOTE_COMMAND="$(python3 - <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
|
||||
decoder = r'''$payload = [Console]::In.ReadToEnd()
|
||||
decoder = r'''$sentinel = "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__"
|
||||
$lines = New-Object System.Collections.Generic.List[string]
|
||||
$lineCount = 0
|
||||
$charCount = 0
|
||||
$terminated = $false
|
||||
while (($line = [Console]::In.ReadLine()) -ne $null) {
|
||||
if ($line -ceq $sentinel) {
|
||||
$terminated = $true
|
||||
break
|
||||
}
|
||||
$lineCount += 1
|
||||
$charCount += $line.Length + 2
|
||||
if ($lineCount -gt 8 -or $charCount -gt 1048576) {
|
||||
throw "remote_bootstrap_input_limit_exceeded"
|
||||
}
|
||||
[void]$lines.Add($line)
|
||||
}
|
||||
if (-not $terminated) {
|
||||
throw "remote_bootstrap_sentinel_missing"
|
||||
}
|
||||
$payload = [string]::Join("", $lines)
|
||||
$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
|
||||
$inputStream = New-Object IO.MemoryStream(,$compressed)
|
||||
$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
|
||||
@@ -369,9 +450,164 @@ print(
|
||||
)
|
||||
PY
|
||||
)"
|
||||
|
||||
readonly REMOTE_PREPARE_COMMAND="$(python3 - \
|
||||
"${REMOTE_TRANSPORT_DIR_WINDOWS}" \
|
||||
"${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import sys
|
||||
|
||||
directory = sys.argv[1].replace("'", "''")
|
||||
path = sys.argv[2].replace("'", "''")
|
||||
script = rf'''$ErrorActionPreference = "Stop"
|
||||
$directory = '{directory}'
|
||||
$payloadPath = '{path}'
|
||||
New-Item -ItemType Directory -Force -Path $directory | Out-Null
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
Remove-Item -LiteralPath $payloadPath -Force
|
||||
}}
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
throw "remote_transport_stale_payload_cleanup_failed"
|
||||
}}'''
|
||||
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
|
||||
print(
|
||||
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
|
||||
f"-EncodedCommand {encoded}"
|
||||
)
|
||||
PY
|
||||
)"
|
||||
|
||||
readonly REMOTE_FILE_COMMAND="$(python3 - \
|
||||
"${REMOTE_TRANSPORT_PATH_WINDOWS}" \
|
||||
"${REMOTE_STDIN_SENTINEL}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import sys
|
||||
|
||||
path = sys.argv[1].replace("'", "''")
|
||||
sentinel = sys.argv[2].replace("'", "''")
|
||||
decoder = rf'''$ErrorActionPreference = "Stop"
|
||||
$payloadPath = '{path}'
|
||||
$sentinel = '{sentinel}'
|
||||
$allLines = @()
|
||||
try {{
|
||||
$payloadFile = Get-Item -LiteralPath $payloadPath -ErrorAction Stop
|
||||
if ($payloadFile.Length -le 0 -or $payloadFile.Length -gt 1048576) {{
|
||||
throw "remote_bootstrap_file_size_invalid"
|
||||
}}
|
||||
$allLines = @([IO.File]::ReadAllLines($payloadPath, [Text.Encoding]::ASCII))
|
||||
}} finally {{
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
Remove-Item -LiteralPath $payloadPath -Force -ErrorAction SilentlyContinue
|
||||
}}
|
||||
}}
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
throw "remote_bootstrap_file_delete_failed"
|
||||
}}
|
||||
if ($allLines.Count -lt 2 -or $allLines[-1] -cne $sentinel) {{
|
||||
throw "remote_bootstrap_sentinel_missing"
|
||||
}}
|
||||
$payloadLines = @($allLines[0..($allLines.Count - 2)])
|
||||
$charCount = [int](($payloadLines | Measure-Object -Property Length -Sum).Sum) + (2 * $payloadLines.Count)
|
||||
if ($payloadLines.Count -gt 8 -or $charCount -gt 1048576) {{
|
||||
throw "remote_bootstrap_input_limit_exceeded"
|
||||
}}
|
||||
$payload = [string]::Join("", $payloadLines)
|
||||
$compressed = [Convert]::FromBase64String(($payload -replace '\s', ''))
|
||||
$inputStream = New-Object IO.MemoryStream(,$compressed)
|
||||
$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress)
|
||||
$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8)
|
||||
try {{
|
||||
$inputText = $reader.ReadToEnd()
|
||||
}} finally {{
|
||||
$reader.Dispose()
|
||||
$gzip.Dispose()
|
||||
$inputStream.Dispose()
|
||||
}}
|
||||
& ([ScriptBlock]::Create($inputText))'''
|
||||
encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii")
|
||||
print(
|
||||
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
|
||||
f"-EncodedCommand {encoded}"
|
||||
)
|
||||
PY
|
||||
)"
|
||||
|
||||
readonly REMOTE_CLEANUP_COMMAND="$(python3 - \
|
||||
"${REMOTE_TRANSPORT_PATH_WINDOWS}" <<'PY'
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import sys
|
||||
|
||||
path = sys.argv[1].replace("'", "''")
|
||||
script = rf'''$ErrorActionPreference = "Stop"
|
||||
$payloadPath = '{path}'
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
Remove-Item -LiteralPath $payloadPath -Force
|
||||
}}
|
||||
if (Test-Path -LiteralPath $payloadPath) {{
|
||||
throw "remote_transport_payload_cleanup_failed"
|
||||
}}'''
|
||||
encoded = base64.b64encode(script.encode("utf-16le")).decode("ascii")
|
||||
print(
|
||||
"powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass "
|
||||
f"-EncodedCommand {encoded}"
|
||||
)
|
||||
PY
|
||||
)"
|
||||
|
||||
cleanup_remote_payload() {
|
||||
timeout --signal=TERM --kill-after=5 30s \
|
||||
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_CLEANUP_COMMAND}" </dev/null \
|
||||
>/dev/null 2>&1
|
||||
}
|
||||
|
||||
if [[ "${MODE}" == "check" ]]; then
|
||||
set +e
|
||||
timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
|
||||
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
|
||||
status=$?
|
||||
set -e
|
||||
exit "${status}"
|
||||
fi
|
||||
|
||||
set +e
|
||||
timeout --signal=TERM --kill-after=5 30s \
|
||||
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_PREPARE_COMMAND}" </dev/null
|
||||
prepare_status=$?
|
||||
set -e
|
||||
if [[ "${prepare_status}" -ne 0 ]]; then
|
||||
printf 'agent99_transport_error=ephemeral_payload_prepare_failed\n' >&2
|
||||
exit "${prepare_status}"
|
||||
fi
|
||||
|
||||
set +e
|
||||
timeout --signal=TERM --kill-after=10 120s \
|
||||
scp "${SCP_OPTIONS[@]}" "${BOOTSTRAP_PAYLOAD_PATH}" \
|
||||
"${TARGET}:${REMOTE_TRANSPORT_PATH_SCP}"
|
||||
copy_status=$?
|
||||
set -e
|
||||
if [[ "${copy_status}" -ne 0 ]]; then
|
||||
cleanup_remote_payload || true
|
||||
printf 'agent99_transport_error=ephemeral_payload_copy_failed\n' >&2
|
||||
exit "${copy_status}"
|
||||
fi
|
||||
|
||||
set +e
|
||||
timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \
|
||||
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}"
|
||||
ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_FILE_COMMAND}" </dev/null
|
||||
status=$?
|
||||
cleanup_remote_payload
|
||||
cleanup_status=$?
|
||||
set -e
|
||||
if [[ "${cleanup_status}" -ne 0 ]]; then
|
||||
printf 'agent99_transport_error=ephemeral_payload_cleanup_verifier_failed\n' >&2
|
||||
if [[ "${status}" -eq 0 ]]; then
|
||||
exit 74
|
||||
fi
|
||||
fi
|
||||
exit "${status}"
|
||||
|
||||
@@ -41,6 +41,7 @@ def _base_case() -> dict[str, Any]:
|
||||
"selfHealthPerformanceIssueCount": 0,
|
||||
"staleAlertExecutionCount": 0,
|
||||
"alertIncomingCount": 0,
|
||||
"staleAlertIncomingCount": 0,
|
||||
"pendingCommandCount": 0,
|
||||
}
|
||||
|
||||
@@ -82,6 +83,7 @@ $parameters = @{{
|
||||
SelfHealthPerformanceIssueCount = [int]$case.selfHealthPerformanceIssueCount
|
||||
StaleAlertExecutionCount = [int]$case.staleAlertExecutionCount
|
||||
AlertIncomingCount = [int]$case.alertIncomingCount
|
||||
StaleAlertIncomingCount = [int]$case.staleAlertIncomingCount
|
||||
PendingCommandCount = [int]$case.pendingCommandCount
|
||||
}}
|
||||
Get-AgentLivePreflightDecision @parameters |
|
||||
@@ -106,6 +108,7 @@ def test_warning_receipt_is_visible_without_weakening_integrity_blockers() -> No
|
||||
|
||||
assert '$warningReasons += "self_health_warning"' in decision
|
||||
assert '$warningReasons += "performance_warning"' in decision
|
||||
assert '$warningReasons += "fresh_alert_pending_next_inbox_tick"' in decision
|
||||
assert '"critical" { $blockingReasons += "self_health_critical" }' in decision
|
||||
assert 'default { $blockingReasons += "self_health_unclassified" }' in decision
|
||||
assert '$blockingReasons += "runtime_manifest_identity_invalid"' in decision
|
||||
@@ -172,6 +175,27 @@ def test_decision_behavior_when_powershell_is_available() -> None:
|
||||
("sre_alert_relay_not_listening",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(alertIncomingCount=1),
|
||||
True,
|
||||
(),
|
||||
("fresh_alert_pending_next_inbox_tick",),
|
||||
),
|
||||
(
|
||||
_with_overrides(
|
||||
alertIncomingCount=1,
|
||||
staleAlertIncomingCount=1,
|
||||
),
|
||||
False,
|
||||
("pending_work_before_reboot",),
|
||||
(),
|
||||
),
|
||||
(
|
||||
_with_overrides(pendingCommandCount=1),
|
||||
False,
|
||||
("pending_work_before_reboot",),
|
||||
(),
|
||||
),
|
||||
)
|
||||
|
||||
for case, expected_eligible, expected_blockers, expected_warnings in cases:
|
||||
|
||||
@@ -89,6 +89,10 @@ def test_sender_is_fixed_to_110_to_windows99_public_key_transport() -> None:
|
||||
assert "source_host_is_not_110" in source
|
||||
assert '"sourceHostAlias": "110"' in source
|
||||
assert '"transport": "110_to_99_ssh_publickey"' in source
|
||||
assert '"ephemeral_file_delete_before_receiver"' in source
|
||||
assert '"transportPayloadContainsSecrets": False' in source
|
||||
assert 'scp "${SCP_OPTIONS[@]}"' in source
|
||||
assert "REMOTE_CLEANUP_COMMAND" in source
|
||||
assert "-o BatchMode=yes" in source
|
||||
assert "-o PreferredAuthentications=publickey" in source
|
||||
assert "-o PubkeyAuthentication=yes" in source
|
||||
@@ -148,7 +152,18 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data()
|
||||
assert "IO.Compression.GzipStream" in source
|
||||
assert 'decoder.encode("utf-16le")' in source
|
||||
assert "-EncodedCommand" in source
|
||||
assert "[Console]::In.ReadToEnd()" in source
|
||||
assert "[Console]::In.ReadToEnd()" not in source
|
||||
assert "$reader.ReadToEnd()" in source
|
||||
assert "__AWOOOI_AGENT99_REMOTE_BOOTSTRAP_END_V1__" in source
|
||||
assert "[Console]::In.ReadLine()" in source
|
||||
assert "remote_bootstrap_input_limit_exceeded" in source
|
||||
assert "remote_bootstrap_sentinel_missing" in source
|
||||
assert "line_width = 65536" in source
|
||||
assert "len(payload_lines) > 8" in source
|
||||
assert '"\\n".join(payload_lines)' in source
|
||||
assert "remote_bootstrap_payload_line_contract_failed" in source
|
||||
assert "$lineCount -gt 8" in source
|
||||
assert "$charCount -gt 1048576" in source
|
||||
assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source
|
||||
assert "rawEnvelope" not in source
|
||||
assert "sshpass" not in source
|
||||
@@ -253,6 +268,11 @@ def test_receiver_has_single_flight_idempotency_and_no_secret_receipt_boundary()
|
||||
assert "tokenValueRead = $false" in source
|
||||
assert "environmentSecretRead = $false" in source
|
||||
assert "rawEnvelopeStored = $false" in source
|
||||
assert "transportPayloadMode" in source
|
||||
assert "transientTransportPayloadStored" in source
|
||||
assert "transientTransportPayloadDeletedBeforeReceiver" in source
|
||||
assert "transportPayloadContainsSecrets" in source
|
||||
assert "TransportPayloadDeletedBeforeReceiver" in source
|
||||
assert "$env:" not in source
|
||||
assert "GetEnvironmentVariable" not in source
|
||||
assert (
|
||||
@@ -502,7 +522,8 @@ def test_receiver_scopes_deploy_health_separately_from_external_asset_health() -
|
||||
assert '$_ -ne "self_health_not_ok"' in preflight
|
||||
assert '[int]$summary.healthyTaskCount -eq [int]$summary.requiredTaskCount' in preflight
|
||||
assert '[int]$summary.requiredEvidenceFailureCount -eq 0' in preflight
|
||||
assert '[int]$summary.alertIncomingCount -eq 0' in preflight
|
||||
assert '[int]$summary.alertIncomingStaleCount -eq 0' in preflight
|
||||
assert '[int]$summary.alertIncomingCount -eq 0' not in preflight
|
||||
assert '[int]$summary.alertRunningCount -eq 0' in preflight
|
||||
assert '[int]$summary.pendingCommandCount -eq 0' in preflight
|
||||
assert '$runtimePlaneBlockingReasons.Count -eq 0' in preflight
|
||||
@@ -640,4 +661,108 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport(
|
||||
assert "Administrator@192.168.0.99" in args
|
||||
assert str(identity) in args
|
||||
payload_bytes = int(bytes_path.read_text(encoding="utf-8"))
|
||||
assert 50_000 < payload_bytes < 500_000
|
||||
assert 50_000 < payload_bytes < 1_048_576
|
||||
|
||||
|
||||
def test_apply_uses_ephemeral_file_transport_and_cleans_up_without_ssh_stdin(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
fake_bin = tmp_path / "bin"
|
||||
fake_bin.mkdir()
|
||||
ssh_count_path = tmp_path / "ssh-count.txt"
|
||||
ssh_stdin_path = tmp_path / "ssh-stdin-bytes.txt"
|
||||
scp_args_path = tmp_path / "scp-args.txt"
|
||||
known_hosts = tmp_path / "known_hosts"
|
||||
identity = tmp_path / "existing_identity"
|
||||
known_hosts.write_text("fixed-host-key-placeholder\n", encoding="utf-8")
|
||||
identity.write_text("identity-content-is-not-read-by-wrapper\n", encoding="utf-8")
|
||||
|
||||
fake_hostname = fake_bin / "hostname"
|
||||
fake_hostname.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"[[ \"${1:-}\" == '-I' ]] || exit 65\n"
|
||||
"printf '%s\\n' '192.168.0.110 127.0.0.1'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_ssh = fake_bin / "ssh"
|
||||
fake_ssh.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
f"count=$(cat {shlex.quote(str(ssh_count_path))} 2>/dev/null || printf '0')\n"
|
||||
"count=$((count + 1))\n"
|
||||
f"printf '%s' \"$count\" >{shlex.quote(str(ssh_count_path))}\n"
|
||||
f"wc -c | tr -d ' ' >>{shlex.quote(str(ssh_stdin_path))}\n"
|
||||
"if [[ \"$count\" -eq 2 ]]; then\n"
|
||||
" printf '%s\\n' '{\"status\":\"deployed_verified\"}'\n"
|
||||
"fi\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_scp = fake_bin / "scp"
|
||||
fake_scp.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
f"printf '%s\\n' \"$@\" >{shlex.quote(str(scp_args_path))}\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
fake_timeout = fake_bin / "timeout"
|
||||
fake_timeout.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"shift 3\n"
|
||||
"exec \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
real_git = shutil.which("git")
|
||||
assert real_git is not None
|
||||
fake_git = fake_bin / "git"
|
||||
fake_git.write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"case \" $* \" in\n"
|
||||
" *\" remote get-url origin \"*) "
|
||||
"printf '%s\\n' 'https://gitea.wooo.work/wooo/awoooi.git'; exit 0 ;;\n"
|
||||
" *\" fetch --quiet --no-tags origin \"*) exit 0 ;;\n"
|
||||
" *\" rev-parse --verify refs/remotes/origin/main^{commit} \"*) "
|
||||
"printf '%040d\\n' 0; exit 0 ;;\n"
|
||||
" *\" cat-file -e \"*) exit 0 ;;\n"
|
||||
" *\" diff --quiet \"*) exit 0 ;;\n"
|
||||
"esac\n"
|
||||
f"exec {shlex.quote(real_git)} \"$@\"\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
for executable in (
|
||||
fake_hostname,
|
||||
fake_ssh,
|
||||
fake_scp,
|
||||
fake_timeout,
|
||||
fake_git,
|
||||
):
|
||||
executable.chmod(0o700)
|
||||
|
||||
environment = os.environ.copy()
|
||||
environment["PATH"] = f"{fake_bin}:{environment['PATH']}"
|
||||
result = subprocess.run(
|
||||
[
|
||||
"bash",
|
||||
str(SENDER),
|
||||
"--apply",
|
||||
"--trace-id",
|
||||
"trace-fixture",
|
||||
"--run-id",
|
||||
"run-fixture",
|
||||
"--work-item-id",
|
||||
"AIA-P0-006",
|
||||
"--known-hosts-file",
|
||||
str(known_hosts),
|
||||
"--identity-file",
|
||||
str(identity),
|
||||
],
|
||||
check=False,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=environment,
|
||||
)
|
||||
|
||||
assert result.returncode == 0, result.stderr
|
||||
assert '"status":"deployed_verified"' in result.stdout
|
||||
assert ssh_count_path.read_text(encoding="utf-8") == "3"
|
||||
assert ssh_stdin_path.read_text(encoding="utf-8").splitlines() == ["0", "0", "0"]
|
||||
scp_args = scp_args_path.read_text(encoding="utf-8")
|
||||
assert "Administrator@192.168.0.99:C:/Wooo/Agent99/deploy/transport-inbox/" in scp_args
|
||||
assert "agent99-bootstrap-" in scp_args
|
||||
|
||||
Reference in New Issue
Block a user