diff --git a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh index dd125e9c4..3bc93502c 100755 --- a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh +++ b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh @@ -3,6 +3,9 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" MODE="check" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" CONTROL_HOST="${CONTROL_HOST:-wooo@192.168.0.110}" CONTROL_PUBLIC_KEY_PATH="${CONTROL_PUBLIC_KEY_PATH:-/home/wooo/.ssh/id_ed25519.pub}" @@ -22,8 +25,8 @@ SSH_OPTS=( usage() { cat <<'USAGE' Usage: - install-macos111-host-boot-readback.sh --check - install-macos111-host-boot-readback.sh --apply + install-macos111-host-boot-readback.sh --check [controlled identity] + install-macos111-host-boot-readback.sh --apply --trace-id ID --run-id ID --work-item-id ID Installs a no-secret Darwin boot and performance readback and authorizes host 110's existing public key with a forced command. The key cannot open a shell, @@ -31,6 +34,18 @@ request a PTY, or open forwarding. USAGE } +require_value() { + local flag="$1" value="${2:-}" + if [ -z "$value" ]; then + echo "BLOCKER missing_value=$flag" >&2 + exit 64 + fi +} + +safe_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]] +} + while [ "$#" -gt 0 ]; do case "$1" in --check) @@ -39,6 +54,21 @@ while [ "$#" -gt 0 ]; do --apply) MODE="apply" ;; + --trace-id) + require_value "$1" "${2:-}" + TRACE_ID="$2" + shift + ;; + --run-id) + require_value "$1" "${2:-}" + RUN_ID="$2" + shift + ;; + --work-item-id) + require_value "$1" "${2:-}" + WORK_ITEM_ID="$2" + shift + ;; -h|--help) usage exit 0 @@ -52,6 +82,27 @@ while [ "$#" -gt 0 ]; do shift done +identity_count=0 +[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1)) +if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER incomplete_control_identity" >&2 + exit 64 +fi +if [ "$identity_count" -eq 3 ]; then + for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do + if ! safe_identity "$identity"; then + echo "BLOCKER unsafe_control_identity" >&2 + exit 64 + fi + done +fi +if [ "$MODE" = "apply" ] && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER controlled_identity_required" >&2 + exit 64 +fi + if [ ! -f "$LOCAL_SCRIPT" ]; then echo "BLOCKER macos_boot_readback_source_missing" exit 66 @@ -72,6 +123,9 @@ check() { local direct_output control_output echo "AWOOOI_MACOS111_BOOT_READBACK=1" echo "MODE=check" + echo "TRACE_ID=${TRACE_ID:-none}" + echo "RUN_ID=${RUN_ID:-none}" + echo "WORK_ITEM_ID=${WORK_ITEM_ID:-none}" if ! direct_output="$(ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "'$REMOTE_SCRIPT'")"; then echo "MACOS111_READBACK_SCRIPT_READY=0" echo "BLOCKER macos111_readback_script_unavailable" @@ -99,12 +153,84 @@ check() { echo "MACOS111_CONTROL_PATH_READY=1" } +rollback_apply() { + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ + "TRACE_ID='$TRACE_ID' RUN_ID='$RUN_ID' WORK_ITEM_ID='$WORK_ITEM_ID' REMOTE_SCRIPT='$REMOTE_SCRIPT' bash -s" <<'REMOTE_ROLLBACK' +set -euo pipefail + +auth_file="$HOME/.ssh/authorized_keys" +backup_dir="$HOME/.ssh/awoooi-backups/$RUN_ID" +rollback_verified=1 + +if [ -f "$backup_dir/authorized_keys" ]; then + cp -p "$backup_dir/authorized_keys" "$auth_file" + cmp -s "$backup_dir/authorized_keys" "$auth_file" || rollback_verified=0 +elif [ -f "$backup_dir/authorized_keys.missing" ]; then + rm -f "$auth_file" + [ ! -e "$auth_file" ] || rollback_verified=0 +else + rollback_verified=0 +fi + +if [ -f "$backup_dir/macos-host-boot-readback.sh" ]; then + cp -p "$backup_dir/macos-host-boot-readback.sh" "$REMOTE_SCRIPT" + cmp -s "$backup_dir/macos-host-boot-readback.sh" "$REMOTE_SCRIPT" || rollback_verified=0 +elif [ -f "$backup_dir/macos-host-boot-readback.sh.missing" ]; then + rm -f "$REMOTE_SCRIPT" + [ ! -e "$REMOTE_SCRIPT" ] || rollback_verified=0 +else + rollback_verified=0 +fi + +receipt="$backup_dir/rollback-receipt.txt" +receipt_tmp="${receipt}.tmp.$$" +[ ! -e "$receipt" ] || { + echo "BLOCKER rollback_receipt_already_exists" + exit 65 +} +terminal="rollback_unverified" +[ "$rollback_verified" -eq 1 ] && terminal="rolled_back_verified" +printf 'schema_version=macos111_readback_rollback_receipt_v1 trace_id=%s run_id=%s work_item_id=%s terminal=%s rollback_verified=%s secret_value_read=0 private_key_value_read=0\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$terminal" "$rollback_verified" >"$receipt_tmp" +chmod 0600 "$receipt_tmp" +mv "$receipt_tmp" "$receipt" +echo "ROLLBACK_RECEIPT=$receipt" +echo "ROLLBACK_VERIFIED=$rollback_verified" +[ "$rollback_verified" -eq 1 ] +REMOTE_ROLLBACK +} + +write_verified_receipt() { + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ + "TRACE_ID='$TRACE_ID' RUN_ID='$RUN_ID' WORK_ITEM_ID='$WORK_ITEM_ID' REMOTE_SCRIPT='$REMOTE_SCRIPT' bash -s" <<'REMOTE_RECEIPT' +set -euo pipefail + +backup_dir="$HOME/.ssh/awoooi-backups/$RUN_ID" +receipt="$backup_dir/verifier-receipt.txt" +receipt_tmp="${receipt}.tmp.$$" +[ -d "$backup_dir" ] || { + echo "BLOCKER apply_backup_missing" + exit 65 +} +[ ! -e "$receipt" ] || { + echo "BLOCKER verifier_receipt_already_exists" + exit 65 +} +script_sha256="$(shasum -a 256 "$REMOTE_SCRIPT" | awk '{print $1}')" +printf 'schema_version=macos111_readback_apply_receipt_v1 trace_id=%s run_id=%s work_item_id=%s terminal=verified_ready script_sha256=%s rollback_available=1 secret_value_read=0 private_key_value_read=0\n' \ + "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$script_sha256" >"$receipt_tmp" +chmod 0600 "$receipt_tmp" +mv "$receipt_tmp" "$receipt" +echo "CONTROLLED_APPLY_RECEIPT=$receipt" +echo "CONTROLLED_APPLY_TERMINAL=verified_ready" +REMOTE_RECEIPT +} + if [ "$MODE" = "check" ]; then check exit $? fi -RUN_ID="macos111-boot-readback-$(date +%Y%m%dT%H%M%S%z)-$$" tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/macos111-readback.XXXXXX")" trap 'rm -rf "$tmp_dir"' EXIT @@ -113,23 +239,50 @@ scp -q "${SSH_OPTS[@]}" \ ssh-keygen -lf "$tmp_dir/control.pub" >/dev/null remote_stage="/Users/ooo/.cache/${RUN_ID}" -ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "install -d -m 0700 '$remote_stage'" -scp -q "${SSH_OPTS[@]}" "$LOCAL_SCRIPT" "$tmp_dir/control.pub" \ - "${TARGET_HOST}:${remote_stage}/" +if ! preflight_output="$(ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ + "if [ -e '$remote_stage' ] || [ -e \"\$HOME/.ssh/awoooi-backups/$RUN_ID\" ]; then echo CONTROLLED_RUN_EXISTS; exit 65; fi; echo CONTROLLED_RUN_READY" 2>&1)"; then + if grep -q '^CONTROLLED_RUN_EXISTS$' <<<"$preflight_output"; then + echo "BLOCKER controlled_run_already_exists" + exit 65 + fi + echo "BLOCKER macos111_apply_preflight_transport_failed" + exit 76 +fi +if [ "$preflight_output" != "CONTROLLED_RUN_READY" ]; then + echo "BLOCKER macos111_apply_preflight_unverified" + exit 76 +fi +if ! ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "install -d -m 0700 '$remote_stage'"; then + echo "BLOCKER macos111_stage_create_failed" + exit 76 +fi +if ! scp -q "${SSH_OPTS[@]}" "$LOCAL_SCRIPT" "$tmp_dir/control.pub" \ + "${TARGET_HOST}:${remote_stage}/"; then + ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "rm -rf '$remote_stage'" || true + echo "BLOCKER macos111_stage_transfer_failed" + echo "NO_WRITE_TERMINAL=staging_failed_cleanup_attempted" + exit 76 +fi echo "AWOOOI_MACOS111_BOOT_READBACK=1" echo "MODE=apply" +echo "TRACE_ID=$TRACE_ID" echo "RUN_ID=$RUN_ID" +echo "WORK_ITEM_ID=$WORK_ITEM_ID" echo "LOCAL_SCRIPT_SHA256=$(shasum -a 256 "$LOCAL_SCRIPT" | awk '{print $1}')" echo "CONTROL_KEY_FINGERPRINT=$(ssh-keygen -lf "$tmp_dir/control.pub" | awk '{print $2}')" -ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ - "RUN_ID='$RUN_ID' REMOTE_STAGE='$remote_stage' REMOTE_SCRIPT='$REMOTE_SCRIPT' bash -s" <<'REMOTE' +if ! ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ + "TRACE_ID='$TRACE_ID' RUN_ID='$RUN_ID' WORK_ITEM_ID='$WORK_ITEM_ID' REMOTE_STAGE='$remote_stage' REMOTE_SCRIPT='$REMOTE_SCRIPT' bash -s" <<'REMOTE' set -euo pipefail auth_dir="$HOME/.ssh" auth_file="$auth_dir/authorized_keys" backup_dir="$auth_dir/awoooi-backups/$RUN_ID" +[ ! -e "$backup_dir" ] || { + echo "BLOCKER controlled_run_already_exists" + exit 65 +} install -d -m 0700 "$auth_dir" "$backup_dir" "$(dirname "$REMOTE_SCRIPT")" if [ -f "$auth_file" ]; then cp -p "$auth_file" "$backup_dir/authorized_keys" @@ -137,6 +290,11 @@ else : >"$backup_dir/authorized_keys.missing" : >"$auth_file" fi +if [ -f "$REMOTE_SCRIPT" ]; then + cp -p "$REMOTE_SCRIPT" "$backup_dir/macos-host-boot-readback.sh" +else + : >"$backup_dir/macos-host-boot-readback.sh.missing" +fi install -m 0755 "$REMOTE_STAGE/macos-host-boot-readback.sh" "$REMOTE_SCRIPT" read -r key_type key_blob _ <"$REMOTE_STAGE/control.pub" @@ -165,5 +323,19 @@ echo "AUTHORIZED_KEYS_BACKUP=$backup_dir" echo "READBACK_SCRIPT_SHA256=$(shasum -a 256 "$REMOTE_SCRIPT" | awk '{print $1}')" echo "RESTRICTED_AUTHORIZED_KEY_INSTALLED=1" REMOTE +then + echo "BLOCKER macos111_apply_transport_failed" + rollback_apply || true + exit 76 +fi -check +if ! check; then + echo "BLOCKER macos111_post_verifier_failed" + rollback_apply || exit 77 + exit 75 +fi +if ! write_verified_receipt; then + echo "BLOCKER macos111_durable_receipt_failed" + rollback_apply || exit 77 + exit 76 +fi diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 7a57ed211..dcfec31b8 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -152,6 +152,18 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "macos111_readback_schema_incomplete" in macos_installer assert "macos111_control_readback_schema_incomplete" in macos_installer assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_installer + assert "controlled_identity_required" in macos_installer + assert "incomplete_control_identity" in macos_installer + assert "controlled_run_already_exists" in macos_installer + assert "macos111_apply_preflight_transport_failed" in macos_installer + assert "macos111_stage_transfer_failed" in macos_installer + assert "NO_WRITE_TERMINAL=staging_failed_cleanup_attempted" in macos_installer + assert "rollback_apply" in macos_installer + assert "macos-host-boot-readback.sh.missing" in macos_installer + assert "macos111_readback_apply_receipt_v1" in macos_installer + assert "macos111_readback_rollback_receipt_v1" in macos_installer + assert "macos111_post_verifier_failed" in macos_installer + assert "macos111_durable_receipt_failed" in macos_installer assert "IdentitiesOnly=yes" in macos_installer assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer assert "authorized_keys" in macos_installer @@ -208,6 +220,48 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "Stop-Computer" not in windows99 +def test_macos111_apply_requires_complete_control_identity_before_transport() -> None: + installer = ROOT / "scripts/reboot-recovery/install-macos111-host-boot-readback.sh" + missing = subprocess.run( + ["bash", str(installer), "--apply"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + partial = subprocess.run( + ["bash", str(installer), "--apply", "--trace-id", "trace-1"], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + unsafe = subprocess.run( + [ + "bash", + str(installer), + "--apply", + "--trace-id", + "trace/unsafe", + "--run-id", + "run-1", + "--work-item-id", + "AIA-P0-006", + ], + cwd=ROOT, + check=False, + capture_output=True, + text=True, + ) + + assert missing.returncode == 64 + assert "controlled_identity_required" in missing.stderr + assert partial.returncode == 64 + assert "incomplete_control_identity" in partial.stderr + assert unsafe.returncode == 64 + assert "unsafe_control_identity" in unsafe.stderr + + def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: alerts = read("ops/monitoring/alerts-unified.yml") runbook = read("docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md")