wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 4 Packages Projects Releases Wiki Activity

fix(recovery): transact host112 recovery install

Browse Source
This commit is contained in:
ogt
2026-07-14 11:45:18 +08:00
parent 6c8fdea99f
commit 316605df87
2 changed files with 707 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

677
scripts/reboot-recovery/install-host112-guest-recovery.sh
View File

@@ -1,18 +1,24 @@
#!/usr/bin/env bash
# Host 112 guest recovery installer.
# Version: 2.0
# Last modified: 2026-07-14 11:00 (Asia/Taipei)
# Version: 2.1
# Last modified: 2026-07-14 12:10 (Asia/Taipei)
# Last modified by: Codex
# Change: install and roll back the argument-regex sudo boundary required for
# correlated Wazuh manager recovery receipts.
# Change: make installation transactional, verify the Windows99 direct route
# and exact sudo policy, and keep installation separate from recovery runs.
set -euo pipefail
set -Eeuo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
MODE="check"
MODE_EXPLICIT=0
ROLLBACK_DIR=""
TRACE_ID=""
RUN_ID=""
WORK_ITEM_ID=""
TARGET_USER="${HOST112_TARGET_USER:-kali}"
CONTROL_SOURCE_ADDRESS="${HOST112_CONTROL_SOURCE_ADDRESS:-192.168.0.110}"
AGENT99_SOURCE_ADDRESS="${HOST112_AGENT99_SOURCE_ADDRESS:-192.168.0.99}"
AGENT99_EXPECTED_KEY_FINGERPRINT="${HOST112_AGENT99_EXPECTED_KEY_FINGERPRINT:-SHA256:4ccZZSe0buoWyivi2mfTMXE9LZfCmFBPr/evCzZ0nzA}"
CONTROL_PUBLIC_KEY_PATH="${HOST112_CONTROL_PUBLIC_KEY_PATH:-${SCRIPT_DIR}/control.pub}"
READINESS_SOURCE="${SCRIPT_DIR}/host112-guest-readiness.sh"
SERVICE_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.service"
@@ -22,29 +28,89 @@ INSTALLER_TARGET="/usr/local/sbin/awoooi-install-host112-guest-recovery"
SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service"
TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer"
SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99"
STATE_ROOT="/var/lib/awoooi-host112-recovery"
STATE_ROOT="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}"
MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}"
SUDOERS_ARGUMENT_REGEX='^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
SUDOERS_DRY_RUN_ARGUMENT_REGEX='^--dry-run --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$'
TRANSACTION_ACTIVE=0
ROLLBACK_IN_PROGRESS=0
ROLLBACK_ATTEMPTED=0
ROLLBACK_VERIFIED=0
INSTALL_VERIFIED=0
INSTALL_RECEIPT_WRITTEN=0
INSTALL_RECEIPT_REF="none"
INSTALL_RECEIPT_PATH=""
INSTALL_ATTEMPT_ID=""
BACKUP_DIR=""
sudoers_tmp=""
AUTH_TMP=""
TIMER_ACTIVATION_PERFORMED=0
USER_HOME=""
USER_GROUP=""
AUTH_DIR=""
AUTH_FILE=""
CONTROL_KEY_TYPE=""
CONTROL_KEY_BLOB=""
CONTROL_KEY_FINGERPRINT="unknown"
usage() {
cat <<'USAGE'
Usage:
install-host112-guest-recovery.sh --check
install-host112-guest-recovery.sh --apply
install-host112-guest-recovery.sh --rollback BACKUP_DIR
install-host112-guest-recovery.sh --apply --trace-id ID --run-id ID --work-item-id ID
install-host112-guest-recovery.sh --rollback BACKUP_DIR --trace-id ID --run-id ID --work-item-id ID
Apply installs the bounded host-112 recovery timer, an exact-command sudo rule,
and a forced-command readback key for host 110. It never reboots the host.
Apply installs and statically verifies the bounded host-112 recovery timer,
exact correlated apply/dry-run sudo rules, the host-110 forced readback key,
and the existing Windows99 direct Agent99 authorization. It starts only the
timer after all static gates pass; it never synchronously starts the recovery
oneshot, reboots the host, or changes VM power.
USAGE
}
set_mode() {
local requested="$1"
if [ "$MODE_EXPLICIT" -eq 1 ]; then
echo "BLOCKER duplicate_mode" >&2
exit 64
fi
MODE="$requested"
MODE_EXPLICIT=1
}
require_value() {
local flag="$1" value="${2:-}"
if [ -z "$value" ]; then
echo "BLOCKER missing_value=$flag" >&2
exit 64
fi
}
while [ "$#" -gt 0 ]; do
case "$1" in
--check) MODE="check" ;;
--apply) MODE="apply" ;;
--check) set_mode "check" ;;
--apply) set_mode "apply" ;;
--rollback)
MODE="rollback"
set_mode "rollback"
require_value "$1" "${2:-}"
ROLLBACK_DIR="$2"
shift
;;
--trace-id)
require_value "$1" "${2:-}"
TRACE_ID="$2"
shift
;;
--run-id)
require_value "$1" "${2:-}"
RUN_ID="$2"
shift
;;
--work-item-id)
require_value "$1" "${2:-}"
WORK_ITEM_ID="$2"
shift
ROLLBACK_DIR="${1:-}"
;;
-h|--help) usage; exit 0 ;;
*) echo "UNKNOWN_ARG=$1" >&2; usage >&2; exit 64 ;;
@@ -52,21 +118,192 @@ while [ "$#" -gt 0 ]; do
shift
done
sudo_policy_readback() {
safe_identity() {
[[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]]
}
identity_count=0
[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1))
[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1))
[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1))
if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER incomplete_control_identity" >&2
exit 64
fi
if [ "$identity_count" -eq 3 ]; then
for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do
if ! safe_identity "$identity"; then
echo "BLOCKER unsafe_control_identity" >&2
exit 64
fi
done
fi
if { [ "$MODE" = "apply" ] || [ "$MODE" = "rollback" ]; } && [ "$identity_count" -ne 3 ]; then
echo "BLOCKER installer_control_identity_required" >&2
exit 64
fi
cleanup() {
[ -z "$sudoers_tmp" ] || rm -f "$sudoers_tmp" >/dev/null 2>&1 || true
[ -z "$AUTH_TMP" ] || rm -f "$AUTH_TMP" >/dev/null 2>&1 || true
}
trap cleanup EXIT
resolve_target_user() {
USER_HOME="$(getent passwd "$TARGET_USER" 2>/dev/null | cut -d: -f6)"
USER_GROUP="$(id -gn "$TARGET_USER" 2>/dev/null || true)"
[ -n "$USER_HOME" ] || { echo "BLOCKER target_user_home_missing" >&2; return 1; }
[ -n "$USER_GROUP" ] || { echo "BLOCKER target_user_group_missing" >&2; return 1; }
AUTH_DIR="${USER_HOME}/.ssh"
AUTH_FILE="${AUTH_DIR}/authorized_keys"
}
sudo_policy_allows() {
if [ "${EUID:-$(id -u)}" -eq 0 ]; then
sudo -n -U "$TARGET_USER" -l "$@" 2>/dev/null || true
sudo -n -U "$TARGET_USER" -l "$@" >/dev/null 2>&1
else
sudo -n -l "$@" 2>/dev/null || true
sudo -n -l "$@" >/dev/null 2>&1
fi
}
sudo_policy_has_nopasswd() {
local output
output="$(sudo_policy_readback "$@")"
printf '%s\n' "$output" | grep -Fq "NOPASSWD: $READINESS_TARGET"
agent99_direct_authorization_ready() {
local key_type key_blob fingerprint
[ -f "$AUTH_FILE" ] || return 1
while IFS=$'\t' read -r key_type key_blob; do
[ -n "$key_type" ] && [ -n "$key_blob" ] || continue
[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || continue
fingerprint="$({ printf '%s %s\n' "$key_type" "$key_blob" | ssh-keygen -lf - 2>/dev/null || true; } | awk 'NR == 1 {print $2}')"
if [ "$fingerprint" = "$AGENT99_EXPECTED_KEY_FINGERPRINT" ]; then
return 0
fi
done < <(
awk -v source="$AGENT99_SOURCE_ADDRESS" '
function has_restrict(options, count, parts, i) {
count = split(options, parts, ",")
for (i = 1; i <= count; i++) {
if (parts[i] == "restrict") return 1
}
return 0
}
index($1, "from=\"" source "\"") > 0 \
&& has_restrict($1) \
&& index($1, "command=") == 0 \
&& $2 ~ /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-)/ {
print $2 "\t" $3
}
' "$AUTH_FILE"
)
return 1
}
readiness_check_no_write_ready() {
local readiness_args
readiness_args=("$READINESS_TARGET" --check)
if [ "$identity_count" -eq 3 ]; then
readiness_args+=(
--trace-id "$TRACE_ID"
--run-id "$RUN_ID"
--work-item-id "$WORK_ITEM_ID"
)
fi
if READINESS_CHECK_OUTPUT="$("${readiness_args[@]}" 2>&1)"; then
READINESS_CHECK_EXIT=0
else
READINESS_CHECK_EXIT=$?
fi
grep -Eq '(^| )schema_version=host112_guest_recovery_v2($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )mode=check($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )runtime_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )manager_runtime_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )receipt_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )ssh_port_listening=1($| )' <<<"$READINESS_CHECK_OUTPUT" \
&& grep -Eq '(^| )ssh_ready=1($| )' <<<"$READINESS_CHECK_OUTPUT"
}
refresh_gate_state() {
TARGET_FILES_READY=1
[ -x "$READINESS_TARGET" ] || TARGET_FILES_READY=0
[ -x "$INSTALLER_TARGET" ] || TARGET_FILES_READY=0
[ -f "$SERVICE_TARGET" ] || TARGET_FILES_READY=0
[ -f "$TIMER_TARGET" ] || TARGET_FILES_READY=0
[ -f "$SUDOERS_TARGET" ] || TARGET_FILES_READY=0
UNIT_CONTRACT_READY=0
if [ -f "$SERVICE_TARGET" ] \
&& [ -f "$TIMER_TARGET" ] \
&& grep -Fq 'ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer' "$SERVICE_TARGET" \
&& grep -Fq 'TimeoutStartSec=480' "$SERVICE_TARGET" \
&& grep -Fq 'Unit=awoooi-host112-guest-recovery.service' "$TIMER_TARGET"; then
UNIT_CONTRACT_READY=1
fi
SUDOERS_VALID=0
if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then
SUDOERS_VALID=1
fi
SUDO_CORRELATED_APPLY_NOPASSWD=0
if sudo_policy_allows "$READINESS_TARGET" --apply \
--trace-id host112-policy-check \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
SUDO_CORRELATED_APPLY_NOPASSWD=1
fi
SUDO_CORRELATED_DRY_RUN_NOPASSWD=0
if sudo_policy_allows "$READINESS_TARGET" --dry-run \
--trace-id host112-policy-check \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
SUDO_CORRELATED_DRY_RUN_NOPASSWD=1
fi
SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0
if sudo_policy_allows "$READINESS_TARGET" --apply; then
SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1
fi
SUDO_UNSAFE_IDENTITY_NOPASSWD=0
if sudo_policy_allows "$READINESS_TARGET" --apply \
--trace-id 'unsafe value' \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
SUDO_UNSAFE_IDENTITY_NOPASSWD=1
fi
SUDO_POLICY_READY=0
if [ "$SUDOERS_VALID" -eq 1 ] \
&& [ "$SUDO_CORRELATED_APPLY_NOPASSWD" -eq 1 ] \
&& [ "$SUDO_CORRELATED_DRY_RUN_NOPASSWD" -eq 1 ] \
&& [ "$SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" -eq 0 ] \
&& [ "$SUDO_UNSAFE_IDENTITY_NOPASSWD" -eq 0 ]; then
SUDO_POLICY_READY=1
fi
AGENT99_99_DIRECT_ROUTE_READY=0
if agent99_direct_authorization_ready; then
AGENT99_99_DIRECT_ROUTE_READY=1
fi
READINESS_NO_WRITE_READY=0
READINESS_CHECK_EXIT=-1
READINESS_CHECK_OUTPUT=""
if [ -x "$READINESS_TARGET" ] && readiness_check_no_write_ready; then
READINESS_NO_WRITE_READY=1
fi
TIMER_ENABLED="$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
TIMER_ACTIVE="$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
TIMER_ENABLED="${TIMER_ENABLED:-unknown}"
TIMER_ACTIVE="${TIMER_ACTIVE:-unknown}"
}
check() {
local gate_ready=0 path
refresh_gate_state
if [ "$TARGET_FILES_READY" -eq 1 ] \
&& [ "$UNIT_CONTRACT_READY" -eq 1 ] \
&& [ "$SUDO_POLICY_READY" -eq 1 ] \
&& [ "$AGENT99_99_DIRECT_ROUTE_READY" -eq 1 ] \
&& [ "$READINESS_NO_WRITE_READY" -eq 1 ] \
&& [ "$TIMER_ENABLED" = "enabled" ] \
&& [ "$TIMER_ACTIVE" = "active" ]; then
gate_ready=1
fi
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
echo "MODE=check"
for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET"; do
@@ -76,37 +313,24 @@ check() {
echo "TARGET path=$path present=0 sha256=missing"
fi
done
echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)"
if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then
echo "SUDOERS_VALID=1"
else
echo "SUDOERS_VALID=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
--trace-id host112-policy-check \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
echo "SUDO_CORRELATED_APPLY_NOPASSWD=1"
else
echo "SUDO_CORRELATED_APPLY_NOPASSWD=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply; then
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1"
else
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0"
fi
if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \
--trace-id 'unsafe value' \
--run-id host112-policy-check \
--work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=1"
else
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=0"
fi
if [ -x "$READINESS_TARGET" ]; then
"$READINESS_TARGET" --check || true
fi
echo "TARGET_FILES_READY=$TARGET_FILES_READY"
echo "UNIT_CONTRACT_READY=$UNIT_CONTRACT_READY"
echo "TIMER_ENABLED=$TIMER_ENABLED"
echo "TIMER_ACTIVE=$TIMER_ACTIVE"
echo "SUDOERS_VALID=$SUDOERS_VALID"
echo "SUDO_CORRELATED_APPLY_NOPASSWD=$SUDO_CORRELATED_APPLY_NOPASSWD"
echo "SUDO_CORRELATED_DRY_RUN_NOPASSWD=$SUDO_CORRELATED_DRY_RUN_NOPASSWD"
echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=$SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD"
echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=$SUDO_UNSAFE_IDENTITY_NOPASSWD"
echo "SUDO_POLICY_READY=$SUDO_POLICY_READY"
echo "AGENT99_99_DIRECT_ROUTE_READY=$AGENT99_99_DIRECT_ROUTE_READY"
echo "AGENT99_99_DIRECT_RUNTIME_PROBE=not_started"
echo "READINESS_NO_WRITE_READY=$READINESS_NO_WRITE_READY"
echo "READINESS_CHECK_EXIT=$READINESS_CHECK_EXIT"
echo "CHECK_GATE_READY=$gate_ready"
echo "INSTALLER_SYNCHRONOUS_RECOVERY_START=0"
echo "RECOVERY_EXECUTION_STATUS=separate_runtime_run_required"
[ "$gate_ready" -eq 1 ]
}
require_root() {
@@ -128,105 +352,318 @@ backup_path() {
fi
}
create_backup() {
local run_slug
run_slug="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')"
BACKUP_DIR="${STATE_ROOT}/backups/host112-installer-${run_slug}-${INSTALL_ATTEMPT_ID}"
install -d -m 0750 "$BACKUP_DIR"
: >"${BACKUP_DIR}/manifest.tsv"
for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$AUTH_FILE"; do
backup_path "$path" "$BACKUP_DIR"
done
systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 \
&& touch "$BACKUP_DIR/timer_was_enabled" || true
systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 \
&& touch "$BACKUP_DIR/timer_was_active" || true
}
restore_from_backup() {
local backup_dir="$1" state path ok=1
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true
if systemctl is-active awoooi-host112-guest-recovery.service >/dev/null 2>&1; then
timeout 30 systemctl stop awoooi-host112-guest-recovery.service >/dev/null 2>&1 || ok=0
fi
systemctl disable awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true
while IFS=$'\t' read -r state path; do
[ -n "$path" ] || continue
if [ "$state" = "present" ]; then
install -d "$(dirname "$path")" || ok=0
cp -a "$backup_dir/rootfs/${path#/}" "$path" || ok=0
elif [ "$state" = "missing" ]; then
rm -f "$path" || ok=0
else
ok=0
fi
done <"$backup_dir/manifest.tsv"
systemctl daemon-reload >/dev/null 2>&1 || ok=0
if [ -f "$backup_dir/timer_was_enabled" ]; then
systemctl enable awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || ok=0
fi
if [ -f "$backup_dir/timer_was_active" ]; then
systemctl start awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || ok=0
fi
[ "$ok" -eq 1 ]
}
verify_rollback() {
local backup_dir="$1" state path
while IFS=$'\t' read -r state path; do
[ -n "$path" ] || continue
if [ "$state" = "present" ]; then
[ -e "$path" ] || return 1
cmp -s "$backup_dir/rootfs/${path#/}" "$path" || return 1
elif [ "$state" = "missing" ]; then
[ ! -e "$path" ] || return 1
else
return 1
fi
done <"$backup_dir/manifest.tsv"
if [ -f "$SUDOERS_TARGET" ]; then
visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1 || return 1
fi
if [ -f "$backup_dir/timer_was_enabled" ]; then
systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1
else
! systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1
fi
if [ -f "$backup_dir/timer_was_active" ]; then
systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1
else
! systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1
fi
}
write_install_receipt() {
local terminal="$1" install_verified="$2" rollback_attempted="$3" rollback_verified="$4" error_exit="$5"
local receipt_dir receipt_tmp generated_at readiness_sha installer_sha
receipt_dir="${STATE_ROOT}/install-receipts"
install -d -m 0750 "$receipt_dir"
if [ -z "$INSTALL_RECEIPT_PATH" ]; then
INSTALL_RECEIPT_PATH="${receipt_dir}/install-${RUN_ID//:/_}-${INSTALL_ATTEMPT_ID}.txt"
fi
generated_at="$(date --iso-8601=seconds)"
readiness_sha="$(sha256sum "$READINESS_TARGET" 2>/dev/null | awk '{print $1}')"
installer_sha="$(sha256sum "$INSTALLER_TARGET" 2>/dev/null | awk '{print $1}')"
readiness_sha="${readiness_sha:-missing}"
installer_sha="${installer_sha:-missing}"
receipt_tmp="${INSTALL_RECEIPT_PATH}.tmp.$$"
printf '%s\n' \
"schema_version=host112_guest_recovery_install_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID terminal=$terminal install_verified=$install_verified installer_runtime_write_performed=1 manager_runtime_write_performed=0 static_post_gate_ready=${STATIC_POST_GATE_READY:-0} check_gate_ready=${CHECK_GATE_READY:-0} sudo_policy_ready=${SUDO_POLICY_READY:-0} agent99_99_direct_route_ready=${AGENT99_99_DIRECT_ROUTE_READY:-0} agent99_99_direct_runtime_probe=not_started timer_enabled=${TIMER_ENABLED:-unknown} timer_active=${TIMER_ACTIVE:-unknown} timer_activation_performed=$TIMER_ACTIVATION_PERFORMED synchronous_recovery_start=0 recovery_execution_performed=0 recovery_run_id=none recovery_receipt_ref=none rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified error_exit=$error_exit backup_ref=$BACKUP_DIR readiness_sha256=$readiness_sha installer_sha256=$installer_sha prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" \
>"$receipt_tmp"
chmod 0640 "$receipt_tmp"
if ! mv -n "$receipt_tmp" "$INSTALL_RECEIPT_PATH" \
|| [ -e "$receipt_tmp" ]; then
rm -f "$receipt_tmp" >/dev/null 2>&1 || true
return 1
fi
INSTALL_RECEIPT_WRITTEN=1
INSTALL_RECEIPT_REF="host112-install:${RUN_ID}:${INSTALL_ATTEMPT_ID}"
}
handle_apply_failure() {
local original_exit="${1:-1}" terminal="install_failed_rollback_failed"
trap - ERR INT TERM
set +e
if [ "$ROLLBACK_IN_PROGRESS" -eq 1 ]; then
exit 74
fi
ROLLBACK_IN_PROGRESS=1
if [ "$TRANSACTION_ACTIVE" -eq 1 ] && [ -n "$BACKUP_DIR" ]; then
ROLLBACK_ATTEMPTED=1
if restore_from_backup "$BACKUP_DIR" && verify_rollback "$BACKUP_DIR"; then
ROLLBACK_VERIFIED=1
terminal="install_failed_rolled_back_verified"
fi
fi
refresh_gate_state >/dev/null 2>&1 || true
write_install_receipt "$terminal" 0 "$ROLLBACK_ATTEMPTED" "$ROLLBACK_VERIFIED" "$original_exit" >/dev/null 2>&1 || true
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" >&2
echo "MODE=apply_failed" >&2
echo "TERMINAL=$terminal" >&2
echo "BACKUP_DIR=${BACKUP_DIR:-none}" >&2
echo "ROLLBACK_ATTEMPTED=$ROLLBACK_ATTEMPTED" >&2
echo "ROLLBACK_VERIFIED=$ROLLBACK_VERIFIED" >&2
echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF" >&2
echo "SYNCHRONOUS_RECOVERY_START=0" >&2
if [ "$ROLLBACK_VERIFIED" -ne 1 ]; then
exit 74
fi
[ "$original_exit" -ne 0 ] || original_exit=1
exit "$original_exit"
}
static_source_preflight() {
local source service_state
for tool in bash cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do
command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; }
done
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do
[ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; return 1; }
done
bash -n "$READINESS_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"
grep -Fq 'ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer' "$SERVICE_SOURCE"
grep -Fq 'TimeoutStartSec=480' "$SERVICE_SOURCE"
grep -Fq 'Unit=awoooi-host112-guest-recovery.service' "$TIMER_SOURCE"
read -r CONTROL_KEY_TYPE CONTROL_KEY_BLOB _ <"$CONTROL_PUBLIC_KEY_PATH"
case "$CONTROL_KEY_TYPE" in
ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;;
*) echo "BLOCKER unsupported_control_public_key_type" >&2; return 1 ;;
esac
[[ "$CONTROL_KEY_BLOB" =~ ^[A-Za-z0-9+/=]+$ ]] \
|| { echo "BLOCKER invalid_control_public_key_blob" >&2; return 1; }
CONTROL_KEY_FINGERPRINT="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk 'NR == 1 {print $2}')"
[ -n "$CONTROL_KEY_FINGERPRINT" ] || { echo "BLOCKER control_public_key_fingerprint_missing" >&2; return 1; }
agent99_direct_authorization_ready \
|| { echo "BLOCKER agent99_99_direct_authorization_not_ready" >&2; return 1; }
service_state="$(systemctl is-active awoooi-host112-guest-recovery.service 2>/dev/null || true)"
case "$service_state" in
active|activating) echo "BLOCKER host112_recovery_service_busy" >&2; return 1 ;;
esac
sudoers_tmp="$(mktemp)"
{
printf '%s ALL=(root) NOPASSWD: %s %s\n' \
"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"
printf '%s ALL=(root) NOPASSWD: %s %s\n' \
"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_DRY_RUN_ARGUMENT_REGEX"
} >"$sudoers_tmp"
visudo -cf "$sudoers_tmp" >/dev/null
}
post_install_static_gate() {
STATIC_POST_GATE_READY=0
TARGET_HASHES_MATCH=0
UNIT_STATIC_VERIFY_READY=0
if cmp -s "$READINESS_SOURCE" "$READINESS_TARGET" \
&& cmp -s "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET" \
&& cmp -s "$SERVICE_SOURCE" "$SERVICE_TARGET" \
&& cmp -s "$TIMER_SOURCE" "$TIMER_TARGET"; then
TARGET_HASHES_MATCH=1
fi
if systemd-analyze verify "$SERVICE_TARGET" "$TIMER_TARGET" >/dev/null 2>&1; then
UNIT_STATIC_VERIFY_READY=1
fi
refresh_gate_state
if [ "$TARGET_HASHES_MATCH" -eq 1 ] \
&& [ "$UNIT_STATIC_VERIFY_READY" -eq 1 ] \
&& [ "$TARGET_FILES_READY" -eq 1 ] \
&& [ "$UNIT_CONTRACT_READY" -eq 1 ] \
&& [ "$SUDO_POLICY_READY" -eq 1 ] \
&& [ "$AGENT99_99_DIRECT_ROUTE_READY" -eq 1 ] \
&& [ "$READINESS_NO_WRITE_READY" -eq 1 ] \
&& [ "$TIMER_ENABLED" = "enabled" ]; then
STATIC_POST_GATE_READY=1
fi
echo "TARGET_HASHES_MATCH=$TARGET_HASHES_MATCH"
echo "UNIT_STATIC_VERIFY_READY=$UNIT_STATIC_VERIFY_READY"
echo "STATIC_POST_GATE_READY=$STATIC_POST_GATE_READY"
[ "$STATIC_POST_GATE_READY" -eq 1 ]
}
wait_for_recovery_service_idle() {
local deadline state
deadline=$(( $(date +%s) + 30 ))
while true; do
state="$(systemctl is-active awoooi-host112-guest-recovery.service 2>/dev/null || true)"
case "$state" in
active|activating) ;;
*) return 0 ;;
esac
[ "$(date +%s)" -lt "$deadline" ] || return 1
sleep 1
done
}
resolve_target_user
if [ "$MODE" = "check" ]; then
check
exit 0
exit $?
fi
require_root
INSTALL_ATTEMPT_ID="$(date +%Y%m%dT%H%M%S%z)-$$"
if [ "$MODE" = "rollback" ]; then
case "$ROLLBACK_DIR" in
"$STATE_ROOT"/backups/*) ;;
*) echo "BLOCKER invalid_host112_rollback_dir" >&2; exit 64 ;;
esac
[ -f "$ROLLBACK_DIR/manifest.tsv" ] || { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; }
systemctl disable --now awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true
while IFS=$'\t' read -r state path; do
[ -n "$path" ] || continue
if [ "$state" = "present" ]; then
install -d "$(dirname "$path")"
cp -a "$ROLLBACK_DIR/rootfs/${path#/}" "$path"
else
rm -f "$path"
fi
done <"$ROLLBACK_DIR/manifest.tsv"
systemctl daemon-reload
if [ -f "$ROLLBACK_DIR/timer_was_enabled" ]; then
systemctl enable awoooi-host112-guest-recovery.timer >/dev/null
fi
if [ -f "$ROLLBACK_DIR/timer_was_active" ]; then
systemctl start awoooi-host112-guest-recovery.timer
[ -f "$ROLLBACK_DIR/manifest.tsv" ] \
|| { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; }
BACKUP_DIR="$ROLLBACK_DIR"
exec 9>"$MANAGER_LOCK_FILE"
flock -w 30 9 || { echo "BLOCKER host112_installer_single_flight_busy" >&2; exit 75; }
ROLLBACK_ATTEMPTED=1
if restore_from_backup "$ROLLBACK_DIR" && verify_rollback "$ROLLBACK_DIR"; then
ROLLBACK_VERIFIED=1
fi
refresh_gate_state >/dev/null 2>&1 || true
write_install_receipt \
"manual_rollback_$(if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then printf verified; else printf failed; fi)" \
0 1 "$ROLLBACK_VERIFIED" 0
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
echo "MODE=rollback"
echo "ROLLBACK_APPLIED=1"
check
exit 0
echo "ROLLBACK_APPLIED=$ROLLBACK_VERIFIED"
echo "ROLLBACK_VERIFIED=$ROLLBACK_VERIFIED"
echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF"
echo "RECOVERY_EXECUTION_PERFORMED=0"
[ "$ROLLBACK_VERIFIED" -eq 1 ]
exit $?
fi
for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH"; do
[ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; exit 66; }
done
static_source_preflight
exec 9>"$MANAGER_LOCK_FILE"
flock -w 30 9 || { echo "BLOCKER host112_installer_single_flight_busy" >&2; exit 75; }
create_backup
echo "BACKUP_DIR=$BACKUP_DIR"
TRANSACTION_ACTIVE=1
trap 'handle_apply_failure $?' ERR
trap 'handle_apply_failure 130' INT TERM
user_home="$(getent passwd "$TARGET_USER" | cut -d: -f6)"
user_group="$(id -gn "$TARGET_USER")"
[ -n "$user_home" ] || { echo "BLOCKER target_user_home_missing" >&2; exit 66; }
auth_dir="${user_home}/.ssh"
auth_file="${auth_dir}/authorized_keys"
read -r key_type key_blob _ <"$CONTROL_PUBLIC_KEY_PATH"
case "$key_type" in
ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;;
*) echo "BLOCKER unsupported_control_public_key_type" >&2; exit 65 ;;
esac
[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || { echo "BLOCKER invalid_control_public_key_blob" >&2; exit 65; }
key_fingerprint="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk '{print $2}')"
run_id="host112-guest-recovery-$(date +%Y%m%dT%H%M%S%z)-$$"
backup_dir="${STATE_ROOT}/backups/${run_id}"
install -d -m 0750 "$backup_dir"
: >"${backup_dir}/manifest.tsv"
for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$auth_file"; do
backup_path "$path" "$backup_dir"
done
systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_enabled" || true
systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_active" || true
timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1
wait_for_recovery_service_idle
install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET"
install -m 0755 "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET"
install -m 0644 "$SERVICE_SOURCE" "$SERVICE_TARGET"
install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET"
sudoers_tmp="$(mktemp)"
trap 'rm -f "$sudoers_tmp"' EXIT
printf '%s ALL=(root) NOPASSWD: %s %s\n' \
"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" >"$sudoers_tmp"
visudo -cf "$sudoers_tmp" >/dev/null
install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET"
install -d -m 0700 -o "$TARGET_USER" -g "$user_group" "$auth_dir"
if [ ! -f "$auth_file" ]; then
install -m 0600 -o "$TARGET_USER" -g "$user_group" /dev/null "$auth_file"
install -d -m 0700 -o "$TARGET_USER" -g "$USER_GROUP" "$AUTH_DIR"
if [ ! -f "$AUTH_FILE" ]; then
install -m 0600 -o "$TARGET_USER" -g "$USER_GROUP" /dev/null "$AUTH_FILE"
fi
auth_tmp="$(mktemp "${auth_file}.tmp.XXXXXX")"
awk -v blob="$key_blob" 'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' "$auth_file" >"$auth_tmp"
AUTH_TMP="$(mktemp "${AUTH_FILE}.tmp.XXXXXX")"
awk -v blob="$CONTROL_KEY_BLOB" \
'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' \
"$AUTH_FILE" >"$AUTH_TMP"
printf 'from="%s",restrict,command="%s --check" %s %s awoooi-host112-probe-110\n' \
"$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$key_type" "$key_blob" >>"$auth_tmp"
chown "$TARGET_USER:$user_group" "$auth_tmp"
chmod 0600 "$auth_tmp"
mv "$auth_tmp" "$auth_file"
"$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$CONTROL_KEY_TYPE" "$CONTROL_KEY_BLOB" >>"$AUTH_TMP"
chown "$TARGET_USER:$USER_GROUP" "$AUTH_TMP"
chmod 0600 "$AUTH_TMP"
mv "$AUTH_TMP" "$AUTH_FILE"
AUTH_TMP=""
systemctl daemon-reload
systemctl enable --now awoooi-host112-guest-recovery.timer >/dev/null
systemctl start awoooi-host112-guest-recovery.service
systemctl enable awoooi-host112-guest-recovery.timer >/dev/null
post_install_static_gate
systemctl start awoooi-host112-guest-recovery.timer
TIMER_ACTIVATION_PERFORMED=1
check
CHECK_GATE_READY=1
INSTALL_VERIFIED=1
write_install_receipt "installed_timer_activated_recovery_separate" 1 0 0 0
TRANSACTION_ACTIVE=0
trap - ERR INT TERM
echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1"
echo "MODE=apply"
echo "RUN_ID=$run_id"
echo "BACKUP_DIR=$backup_dir"
echo "CONTROL_KEY_FINGERPRINT=$key_fingerprint"
echo "TRACE_ID=$TRACE_ID"
echo "RUN_ID=$RUN_ID"
echo "WORK_ITEM_ID=$WORK_ITEM_ID"
echo "BACKUP_DIR=$BACKUP_DIR"
echo "CONTROL_KEY_FINGERPRINT=$CONTROL_KEY_FINGERPRINT"
echo "INSTALL_VERIFIED=$INSTALL_VERIFIED"
echo "INSTALL_RECEIPT_WRITTEN=$INSTALL_RECEIPT_WRITTEN"
echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF"
echo "RECOVERY_EXECUTION_PERFORMED=0"
echo "RECOVERY_RUN_ID=none"
echo "RECOVERY_RECEIPT_REF=none"
echo "RECOVERY_EXECUTION_STATUS=separate_timer_or_agent99_run_required"
echo "AGENT99_99_DIRECT_RUNTIME_PROBE=not_started"
echo "REBOOT_PERFORMED=0"
echo "VM_POWER_CHANGE_PERFORMED=0"
echo "SECRET_READ=0"
echo "RESTRICTED_CONTROL_KEY_INSTALLED=1"
check

150
scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py Normal file
View File

@@ -0,0 +1,150 @@
from __future__ import annotations
import subprocess
from pathlib import Path
ROOT = Path(__file__).resolve().parents[3]
INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh"
def source() -> str:
return INSTALLER.read_text(encoding="utf-8")
def test_installer_shell_syntax_is_valid() -> None:
result = subprocess.run(
["bash", "-n", str(INSTALLER)],
text=True,
capture_output=True,
check=False,
timeout=10,
)
assert result.returncode == 0, result.stdout + result.stderr
def test_apply_and_rollback_require_safe_control_identity() -> None:
text = source()
assert '--trace-id ID --run-id ID --work-item-id ID' in text
assert 'BLOCKER incomplete_control_identity' in text
assert 'BLOCKER unsafe_control_identity' in text
assert 'BLOCKER installer_control_identity_required' in text
assert '^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' in text
def test_sudo_policy_has_exact_correlated_apply_and_dry_run_only() -> None:
text = source()
assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in text
assert "SUDOERS_DRY_RUN_ARGUMENT_REGEX='^--dry-run --trace-id " in text
assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in text
assert (
'"$TARGET_USER" "$READINESS_TARGET" '
'"$SUDOERS_DRY_RUN_ARGUMENT_REGEX"' in text
)
assert 'SUDO_CORRELATED_APPLY_NOPASSWD' in text
assert 'SUDO_CORRELATED_DRY_RUN_NOPASSWD' in text
assert 'SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD' in text
assert 'SUDO_UNSAFE_IDENTITY_NOPASSWD' in text
assert 'SUDO_POLICY_READY' in text
def test_windows99_direct_route_is_fingerprint_bound_and_not_forced() -> None:
text = source()
route = text[
text.index("agent99_direct_authorization_ready() {") : text.index(
"readiness_check_no_write_ready() {"
)
]
assert 'HOST112_AGENT99_SOURCE_ADDRESS:-192.168.0.99' in text
assert 'HOST112_AGENT99_EXPECTED_KEY_FINGERPRINT' in text
assert 'AGENT99_99_DIRECT_ROUTE_READY' in text
assert 'index($1, "command=") == 0' in route
assert 'has_restrict($1)' in route
assert 'ssh-keygen -lf -' in route
assert 'cat "$AUTH_FILE"' not in text
assert 'from="%s",restrict,command="%s --check"' in text
assert 'CONTROL_SOURCE_ADDRESS:-192.168.0.110' in text
def test_apply_is_transactional_and_has_a_verified_rollback() -> None:
text = source()
apply_path = text[text.index("static_source_preflight\n") :]
handler = text[
text.index("handle_apply_failure() {") : text.index(
"static_source_preflight() {"
)
]
assert 'set -Eeuo pipefail' in text
assert 'TRANSACTION_ACTIVE=1' in apply_path
assert "trap 'handle_apply_failure $?' ERR" in apply_path
assert "trap 'handle_apply_failure 130' INT TERM" in apply_path
assert 'restore_from_backup "$BACKUP_DIR"' in handler
assert 'verify_rollback "$BACKUP_DIR"' in handler
assert 'ROLLBACK_VERIFIED=1' in handler
assert 'install_failed_rolled_back_verified' in handler
assert 'cmp -s "$backup_dir/rootfs/${path#/}" "$path"' in text
assert 'timer_was_enabled' in text
assert 'timer_was_active' in text
def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> None:
text = source()
apply_path = text[text.index("static_source_preflight\n") :]
static_gate = apply_path.index("post_install_static_gate\n")
timer_start = apply_path.index(
"systemctl start awoooi-host112-guest-recovery.timer"
)
final_gate = apply_path.index("check\n", timer_start)
receipt = apply_path.index(
'write_install_receipt "installed_timer_activated_recovery_separate"'
)
assert static_gate < timer_start < final_gate < receipt
assert 'systemctl enable awoooi-host112-guest-recovery.timer' in apply_path
assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text
assert 'systemctl start awoooi-host112-guest-recovery.service' not in text
assert "grep -Fq 'TimeoutStartSec=480'" in text
assert "TimeoutStartSec=600" not in text
assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text
assert 'RECOVERY_EXECUTION_PERFORMED=0' in text
def test_install_receipt_does_not_claim_a_recovery_run() -> None:
text = source()
receipt = text[
text.index("write_install_receipt() {") : text.index(
"handle_apply_failure() {"
)
]
assert 'schema_version=host112_guest_recovery_install_receipt_v1' in receipt
assert 'install_verified=$install_verified' in receipt
assert 'manager_runtime_write_performed=0' in receipt
assert 'recovery_execution_performed=0' in receipt
assert 'recovery_run_id=none' in receipt
assert 'recovery_receipt_ref=none' in receipt
assert 'agent99_99_direct_runtime_probe=not_started' in receipt
assert 'rollback_attempted=$rollback_attempted' in receipt
assert 'rollback_verified=$rollback_verified' in receipt
def test_check_gate_requires_policy_route_timer_and_no_write_readback() -> None:
text = source()
check = text[text.index("check() {") : text.index("require_root() {")]
for required in (
'SUDO_POLICY_READY',
'AGENT99_99_DIRECT_ROUTE_READY',
'READINESS_NO_WRITE_READY',
'TIMER_ENABLED',
'TIMER_ACTIVE',
'CHECK_GATE_READY',
):
assert required in check
assert 'runtime_write_performed=0' in text
assert 'manager_runtime_write_performed=0' in text
assert 'receipt_write_performed=0' in text