diff --git a/scripts/reboot-recovery/install-host112-guest-recovery.sh b/scripts/reboot-recovery/install-host112-guest-recovery.sh index f5ca4c85c..38a750107 100755 --- a/scripts/reboot-recovery/install-host112-guest-recovery.sh +++ b/scripts/reboot-recovery/install-host112-guest-recovery.sh @@ -1,18 +1,24 @@ #!/usr/bin/env bash # Host 112 guest recovery installer. -# Version: 2.0 -# Last modified: 2026-07-14 11:00 (Asia/Taipei) +# Version: 2.1 +# Last modified: 2026-07-14 12:10 (Asia/Taipei) # Last modified by: Codex -# Change: install and roll back the argument-regex sudo boundary required for -# correlated Wazuh manager recovery receipts. +# Change: make installation transactional, verify the Windows99 direct route +# and exact sudo policy, and keep installation separate from recovery runs. -set -euo pipefail +set -Eeuo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" MODE="check" +MODE_EXPLICIT=0 ROLLBACK_DIR="" +TRACE_ID="" +RUN_ID="" +WORK_ITEM_ID="" TARGET_USER="${HOST112_TARGET_USER:-kali}" CONTROL_SOURCE_ADDRESS="${HOST112_CONTROL_SOURCE_ADDRESS:-192.168.0.110}" +AGENT99_SOURCE_ADDRESS="${HOST112_AGENT99_SOURCE_ADDRESS:-192.168.0.99}" +AGENT99_EXPECTED_KEY_FINGERPRINT="${HOST112_AGENT99_EXPECTED_KEY_FINGERPRINT:-SHA256:4ccZZSe0buoWyivi2mfTMXE9LZfCmFBPr/evCzZ0nzA}" CONTROL_PUBLIC_KEY_PATH="${HOST112_CONTROL_PUBLIC_KEY_PATH:-${SCRIPT_DIR}/control.pub}" READINESS_SOURCE="${SCRIPT_DIR}/host112-guest-readiness.sh" SERVICE_SOURCE="${SCRIPT_DIR}/awoooi-host112-guest-recovery.service" @@ -22,29 +28,89 @@ INSTALLER_TARGET="/usr/local/sbin/awoooi-install-host112-guest-recovery" SERVICE_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.service" TIMER_TARGET="/etc/systemd/system/awoooi-host112-guest-recovery.timer" SUDOERS_TARGET="/etc/sudoers.d/awoooi-host112-agent99" -STATE_ROOT="/var/lib/awoooi-host112-recovery" +STATE_ROOT="${HOST112_RECOVERY_STATE_DIR:-/var/lib/awoooi-host112-recovery}" +MANAGER_LOCK_FILE="${HOST112_MANAGER_LOCK_FILE:-/run/awoooi-host112-manager-recovery.lock}" SUDOERS_ARGUMENT_REGEX='^--apply --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' +SUDOERS_DRY_RUN_ARGUMENT_REGEX='^--dry-run --trace-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --run-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127} --work-item-id [A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' + +TRANSACTION_ACTIVE=0 +ROLLBACK_IN_PROGRESS=0 +ROLLBACK_ATTEMPTED=0 +ROLLBACK_VERIFIED=0 +INSTALL_VERIFIED=0 +INSTALL_RECEIPT_WRITTEN=0 +INSTALL_RECEIPT_REF="none" +INSTALL_RECEIPT_PATH="" +INSTALL_ATTEMPT_ID="" +BACKUP_DIR="" +sudoers_tmp="" +AUTH_TMP="" +TIMER_ACTIVATION_PERFORMED=0 +USER_HOME="" +USER_GROUP="" +AUTH_DIR="" +AUTH_FILE="" +CONTROL_KEY_TYPE="" +CONTROL_KEY_BLOB="" +CONTROL_KEY_FINGERPRINT="unknown" usage() { cat <<'USAGE' Usage: install-host112-guest-recovery.sh --check - install-host112-guest-recovery.sh --apply - install-host112-guest-recovery.sh --rollback BACKUP_DIR + install-host112-guest-recovery.sh --apply --trace-id ID --run-id ID --work-item-id ID + install-host112-guest-recovery.sh --rollback BACKUP_DIR --trace-id ID --run-id ID --work-item-id ID -Apply installs the bounded host-112 recovery timer, an exact-command sudo rule, -and a forced-command readback key for host 110. It never reboots the host. +Apply installs and statically verifies the bounded host-112 recovery timer, +exact correlated apply/dry-run sudo rules, the host-110 forced readback key, +and the existing Windows99 direct Agent99 authorization. It starts only the +timer after all static gates pass; it never synchronously starts the recovery +oneshot, reboots the host, or changes VM power. USAGE } +set_mode() { + local requested="$1" + if [ "$MODE_EXPLICIT" -eq 1 ]; then + echo "BLOCKER duplicate_mode" >&2 + exit 64 + fi + MODE="$requested" + MODE_EXPLICIT=1 +} + +require_value() { + local flag="$1" value="${2:-}" + if [ -z "$value" ]; then + echo "BLOCKER missing_value=$flag" >&2 + exit 64 + fi +} + while [ "$#" -gt 0 ]; do case "$1" in - --check) MODE="check" ;; - --apply) MODE="apply" ;; + --check) set_mode "check" ;; + --apply) set_mode "apply" ;; --rollback) - MODE="rollback" + set_mode "rollback" + require_value "$1" "${2:-}" + ROLLBACK_DIR="$2" + shift + ;; + --trace-id) + require_value "$1" "${2:-}" + TRACE_ID="$2" + shift + ;; + --run-id) + require_value "$1" "${2:-}" + RUN_ID="$2" + shift + ;; + --work-item-id) + require_value "$1" "${2:-}" + WORK_ITEM_ID="$2" shift - ROLLBACK_DIR="${1:-}" ;; -h|--help) usage; exit 0 ;; *) echo "UNKNOWN_ARG=$1" >&2; usage >&2; exit 64 ;; @@ -52,21 +118,192 @@ while [ "$#" -gt 0 ]; do shift done -sudo_policy_readback() { +safe_identity() { + [[ "$1" =~ ^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$ ]] +} + +identity_count=0 +[ -n "$TRACE_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$RUN_ID" ] && identity_count=$((identity_count + 1)) +[ -n "$WORK_ITEM_ID" ] && identity_count=$((identity_count + 1)) +if [ "$identity_count" -ne 0 ] && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER incomplete_control_identity" >&2 + exit 64 +fi +if [ "$identity_count" -eq 3 ]; then + for identity in "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID"; do + if ! safe_identity "$identity"; then + echo "BLOCKER unsafe_control_identity" >&2 + exit 64 + fi + done +fi +if { [ "$MODE" = "apply" ] || [ "$MODE" = "rollback" ]; } && [ "$identity_count" -ne 3 ]; then + echo "BLOCKER installer_control_identity_required" >&2 + exit 64 +fi + +cleanup() { + [ -z "$sudoers_tmp" ] || rm -f "$sudoers_tmp" >/dev/null 2>&1 || true + [ -z "$AUTH_TMP" ] || rm -f "$AUTH_TMP" >/dev/null 2>&1 || true +} +trap cleanup EXIT + +resolve_target_user() { + USER_HOME="$(getent passwd "$TARGET_USER" 2>/dev/null | cut -d: -f6)" + USER_GROUP="$(id -gn "$TARGET_USER" 2>/dev/null || true)" + [ -n "$USER_HOME" ] || { echo "BLOCKER target_user_home_missing" >&2; return 1; } + [ -n "$USER_GROUP" ] || { echo "BLOCKER target_user_group_missing" >&2; return 1; } + AUTH_DIR="${USER_HOME}/.ssh" + AUTH_FILE="${AUTH_DIR}/authorized_keys" +} + +sudo_policy_allows() { if [ "${EUID:-$(id -u)}" -eq 0 ]; then - sudo -n -U "$TARGET_USER" -l "$@" 2>/dev/null || true + sudo -n -U "$TARGET_USER" -l "$@" >/dev/null 2>&1 else - sudo -n -l "$@" 2>/dev/null || true + sudo -n -l "$@" >/dev/null 2>&1 fi } -sudo_policy_has_nopasswd() { - local output - output="$(sudo_policy_readback "$@")" - printf '%s\n' "$output" | grep -Fq "NOPASSWD: $READINESS_TARGET" +agent99_direct_authorization_ready() { + local key_type key_blob fingerprint + [ -f "$AUTH_FILE" ] || return 1 + while IFS=$'\t' read -r key_type key_blob; do + [ -n "$key_type" ] && [ -n "$key_blob" ] || continue + [[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || continue + fingerprint="$({ printf '%s %s\n' "$key_type" "$key_blob" | ssh-keygen -lf - 2>/dev/null || true; } | awk 'NR == 1 {print $2}')" + if [ "$fingerprint" = "$AGENT99_EXPECTED_KEY_FINGERPRINT" ]; then + return 0 + fi + done < <( + awk -v source="$AGENT99_SOURCE_ADDRESS" ' + function has_restrict(options, count, parts, i) { + count = split(options, parts, ",") + for (i = 1; i <= count; i++) { + if (parts[i] == "restrict") return 1 + } + return 0 + } + index($1, "from=\"" source "\"") > 0 \ + && has_restrict($1) \ + && index($1, "command=") == 0 \ + && $2 ~ /^(ssh-ed25519|ssh-rsa|ecdsa-sha2-)/ { + print $2 "\t" $3 + } + ' "$AUTH_FILE" + ) + return 1 +} + +readiness_check_no_write_ready() { + local readiness_args + readiness_args=("$READINESS_TARGET" --check) + if [ "$identity_count" -eq 3 ]; then + readiness_args+=( + --trace-id "$TRACE_ID" + --run-id "$RUN_ID" + --work-item-id "$WORK_ITEM_ID" + ) + fi + if READINESS_CHECK_OUTPUT="$("${readiness_args[@]}" 2>&1)"; then + READINESS_CHECK_EXIT=0 + else + READINESS_CHECK_EXIT=$? + fi + grep -Eq '(^| )schema_version=host112_guest_recovery_v2($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )mode=check($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )runtime_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )manager_runtime_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )receipt_write_performed=0($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )ssh_port_listening=1($| )' <<<"$READINESS_CHECK_OUTPUT" \ + && grep -Eq '(^| )ssh_ready=1($| )' <<<"$READINESS_CHECK_OUTPUT" +} + +refresh_gate_state() { + TARGET_FILES_READY=1 + [ -x "$READINESS_TARGET" ] || TARGET_FILES_READY=0 + [ -x "$INSTALLER_TARGET" ] || TARGET_FILES_READY=0 + [ -f "$SERVICE_TARGET" ] || TARGET_FILES_READY=0 + [ -f "$TIMER_TARGET" ] || TARGET_FILES_READY=0 + [ -f "$SUDOERS_TARGET" ] || TARGET_FILES_READY=0 + + UNIT_CONTRACT_READY=0 + if [ -f "$SERVICE_TARGET" ] \ + && [ -f "$TIMER_TARGET" ] \ + && grep -Fq 'ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer' "$SERVICE_TARGET" \ + && grep -Fq 'TimeoutStartSec=480' "$SERVICE_TARGET" \ + && grep -Fq 'Unit=awoooi-host112-guest-recovery.service' "$TIMER_TARGET"; then + UNIT_CONTRACT_READY=1 + fi + + SUDOERS_VALID=0 + if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then + SUDOERS_VALID=1 + fi + SUDO_CORRELATED_APPLY_NOPASSWD=0 + if sudo_policy_allows "$READINESS_TARGET" --apply \ + --trace-id host112-policy-check \ + --run-id host112-policy-check \ + --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then + SUDO_CORRELATED_APPLY_NOPASSWD=1 + fi + SUDO_CORRELATED_DRY_RUN_NOPASSWD=0 + if sudo_policy_allows "$READINESS_TARGET" --dry-run \ + --trace-id host112-policy-check \ + --run-id host112-policy-check \ + --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then + SUDO_CORRELATED_DRY_RUN_NOPASSWD=1 + fi + SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0 + if sudo_policy_allows "$READINESS_TARGET" --apply; then + SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1 + fi + SUDO_UNSAFE_IDENTITY_NOPASSWD=0 + if sudo_policy_allows "$READINESS_TARGET" --apply \ + --trace-id 'unsafe value' \ + --run-id host112-policy-check \ + --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then + SUDO_UNSAFE_IDENTITY_NOPASSWD=1 + fi + SUDO_POLICY_READY=0 + if [ "$SUDOERS_VALID" -eq 1 ] \ + && [ "$SUDO_CORRELATED_APPLY_NOPASSWD" -eq 1 ] \ + && [ "$SUDO_CORRELATED_DRY_RUN_NOPASSWD" -eq 1 ] \ + && [ "$SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" -eq 0 ] \ + && [ "$SUDO_UNSAFE_IDENTITY_NOPASSWD" -eq 0 ]; then + SUDO_POLICY_READY=1 + fi + + AGENT99_99_DIRECT_ROUTE_READY=0 + if agent99_direct_authorization_ready; then + AGENT99_99_DIRECT_ROUTE_READY=1 + fi + READINESS_NO_WRITE_READY=0 + READINESS_CHECK_EXIT=-1 + READINESS_CHECK_OUTPUT="" + if [ -x "$READINESS_TARGET" ] && readiness_check_no_write_ready; then + READINESS_NO_WRITE_READY=1 + fi + TIMER_ENABLED="$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + TIMER_ACTIVE="$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)" + TIMER_ENABLED="${TIMER_ENABLED:-unknown}" + TIMER_ACTIVE="${TIMER_ACTIVE:-unknown}" } check() { + local gate_ready=0 path + refresh_gate_state + if [ "$TARGET_FILES_READY" -eq 1 ] \ + && [ "$UNIT_CONTRACT_READY" -eq 1 ] \ + && [ "$SUDO_POLICY_READY" -eq 1 ] \ + && [ "$AGENT99_99_DIRECT_ROUTE_READY" -eq 1 ] \ + && [ "$READINESS_NO_WRITE_READY" -eq 1 ] \ + && [ "$TIMER_ENABLED" = "enabled" ] \ + && [ "$TIMER_ACTIVE" = "active" ]; then + gate_ready=1 + fi + echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" echo "MODE=check" for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET"; do @@ -76,37 +313,24 @@ check() { echo "TARGET path=$path present=0 sha256=missing" fi done - echo "TIMER_ENABLED=$(systemctl is-enabled awoooi-host112-guest-recovery.timer 2>/dev/null || true)" - echo "TIMER_ACTIVE=$(systemctl is-active awoooi-host112-guest-recovery.timer 2>/dev/null || true)" - if [ -f "$SUDOERS_TARGET" ] && visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1; then - echo "SUDOERS_VALID=1" - else - echo "SUDOERS_VALID=0" - fi - if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \ - --trace-id host112-policy-check \ - --run-id host112-policy-check \ - --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then - echo "SUDO_CORRELATED_APPLY_NOPASSWD=1" - else - echo "SUDO_CORRELATED_APPLY_NOPASSWD=0" - fi - if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply; then - echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=1" - else - echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=0" - fi - if sudo_policy_has_nopasswd "$READINESS_TARGET" --apply \ - --trace-id 'unsafe value' \ - --run-id host112-policy-check \ - --work-item-id HOST112-WAZUH-MANAGER-RECOVERY; then - echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=1" - else - echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=0" - fi - if [ -x "$READINESS_TARGET" ]; then - "$READINESS_TARGET" --check || true - fi + echo "TARGET_FILES_READY=$TARGET_FILES_READY" + echo "UNIT_CONTRACT_READY=$UNIT_CONTRACT_READY" + echo "TIMER_ENABLED=$TIMER_ENABLED" + echo "TIMER_ACTIVE=$TIMER_ACTIVE" + echo "SUDOERS_VALID=$SUDOERS_VALID" + echo "SUDO_CORRELATED_APPLY_NOPASSWD=$SUDO_CORRELATED_APPLY_NOPASSWD" + echo "SUDO_CORRELATED_DRY_RUN_NOPASSWD=$SUDO_CORRELATED_DRY_RUN_NOPASSWD" + echo "SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD=$SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD" + echo "SUDO_UNSAFE_IDENTITY_NOPASSWD=$SUDO_UNSAFE_IDENTITY_NOPASSWD" + echo "SUDO_POLICY_READY=$SUDO_POLICY_READY" + echo "AGENT99_99_DIRECT_ROUTE_READY=$AGENT99_99_DIRECT_ROUTE_READY" + echo "AGENT99_99_DIRECT_RUNTIME_PROBE=not_started" + echo "READINESS_NO_WRITE_READY=$READINESS_NO_WRITE_READY" + echo "READINESS_CHECK_EXIT=$READINESS_CHECK_EXIT" + echo "CHECK_GATE_READY=$gate_ready" + echo "INSTALLER_SYNCHRONOUS_RECOVERY_START=0" + echo "RECOVERY_EXECUTION_STATUS=separate_runtime_run_required" + [ "$gate_ready" -eq 1 ] } require_root() { @@ -128,105 +352,318 @@ backup_path() { fi } +create_backup() { + local run_slug + run_slug="$(printf '%s' "$RUN_ID" | tr -c 'A-Za-z0-9._+-' '_')" + BACKUP_DIR="${STATE_ROOT}/backups/host112-installer-${run_slug}-${INSTALL_ATTEMPT_ID}" + install -d -m 0750 "$BACKUP_DIR" + : >"${BACKUP_DIR}/manifest.tsv" + for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$AUTH_FILE"; do + backup_path "$path" "$BACKUP_DIR" + done + systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 \ + && touch "$BACKUP_DIR/timer_was_enabled" || true + systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 \ + && touch "$BACKUP_DIR/timer_was_active" || true +} + +restore_from_backup() { + local backup_dir="$1" state path ok=1 + timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true + if systemctl is-active awoooi-host112-guest-recovery.service >/dev/null 2>&1; then + timeout 30 systemctl stop awoooi-host112-guest-recovery.service >/dev/null 2>&1 || ok=0 + fi + systemctl disable awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true + while IFS=$'\t' read -r state path; do + [ -n "$path" ] || continue + if [ "$state" = "present" ]; then + install -d "$(dirname "$path")" || ok=0 + cp -a "$backup_dir/rootfs/${path#/}" "$path" || ok=0 + elif [ "$state" = "missing" ]; then + rm -f "$path" || ok=0 + else + ok=0 + fi + done <"$backup_dir/manifest.tsv" + systemctl daemon-reload >/dev/null 2>&1 || ok=0 + if [ -f "$backup_dir/timer_was_enabled" ]; then + systemctl enable awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || ok=0 + fi + if [ -f "$backup_dir/timer_was_active" ]; then + systemctl start awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || ok=0 + fi + [ "$ok" -eq 1 ] +} + +verify_rollback() { + local backup_dir="$1" state path + while IFS=$'\t' read -r state path; do + [ -n "$path" ] || continue + if [ "$state" = "present" ]; then + [ -e "$path" ] || return 1 + cmp -s "$backup_dir/rootfs/${path#/}" "$path" || return 1 + elif [ "$state" = "missing" ]; then + [ ! -e "$path" ] || return 1 + else + return 1 + fi + done <"$backup_dir/manifest.tsv" + if [ -f "$SUDOERS_TARGET" ]; then + visudo -cf "$SUDOERS_TARGET" >/dev/null 2>&1 || return 1 + fi + if [ -f "$backup_dir/timer_was_enabled" ]; then + systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1 + else + ! systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1 + fi + if [ -f "$backup_dir/timer_was_active" ]; then + systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1 + else + ! systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || return 1 + fi +} + +write_install_receipt() { + local terminal="$1" install_verified="$2" rollback_attempted="$3" rollback_verified="$4" error_exit="$5" + local receipt_dir receipt_tmp generated_at readiness_sha installer_sha + receipt_dir="${STATE_ROOT}/install-receipts" + install -d -m 0750 "$receipt_dir" + if [ -z "$INSTALL_RECEIPT_PATH" ]; then + INSTALL_RECEIPT_PATH="${receipt_dir}/install-${RUN_ID//:/_}-${INSTALL_ATTEMPT_ID}.txt" + fi + generated_at="$(date --iso-8601=seconds)" + readiness_sha="$(sha256sum "$READINESS_TARGET" 2>/dev/null | awk '{print $1}')" + installer_sha="$(sha256sum "$INSTALLER_TARGET" 2>/dev/null | awk '{print $1}')" + readiness_sha="${readiness_sha:-missing}" + installer_sha="${installer_sha:-missing}" + receipt_tmp="${INSTALL_RECEIPT_PATH}.tmp.$$" + printf '%s\n' \ + "schema_version=host112_guest_recovery_install_receipt_v1 generated_at=$generated_at trace_id=$TRACE_ID run_id=$RUN_ID work_item_id=$WORK_ITEM_ID terminal=$terminal install_verified=$install_verified installer_runtime_write_performed=1 manager_runtime_write_performed=0 static_post_gate_ready=${STATIC_POST_GATE_READY:-0} check_gate_ready=${CHECK_GATE_READY:-0} sudo_policy_ready=${SUDO_POLICY_READY:-0} agent99_99_direct_route_ready=${AGENT99_99_DIRECT_ROUTE_READY:-0} agent99_99_direct_runtime_probe=not_started timer_enabled=${TIMER_ENABLED:-unknown} timer_active=${TIMER_ACTIVE:-unknown} timer_activation_performed=$TIMER_ACTIVATION_PERFORMED synchronous_recovery_start=0 recovery_execution_performed=0 recovery_run_id=none recovery_receipt_ref=none rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified error_exit=$error_exit backup_ref=$BACKUP_DIR readiness_sha256=$readiness_sha installer_sha256=$installer_sha prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" \ + >"$receipt_tmp" + chmod 0640 "$receipt_tmp" + if ! mv -n "$receipt_tmp" "$INSTALL_RECEIPT_PATH" \ + || [ -e "$receipt_tmp" ]; then + rm -f "$receipt_tmp" >/dev/null 2>&1 || true + return 1 + fi + INSTALL_RECEIPT_WRITTEN=1 + INSTALL_RECEIPT_REF="host112-install:${RUN_ID}:${INSTALL_ATTEMPT_ID}" +} + +handle_apply_failure() { + local original_exit="${1:-1}" terminal="install_failed_rollback_failed" + trap - ERR INT TERM + set +e + if [ "$ROLLBACK_IN_PROGRESS" -eq 1 ]; then + exit 74 + fi + ROLLBACK_IN_PROGRESS=1 + if [ "$TRANSACTION_ACTIVE" -eq 1 ] && [ -n "$BACKUP_DIR" ]; then + ROLLBACK_ATTEMPTED=1 + if restore_from_backup "$BACKUP_DIR" && verify_rollback "$BACKUP_DIR"; then + ROLLBACK_VERIFIED=1 + terminal="install_failed_rolled_back_verified" + fi + fi + refresh_gate_state >/dev/null 2>&1 || true + write_install_receipt "$terminal" 0 "$ROLLBACK_ATTEMPTED" "$ROLLBACK_VERIFIED" "$original_exit" >/dev/null 2>&1 || true + echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" >&2 + echo "MODE=apply_failed" >&2 + echo "TERMINAL=$terminal" >&2 + echo "BACKUP_DIR=${BACKUP_DIR:-none}" >&2 + echo "ROLLBACK_ATTEMPTED=$ROLLBACK_ATTEMPTED" >&2 + echo "ROLLBACK_VERIFIED=$ROLLBACK_VERIFIED" >&2 + echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF" >&2 + echo "SYNCHRONOUS_RECOVERY_START=0" >&2 + if [ "$ROLLBACK_VERIFIED" -ne 1 ]; then + exit 74 + fi + [ "$original_exit" -ne 0 ] || original_exit=1 + exit "$original_exit" +} + +static_source_preflight() { + local source service_state + for tool in bash cmp flock getent install sha256sum ssh-keygen sudo systemctl systemd-analyze timeout visudo; do + command -v "$tool" >/dev/null 2>&1 || { echo "BLOCKER required_tool_missing=$tool" >&2; return 1; } + done + for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH" "${SCRIPT_DIR}/install-host112-guest-recovery.sh"; do + [ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; return 1; } + done + bash -n "$READINESS_SOURCE" "${SCRIPT_DIR}/install-host112-guest-recovery.sh" + grep -Fq 'ExecStart=/usr/local/sbin/awoooi-host112-guest-readiness --apply-timer' "$SERVICE_SOURCE" + grep -Fq 'TimeoutStartSec=480' "$SERVICE_SOURCE" + grep -Fq 'Unit=awoooi-host112-guest-recovery.service' "$TIMER_SOURCE" + + read -r CONTROL_KEY_TYPE CONTROL_KEY_BLOB _ <"$CONTROL_PUBLIC_KEY_PATH" + case "$CONTROL_KEY_TYPE" in + ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;; + *) echo "BLOCKER unsupported_control_public_key_type" >&2; return 1 ;; + esac + [[ "$CONTROL_KEY_BLOB" =~ ^[A-Za-z0-9+/=]+$ ]] \ + || { echo "BLOCKER invalid_control_public_key_blob" >&2; return 1; } + CONTROL_KEY_FINGERPRINT="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk 'NR == 1 {print $2}')" + [ -n "$CONTROL_KEY_FINGERPRINT" ] || { echo "BLOCKER control_public_key_fingerprint_missing" >&2; return 1; } + agent99_direct_authorization_ready \ + || { echo "BLOCKER agent99_99_direct_authorization_not_ready" >&2; return 1; } + service_state="$(systemctl is-active awoooi-host112-guest-recovery.service 2>/dev/null || true)" + case "$service_state" in + active|activating) echo "BLOCKER host112_recovery_service_busy" >&2; return 1 ;; + esac + + sudoers_tmp="$(mktemp)" + { + printf '%s ALL=(root) NOPASSWD: %s %s\n' \ + "$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" + printf '%s ALL=(root) NOPASSWD: %s %s\n' \ + "$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_DRY_RUN_ARGUMENT_REGEX" + } >"$sudoers_tmp" + visudo -cf "$sudoers_tmp" >/dev/null +} + +post_install_static_gate() { + STATIC_POST_GATE_READY=0 + TARGET_HASHES_MATCH=0 + UNIT_STATIC_VERIFY_READY=0 + if cmp -s "$READINESS_SOURCE" "$READINESS_TARGET" \ + && cmp -s "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET" \ + && cmp -s "$SERVICE_SOURCE" "$SERVICE_TARGET" \ + && cmp -s "$TIMER_SOURCE" "$TIMER_TARGET"; then + TARGET_HASHES_MATCH=1 + fi + if systemd-analyze verify "$SERVICE_TARGET" "$TIMER_TARGET" >/dev/null 2>&1; then + UNIT_STATIC_VERIFY_READY=1 + fi + refresh_gate_state + if [ "$TARGET_HASHES_MATCH" -eq 1 ] \ + && [ "$UNIT_STATIC_VERIFY_READY" -eq 1 ] \ + && [ "$TARGET_FILES_READY" -eq 1 ] \ + && [ "$UNIT_CONTRACT_READY" -eq 1 ] \ + && [ "$SUDO_POLICY_READY" -eq 1 ] \ + && [ "$AGENT99_99_DIRECT_ROUTE_READY" -eq 1 ] \ + && [ "$READINESS_NO_WRITE_READY" -eq 1 ] \ + && [ "$TIMER_ENABLED" = "enabled" ]; then + STATIC_POST_GATE_READY=1 + fi + echo "TARGET_HASHES_MATCH=$TARGET_HASHES_MATCH" + echo "UNIT_STATIC_VERIFY_READY=$UNIT_STATIC_VERIFY_READY" + echo "STATIC_POST_GATE_READY=$STATIC_POST_GATE_READY" + [ "$STATIC_POST_GATE_READY" -eq 1 ] +} + +wait_for_recovery_service_idle() { + local deadline state + deadline=$(( $(date +%s) + 30 )) + while true; do + state="$(systemctl is-active awoooi-host112-guest-recovery.service 2>/dev/null || true)" + case "$state" in + active|activating) ;; + *) return 0 ;; + esac + [ "$(date +%s)" -lt "$deadline" ] || return 1 + sleep 1 + done +} + +resolve_target_user + if [ "$MODE" = "check" ]; then check - exit 0 + exit $? fi require_root +INSTALL_ATTEMPT_ID="$(date +%Y%m%dT%H%M%S%z)-$$" if [ "$MODE" = "rollback" ]; then case "$ROLLBACK_DIR" in "$STATE_ROOT"/backups/*) ;; *) echo "BLOCKER invalid_host112_rollback_dir" >&2; exit 64 ;; esac - [ -f "$ROLLBACK_DIR/manifest.tsv" ] || { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; } - systemctl disable --now awoooi-host112-guest-recovery.timer >/dev/null 2>&1 || true - while IFS=$'\t' read -r state path; do - [ -n "$path" ] || continue - if [ "$state" = "present" ]; then - install -d "$(dirname "$path")" - cp -a "$ROLLBACK_DIR/rootfs/${path#/}" "$path" - else - rm -f "$path" - fi - done <"$ROLLBACK_DIR/manifest.tsv" - systemctl daemon-reload - if [ -f "$ROLLBACK_DIR/timer_was_enabled" ]; then - systemctl enable awoooi-host112-guest-recovery.timer >/dev/null - fi - if [ -f "$ROLLBACK_DIR/timer_was_active" ]; then - systemctl start awoooi-host112-guest-recovery.timer + [ -f "$ROLLBACK_DIR/manifest.tsv" ] \ + || { echo "BLOCKER rollback_manifest_missing" >&2; exit 66; } + BACKUP_DIR="$ROLLBACK_DIR" + exec 9>"$MANAGER_LOCK_FILE" + flock -w 30 9 || { echo "BLOCKER host112_installer_single_flight_busy" >&2; exit 75; } + ROLLBACK_ATTEMPTED=1 + if restore_from_backup "$ROLLBACK_DIR" && verify_rollback "$ROLLBACK_DIR"; then + ROLLBACK_VERIFIED=1 fi + refresh_gate_state >/dev/null 2>&1 || true + write_install_receipt \ + "manual_rollback_$(if [ "$ROLLBACK_VERIFIED" -eq 1 ]; then printf verified; else printf failed; fi)" \ + 0 1 "$ROLLBACK_VERIFIED" 0 echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" echo "MODE=rollback" - echo "ROLLBACK_APPLIED=1" - check - exit 0 + echo "ROLLBACK_APPLIED=$ROLLBACK_VERIFIED" + echo "ROLLBACK_VERIFIED=$ROLLBACK_VERIFIED" + echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF" + echo "RECOVERY_EXECUTION_PERFORMED=0" + [ "$ROLLBACK_VERIFIED" -eq 1 ] + exit $? fi -for source in "$READINESS_SOURCE" "$SERVICE_SOURCE" "$TIMER_SOURCE" "$CONTROL_PUBLIC_KEY_PATH"; do - [ -f "$source" ] || { echo "BLOCKER source_missing=$source" >&2; exit 66; } -done +static_source_preflight +exec 9>"$MANAGER_LOCK_FILE" +flock -w 30 9 || { echo "BLOCKER host112_installer_single_flight_busy" >&2; exit 75; } +create_backup +echo "BACKUP_DIR=$BACKUP_DIR" +TRANSACTION_ACTIVE=1 +trap 'handle_apply_failure $?' ERR +trap 'handle_apply_failure 130' INT TERM -user_home="$(getent passwd "$TARGET_USER" | cut -d: -f6)" -user_group="$(id -gn "$TARGET_USER")" -[ -n "$user_home" ] || { echo "BLOCKER target_user_home_missing" >&2; exit 66; } -auth_dir="${user_home}/.ssh" -auth_file="${auth_dir}/authorized_keys" - -read -r key_type key_blob _ <"$CONTROL_PUBLIC_KEY_PATH" -case "$key_type" in - ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;; - *) echo "BLOCKER unsupported_control_public_key_type" >&2; exit 65 ;; -esac -[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || { echo "BLOCKER invalid_control_public_key_blob" >&2; exit 65; } -key_fingerprint="$(ssh-keygen -lf "$CONTROL_PUBLIC_KEY_PATH" | awk '{print $2}')" - -run_id="host112-guest-recovery-$(date +%Y%m%dT%H%M%S%z)-$$" -backup_dir="${STATE_ROOT}/backups/${run_id}" -install -d -m 0750 "$backup_dir" -: >"${backup_dir}/manifest.tsv" -for path in "$READINESS_TARGET" "$INSTALLER_TARGET" "$SERVICE_TARGET" "$TIMER_TARGET" "$SUDOERS_TARGET" "$auth_file"; do - backup_path "$path" "$backup_dir" -done -systemctl is-enabled awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_enabled" || true -systemctl is-active awoooi-host112-guest-recovery.timer >/dev/null 2>&1 && touch "$backup_dir/timer_was_active" || true +timeout 20 systemctl stop awoooi-host112-guest-recovery.timer >/dev/null 2>&1 +wait_for_recovery_service_idle install -m 0755 "$READINESS_SOURCE" "$READINESS_TARGET" install -m 0755 "${SCRIPT_DIR}/install-host112-guest-recovery.sh" "$INSTALLER_TARGET" install -m 0644 "$SERVICE_SOURCE" "$SERVICE_TARGET" install -m 0644 "$TIMER_SOURCE" "$TIMER_TARGET" - -sudoers_tmp="$(mktemp)" -trap 'rm -f "$sudoers_tmp"' EXIT -printf '%s ALL=(root) NOPASSWD: %s %s\n' \ - "$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX" >"$sudoers_tmp" -visudo -cf "$sudoers_tmp" >/dev/null install -m 0440 "$sudoers_tmp" "$SUDOERS_TARGET" -install -d -m 0700 -o "$TARGET_USER" -g "$user_group" "$auth_dir" -if [ ! -f "$auth_file" ]; then - install -m 0600 -o "$TARGET_USER" -g "$user_group" /dev/null "$auth_file" +install -d -m 0700 -o "$TARGET_USER" -g "$USER_GROUP" "$AUTH_DIR" +if [ ! -f "$AUTH_FILE" ]; then + install -m 0600 -o "$TARGET_USER" -g "$USER_GROUP" /dev/null "$AUTH_FILE" fi -auth_tmp="$(mktemp "${auth_file}.tmp.XXXXXX")" -awk -v blob="$key_blob" 'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' "$auth_file" >"$auth_tmp" +AUTH_TMP="$(mktemp "${AUTH_FILE}.tmp.XXXXXX")" +awk -v blob="$CONTROL_KEY_BLOB" \ + 'index($0, blob) == 0 && index($0, "awoooi-host112-probe-110") == 0' \ + "$AUTH_FILE" >"$AUTH_TMP" printf 'from="%s",restrict,command="%s --check" %s %s awoooi-host112-probe-110\n' \ - "$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$key_type" "$key_blob" >>"$auth_tmp" -chown "$TARGET_USER:$user_group" "$auth_tmp" -chmod 0600 "$auth_tmp" -mv "$auth_tmp" "$auth_file" + "$CONTROL_SOURCE_ADDRESS" "$READINESS_TARGET" "$CONTROL_KEY_TYPE" "$CONTROL_KEY_BLOB" >>"$AUTH_TMP" +chown "$TARGET_USER:$USER_GROUP" "$AUTH_TMP" +chmod 0600 "$AUTH_TMP" +mv "$AUTH_TMP" "$AUTH_FILE" +AUTH_TMP="" systemctl daemon-reload -systemctl enable --now awoooi-host112-guest-recovery.timer >/dev/null -systemctl start awoooi-host112-guest-recovery.service +systemctl enable awoooi-host112-guest-recovery.timer >/dev/null +post_install_static_gate +systemctl start awoooi-host112-guest-recovery.timer +TIMER_ACTIVATION_PERFORMED=1 +check +CHECK_GATE_READY=1 +INSTALL_VERIFIED=1 +write_install_receipt "installed_timer_activated_recovery_separate" 1 0 0 0 +TRANSACTION_ACTIVE=0 +trap - ERR INT TERM echo "AWOOOI_HOST112_GUEST_RECOVERY_INSTALLER=1" echo "MODE=apply" -echo "RUN_ID=$run_id" -echo "BACKUP_DIR=$backup_dir" -echo "CONTROL_KEY_FINGERPRINT=$key_fingerprint" +echo "TRACE_ID=$TRACE_ID" +echo "RUN_ID=$RUN_ID" +echo "WORK_ITEM_ID=$WORK_ITEM_ID" +echo "BACKUP_DIR=$BACKUP_DIR" +echo "CONTROL_KEY_FINGERPRINT=$CONTROL_KEY_FINGERPRINT" +echo "INSTALL_VERIFIED=$INSTALL_VERIFIED" +echo "INSTALL_RECEIPT_WRITTEN=$INSTALL_RECEIPT_WRITTEN" +echo "INSTALL_RECEIPT_REF=$INSTALL_RECEIPT_REF" +echo "RECOVERY_EXECUTION_PERFORMED=0" +echo "RECOVERY_RUN_ID=none" +echo "RECOVERY_RECEIPT_REF=none" +echo "RECOVERY_EXECUTION_STATUS=separate_timer_or_agent99_run_required" +echo "AGENT99_99_DIRECT_RUNTIME_PROBE=not_started" echo "REBOOT_PERFORMED=0" +echo "VM_POWER_CHANGE_PERFORMED=0" echo "SECRET_READ=0" echo "RESTRICTED_CONTROL_KEY_INSTALLED=1" -check diff --git a/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py new file mode 100644 index 000000000..2d8903a75 --- /dev/null +++ b/scripts/reboot-recovery/tests/test_install_host112_guest_recovery_contract.py @@ -0,0 +1,150 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +INSTALLER = ROOT / "scripts/reboot-recovery/install-host112-guest-recovery.sh" + + +def source() -> str: + return INSTALLER.read_text(encoding="utf-8") + + +def test_installer_shell_syntax_is_valid() -> None: + result = subprocess.run( + ["bash", "-n", str(INSTALLER)], + text=True, + capture_output=True, + check=False, + timeout=10, + ) + assert result.returncode == 0, result.stdout + result.stderr + + +def test_apply_and_rollback_require_safe_control_identity() -> None: + text = source() + + assert '--trace-id ID --run-id ID --work-item-id ID' in text + assert 'BLOCKER incomplete_control_identity' in text + assert 'BLOCKER unsafe_control_identity' in text + assert 'BLOCKER installer_control_identity_required' in text + assert '^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$' in text + + +def test_sudo_policy_has_exact_correlated_apply_and_dry_run_only() -> None: + text = source() + + assert "SUDOERS_ARGUMENT_REGEX='^--apply --trace-id " in text + assert "SUDOERS_DRY_RUN_ARGUMENT_REGEX='^--dry-run --trace-id " in text + assert '"$TARGET_USER" "$READINESS_TARGET" "$SUDOERS_ARGUMENT_REGEX"' in text + assert ( + '"$TARGET_USER" "$READINESS_TARGET" ' + '"$SUDOERS_DRY_RUN_ARGUMENT_REGEX"' in text + ) + assert 'SUDO_CORRELATED_APPLY_NOPASSWD' in text + assert 'SUDO_CORRELATED_DRY_RUN_NOPASSWD' in text + assert 'SUDO_LEGACY_UNCORRELATED_APPLY_NOPASSWD' in text + assert 'SUDO_UNSAFE_IDENTITY_NOPASSWD' in text + assert 'SUDO_POLICY_READY' in text + + +def test_windows99_direct_route_is_fingerprint_bound_and_not_forced() -> None: + text = source() + route = text[ + text.index("agent99_direct_authorization_ready() {") : text.index( + "readiness_check_no_write_ready() {" + ) + ] + + assert 'HOST112_AGENT99_SOURCE_ADDRESS:-192.168.0.99' in text + assert 'HOST112_AGENT99_EXPECTED_KEY_FINGERPRINT' in text + assert 'AGENT99_99_DIRECT_ROUTE_READY' in text + assert 'index($1, "command=") == 0' in route + assert 'has_restrict($1)' in route + assert 'ssh-keygen -lf -' in route + assert 'cat "$AUTH_FILE"' not in text + assert 'from="%s",restrict,command="%s --check"' in text + assert 'CONTROL_SOURCE_ADDRESS:-192.168.0.110' in text + + +def test_apply_is_transactional_and_has_a_verified_rollback() -> None: + text = source() + apply_path = text[text.index("static_source_preflight\n") :] + handler = text[ + text.index("handle_apply_failure() {") : text.index( + "static_source_preflight() {" + ) + ] + + assert 'set -Eeuo pipefail' in text + assert 'TRANSACTION_ACTIVE=1' in apply_path + assert "trap 'handle_apply_failure $?' ERR" in apply_path + assert "trap 'handle_apply_failure 130' INT TERM" in apply_path + assert 'restore_from_backup "$BACKUP_DIR"' in handler + assert 'verify_rollback "$BACKUP_DIR"' in handler + assert 'ROLLBACK_VERIFIED=1' in handler + assert 'install_failed_rolled_back_verified' in handler + assert 'cmp -s "$backup_dir/rootfs/${path#/}" "$path"' in text + assert 'timer_was_enabled' in text + assert 'timer_was_active' in text + + +def test_static_gate_precedes_timer_activation_and_no_oneshot_is_started() -> None: + text = source() + apply_path = text[text.index("static_source_preflight\n") :] + + static_gate = apply_path.index("post_install_static_gate\n") + timer_start = apply_path.index( + "systemctl start awoooi-host112-guest-recovery.timer" + ) + final_gate = apply_path.index("check\n", timer_start) + receipt = apply_path.index( + 'write_install_receipt "installed_timer_activated_recovery_separate"' + ) + assert static_gate < timer_start < final_gate < receipt + assert 'systemctl enable awoooi-host112-guest-recovery.timer' in apply_path + assert 'systemctl enable --now awoooi-host112-guest-recovery.timer' not in text + assert 'systemctl start awoooi-host112-guest-recovery.service' not in text + assert "grep -Fq 'TimeoutStartSec=480'" in text + assert "TimeoutStartSec=600" not in text + assert 'INSTALLER_SYNCHRONOUS_RECOVERY_START=0' in text + assert 'RECOVERY_EXECUTION_PERFORMED=0' in text + + +def test_install_receipt_does_not_claim_a_recovery_run() -> None: + text = source() + receipt = text[ + text.index("write_install_receipt() {") : text.index( + "handle_apply_failure() {" + ) + ] + + assert 'schema_version=host112_guest_recovery_install_receipt_v1' in receipt + assert 'install_verified=$install_verified' in receipt + assert 'manager_runtime_write_performed=0' in receipt + assert 'recovery_execution_performed=0' in receipt + assert 'recovery_run_id=none' in receipt + assert 'recovery_receipt_ref=none' in receipt + assert 'agent99_99_direct_runtime_probe=not_started' in receipt + assert 'rollback_attempted=$rollback_attempted' in receipt + assert 'rollback_verified=$rollback_verified' in receipt + + +def test_check_gate_requires_policy_route_timer_and_no_write_readback() -> None: + text = source() + check = text[text.index("check() {") : text.index("require_root() {")] + + for required in ( + 'SUDO_POLICY_READY', + 'AGENT99_99_DIRECT_ROUTE_READY', + 'READINESS_NO_WRITE_READY', + 'TIMER_ENABLED', + 'TIMER_ACTIVE', + 'CHECK_GATE_READY', + ): + assert required in check + assert 'runtime_write_performed=0' in text + assert 'manager_runtime_write_performed=0' in text + assert 'receipt_write_performed=0' in text