fix(recovery): close cold-start runtime blockers

2026-07-11 15:08:27 +08:00
parent 7cea825d92
commit 276410fbb6
38 changed files with 1494 additions and 145 deletions
--- a/ops/nginx/awoooi.wooo.work.conf
+++ b/ops/nginx/awoooi.wooo.work.conf
@@ -93,6 +93,20 @@ server {
    # 避免主機或 K3s rollout 期間把 Nginx raw 5xx 直接給使用者。
    proxy_intercept_errors on;

+    # =========================================================================
+    # Next.js API routes
+    # =========================================================================
+    # Keep this before /api/ so browser telemetry reaches the web route handler.
+    location /api/sentry-tunnel {
+        proxy_pass http://awoooi_web;
+        proxy_intercept_errors on;
+        error_page 502 503 504 =503 /__awoooi-maintenance.html;
+        proxy_read_timeout 30s;
+        proxy_send_timeout 30s;
+
+        proxy_set_header Connection "keep-alive";
+    }
+
    # =========================================================================
    # API 路由 (/api/)
    # =========================================================================
--- a/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh
+++ b/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh
@@ -0,0 +1,273 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+umask 077
+
+ACTION="${1:-check}"
+if [ "$#" -ne 1 ]; then
+  echo "BLOCKER invalid_executor_arguments"
+  exit 64
+fi
+
+CANONICAL_DIR="/usr/local/share/awoooi-edge-maintenance"
+CANONICAL_CONF="${CANONICAL_DIR}/awoooi.wooo.work.conf"
+CANONICAL_MAINT="${CANONICAL_DIR}/maintenance.html"
+LIVE_CONF_LINK="/etc/nginx/sites-enabled/awoooi.wooo.work.conf"
+EXPECTED_LIVE_CONF="/etc/nginx/sites-available/awoooi.wooo.work.conf"
+LIVE_MAINT="/var/www/maintenance/maintenance.html"
+STATE_DIR="/var/lib/awoooi-edge-maintenance"
+BACKUP_ROOT="${STATE_DIR}/backups"
+LAST_BACKUP_FILE="${STATE_DIR}/last-backup"
+AUDIT_LOG="/var/log/awoooi-edge-maintenance.log"
+LOCK_FILE="/run/lock/awoooi-edge-maintenance.lock"
+RUN_ID="edge-maintenance-$(date +%Y%m%dT%H%M%S%z)-$$"
+
+audit() {
+  local result="$1"
+  printf '%s run_id=%s action=%s result=%s\n' \
+    "$(date --iso-8601=seconds)" "$RUN_ID" "$ACTION" "$result" >>"$AUDIT_LOG"
+}
+
+block() {
+  local reason="$1"
+  echo "EDGE_EXECUTOR_RESULT=blocked"
+  echo "RUN_ID=$RUN_ID"
+  echo "BLOCKER $reason"
+  audit "blocked:${reason}" || true
+  exit 75
+}
+
+sha256_file() {
+  sha256sum "$1" | awk '{print $1}'
+}
+
+resolve_live_conf() {
+  local resolved
+  resolved="$(readlink -f "$LIVE_CONF_LINK" 2>/dev/null || true)"
+  if [ -z "$resolved" ] && [ ! -e "$LIVE_CONF_LINK" ]; then
+    resolved="$EXPECTED_LIVE_CONF"
+  fi
+  case "$resolved" in
+    "$EXPECTED_LIVE_CONF"|"$LIVE_CONF_LINK")
+      printf '%s\n' "$resolved"
+      ;;
+    *)
+      block "unexpected_live_config_target"
+      ;;
+  esac
+}
+
+assert_root_payload() {
+  local file owner
+  for file in "$CANONICAL_CONF" "$CANONICAL_MAINT"; do
+    [ -f "$file" ] || block "canonical_payload_missing"
+    owner="$(stat -c '%u' "$file")"
+    [ "$owner" = "0" ] || block "canonical_payload_not_root_owned"
+    if find "$file" -prune -perm /022 -print -quit | grep -q .; then
+      block "canonical_payload_is_mutable"
+    fi
+  done
+
+  grep -q 'server_name awoooi.wooo.work;' "$CANONICAL_CONF" \
+    || block "canonical_server_name_missing"
+  grep -q '192.168.0.125:32334' "$CANONICAL_CONF" \
+    || block "canonical_api_vip_missing"
+  grep -q '192.168.0.125:32335' "$CANONICAL_CONF" \
+    || block "canonical_web_vip_missing"
+  grep -q 'location /api/sentry-tunnel {' "$CANONICAL_CONF" \
+    || block "canonical_sentry_tunnel_missing"
+  grep -q 'proxy_intercept_errors on;' "$CANONICAL_CONF" \
+    || block "canonical_intercept_missing"
+  grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$CANONICAL_CONF" \
+    || block "canonical_error_page_missing"
+  grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$CANONICAL_CONF" \
+    || block "canonical_fallback_header_missing"
+  grep -q 'AWOOOI 維護模式' "$CANONICAL_MAINT" \
+    || block "canonical_maintenance_marker_missing"
+  if grep -Ev '^[[:space:]]*#' "$CANONICAL_CONF" \
+    | grep -Eq '(^|[^0-9])(31234|31235)([^0-9]|$)'; then
+    block "canonical_legacy_port_forbidden"
+  fi
+
+  local tunnel_line api_line
+  tunnel_line="$(grep -n -m1 'location /api/sentry-tunnel {' "$CANONICAL_CONF" | cut -d: -f1)"
+  api_line="$(grep -n -m1 'location /api/ {' "$CANONICAL_CONF" | cut -d: -f1)"
+  [ "$tunnel_line" -lt "$api_line" ] || block "canonical_route_order_invalid"
+}
+
+print_status() {
+  local live_conf last_backup=""
+  live_conf="$(resolve_live_conf)"
+  [ -f "$LAST_BACKUP_FILE" ] && last_backup="$(cat "$LAST_BACKUP_FILE")"
+
+  echo "AWOOOI_EDGE_MAINTENANCE_EXECUTOR=1"
+  echo "EDGE_EXECUTOR_ACTION=check"
+  echo "EDGE_EXECUTOR_READY=1"
+  echo "RUN_ID=$RUN_ID"
+  echo "CANONICAL_CONF_SHA256=$(sha256_file "$CANONICAL_CONF")"
+  echo "CANONICAL_MAINT_SHA256=$(sha256_file "$CANONICAL_MAINT")"
+  echo "LIVE_CONF_TARGET=$live_conf"
+  if [ -f "$live_conf" ]; then
+    echo "LIVE_CONF_SHA256=$(sha256_file "$live_conf")"
+  else
+    echo "LIVE_CONF_SHA256="
+  fi
+  if [ -f "$LIVE_MAINT" ]; then
+    echo "LIVE_MAINT_SHA256=$(sha256_file "$LIVE_MAINT")"
+  else
+    echo "LIVE_MAINT_SHA256="
+  fi
+  echo "LAST_BACKUP_DIR=$last_backup"
+}
+
+create_backup() {
+  local label="$1" live_conf backup_dir
+  live_conf="$(resolve_live_conf)"
+  backup_dir="${BACKUP_ROOT}/${RUN_ID}-${label}"
+  install -d -m 0700 "$backup_dir"
+  if [ -f "$live_conf" ]; then
+    cp -a "$live_conf" "$backup_dir/live.conf"
+  else
+    : >"$backup_dir/live.conf.missing"
+  fi
+  if [ -f "$LIVE_MAINT" ]; then
+    cp -a "$LIVE_MAINT" "$backup_dir/maintenance.html"
+  else
+    : >"$backup_dir/maintenance.html.missing"
+  fi
+  printf '%s\n' "$live_conf" >"$backup_dir/live-conf-target"
+  printf '%s\n' "$backup_dir"
+}
+
+restore_backup() {
+  local backup_dir="$1" live_conf
+  case "$backup_dir" in
+    "$BACKUP_ROOT"/*) ;;
+    *) block "rollback_path_outside_backup_root" ;;
+  esac
+  [ -d "$backup_dir" ] || block "rollback_backup_missing"
+  live_conf="$(resolve_live_conf)"
+
+  if [ -f "$backup_dir/live.conf" ]; then
+    install -m 0644 "$backup_dir/live.conf" "$live_conf"
+  elif [ -f "$backup_dir/live.conf.missing" ]; then
+    rm -f "$live_conf"
+  else
+    block "rollback_live_config_receipt_missing"
+  fi
+
+  if [ -f "$backup_dir/maintenance.html" ]; then
+    install -d -m 0755 "$(dirname "$LIVE_MAINT")"
+    install -m 0644 "$backup_dir/maintenance.html" "$LIVE_MAINT"
+  elif [ -f "$backup_dir/maintenance.html.missing" ]; then
+    rm -f "$LIVE_MAINT"
+  else
+    block "rollback_maintenance_receipt_missing"
+  fi
+}
+
+write_last_backup() {
+  local backup_dir="$1" tmp
+  tmp="${LAST_BACKUP_FILE}.tmp.$$"
+  printf '%s\n' "$backup_dir" >"$tmp"
+  install -m 0600 "$tmp" "$LAST_BACKUP_FILE"
+  rm -f "$tmp"
+}
+
+apply_payload() {
+  local live_conf backup_dir
+  live_conf="$(resolve_live_conf)"
+  nginx -t
+  backup_dir="$(create_backup before-apply)"
+
+  install -d -m 0755 "$(dirname "$live_conf")" "$(dirname "$LIVE_MAINT")"
+  install -m 0644 "$CANONICAL_CONF" "$live_conf"
+  install -m 0644 "$CANONICAL_MAINT" "$LIVE_MAINT"
+
+  if ! nginx -t; then
+    restore_backup "$backup_dir"
+    nginx -t || true
+    echo "BACKUP_DIR=$backup_dir"
+    block "nginx_test_failed_rollback_restored"
+  fi
+  if ! systemctl reload nginx; then
+    restore_backup "$backup_dir"
+    nginx -t || true
+    systemctl reload nginx || true
+    echo "BACKUP_DIR=$backup_dir"
+    block "nginx_reload_failed_rollback_restored"
+  fi
+  if ! systemctl is-active --quiet nginx; then
+    restore_backup "$backup_dir"
+    nginx -t || true
+    systemctl reload nginx || true
+    echo "BACKUP_DIR=$backup_dir"
+    block "nginx_inactive_after_reload_rollback_restored"
+  fi
+
+  write_last_backup "$backup_dir"
+  audit "applied"
+  echo "EDGE_EXECUTOR_RESULT=applied"
+  echo "RUN_ID=$RUN_ID"
+  echo "BACKUP_DIR=$backup_dir"
+  echo "BACKUP_CONF=$backup_dir/live.conf"
+  echo "NGINX_TEST_OK=1"
+  echo "NGINX_RELOAD_DONE=1"
+}
+
+rollback_latest() {
+  local rollback_dir safety_backup
+  [ -f "$LAST_BACKUP_FILE" ] || block "last_backup_receipt_missing"
+  rollback_dir="$(cat "$LAST_BACKUP_FILE")"
+  case "$rollback_dir" in
+    "$BACKUP_ROOT"/*) ;;
+    *) block "last_backup_path_invalid" ;;
+  esac
+  safety_backup="$(create_backup before-rollback)"
+  restore_backup "$rollback_dir"
+  if ! nginx -t || ! systemctl reload nginx || ! systemctl is-active --quiet nginx; then
+    restore_backup "$safety_backup"
+    nginx -t || true
+    systemctl reload nginx || true
+    echo "SAFETY_BACKUP_DIR=$safety_backup"
+    block "rollback_verifier_failed_original_restored"
+  fi
+  write_last_backup "$safety_backup"
+  audit "rolled_back"
+  echo "EDGE_EXECUTOR_RESULT=rolled_back"
+  echo "RUN_ID=$RUN_ID"
+  echo "RESTORED_BACKUP_DIR=$rollback_dir"
+  echo "SAFETY_BACKUP_DIR=$safety_backup"
+  echo "NGINX_TEST_OK=1"
+  echo "NGINX_RELOAD_DONE=1"
+}
+
+if [ "${EUID:-$(id -u)}" -ne 0 ]; then
+  echo "BLOCKER executor_requires_root"
+  exit 77
+fi
+
+install -d -m 0700 "$STATE_DIR" "$BACKUP_ROOT"
+touch "$AUDIT_LOG"
+chmod 0600 "$AUDIT_LOG"
+assert_root_payload
+
+exec 9>"$LOCK_FILE"
+flock -w 30 9 || block "executor_lock_timeout"
+
+case "$ACTION" in
+  check)
+    print_status
+    ;;
+  apply)
+    apply_payload
+    print_status
+    ;;
+  rollback-latest)
+    rollback_latest
+    print_status
+    ;;
+  *)
+    block "unsupported_executor_action"
+    ;;
+esac
--- a/ops/reboot-recovery/awoooi-edge-maintenance.sudoers
+++ b/ops/reboot-recovery/awoooi-edge-maintenance.sudoers
@@ -0,0 +1,2 @@
+Cmnd_Alias AWOOOI_EDGE_MAINTENANCE = /usr/local/sbin/awoooi-edge-maintenance check, /usr/local/sbin/awoooi-edge-maintenance apply, /usr/local/sbin/awoooi-edge-maintenance rollback-latest
+ollama ALL=(root) NOPASSWD: AWOOOI_EDGE_MAINTENANCE
--- a/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh
+++ b/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh
@@ -0,0 +1,220 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+umask 077
+
+ACTION="${1:-check}"
+if [ "$#" -ne 1 ]; then
+  echo "BLOCKER invalid_hygiene_executor_arguments"
+  exit 64
+fi
+
+CANONICAL_DIR="/usr/local/share/awoooi-host188-hygiene"
+CANONICAL_SCRIPT="${CANONICAL_DIR}/awoooi-startup.sh"
+CANONICAL_UNIT="${CANONICAL_DIR}/awoooi-startup.service"
+LIVE_SCRIPT="/usr/local/bin/awoooi-startup.sh"
+LIVE_UNIT="/etc/systemd/system/awoooi-startup.service"
+POSTGRES_START_CONF="/etc/postgresql/14/main/start.conf"
+STATE_DIR="/var/lib/awoooi-host188-hygiene"
+BACKUP_ROOT="${STATE_DIR}/backups"
+LAST_BACKUP_FILE="${STATE_DIR}/last-backup"
+AUDIT_LOG="/var/log/awoooi-host188-hygiene.log"
+LOCK_FILE="/run/lock/awoooi-host188-hygiene.lock"
+RUN_ID="host188-hygiene-$(date +%Y%m%dT%H%M%S%z)-$$"
+
+audit() {
+  printf '%s run_id=%s action=%s result=%s\n' \
+    "$(date --iso-8601=seconds)" "$RUN_ID" "$ACTION" "$1" >>"$AUDIT_LOG"
+}
+
+block() {
+  echo "HOST188_HYGIENE_EXECUTOR_RESULT=blocked"
+  echo "RUN_ID=$RUN_ID"
+  echo "BLOCKER $1"
+  audit "blocked:$1" || true
+  exit 75
+}
+
+sha256_file() {
+  sha256sum "$1" | awk '{print $1}'
+}
+
+postgres_start_mode() {
+  awk '!/^[[:space:]]*#/ && NF {print $1; exit}' "$POSTGRES_START_CONF" 2>/dev/null || true
+}
+
+assert_canonical_payload() {
+  local file
+  for file in "$CANONICAL_SCRIPT" "$CANONICAL_UNIT"; do
+    [ -f "$file" ] || block "canonical_hygiene_payload_missing"
+    [ "$(stat -c '%u' "$file")" = "0" ] || block "canonical_hygiene_payload_not_root_owned"
+    if find "$file" -prune -perm /022 -print -quit | grep -q .; then
+      block "canonical_hygiene_payload_is_mutable"
+    fi
+  done
+  grep -q 'CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \
+    || block "canonical_clawbot_rebuild_guard_missing"
+  grep -q 'CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \
+    || block "canonical_containerd_repair_guard_missing"
+  grep -q 'DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \
+    || block "canonical_docker_repair_guard_missing"
+  grep -q 'KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \
+    || block "canonical_kine_write_guard_missing"
+  grep -q 'SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \
+    || block "canonical_signoz_owner_guard_missing"
+  grep -q '^TimeoutStartSec=600$' "$CANONICAL_UNIT" \
+    || block "canonical_startup_timeout_invalid"
+}
+
+assert_recovery_runtime() {
+  local state mount_ok=0
+  state="$(docker inspect -f '{{.State.Running}} {{.HostConfig.NetworkMode}}' k3s-postgres-recovery 2>/dev/null || true)"
+  [ "$state" = "true host" ] || block "postgres_recovery_container_not_host_running"
+  if docker inspect -f '{{range .Mounts}}{{println .Source .Destination}}{{end}}' k3s-postgres-recovery 2>/dev/null \
+    | awk '$2 == "/var/lib/postgresql/14/main" && $1 == "/var/lib/postgresql/14/main" {found=1} END {exit !found}'; then
+    mount_ok=1
+  fi
+  [ "$mount_ok" = "1" ] || block "postgres_recovery_data_mount_mismatch"
+  pg_isready -h localhost -p 5432 >/dev/null 2>&1 \
+    || block "postgres_recovery_not_accepting_connections"
+}
+
+print_status() {
+  local start_mode script_parity=0 unit_parity=0 pg_failed startup_failed last_backup=""
+  start_mode="$(postgres_start_mode)"
+  [ -f "$LIVE_SCRIPT" ] && [ "$(sha256_file "$LIVE_SCRIPT")" = "$(sha256_file "$CANONICAL_SCRIPT")" ] \
+    && script_parity=1
+  [ -f "$LIVE_UNIT" ] && [ "$(sha256_file "$LIVE_UNIT")" = "$(sha256_file "$CANONICAL_UNIT")" ] \
+    && unit_parity=1
+  pg_failed="$(systemctl is-failed postgresql@14-main.service 2>/dev/null || true)"
+  startup_failed="$(systemctl is-failed awoooi-startup.service 2>/dev/null || true)"
+  [ -f "$LAST_BACKUP_FILE" ] && last_backup="$(cat "$LAST_BACKUP_FILE")"
+
+  echo "AWOOOI_HOST188_HYGIENE_EXECUTOR=1"
+  echo "HOST188_HYGIENE_EXECUTOR_READY=1"
+  echo "RUN_ID=$RUN_ID"
+  echo "POSTGRES_RECOVERY_READY=1"
+  echo "POSTGRES_START_MODE=${start_mode:-unknown}"
+  echo "STARTUP_SCRIPT_PARITY=$script_parity"
+  echo "STARTUP_UNIT_PARITY=$unit_parity"
+  echo "POSTGRES_UNIT_STATE=${pg_failed:-unknown}"
+  echo "STARTUP_UNIT_STATE=${startup_failed:-unknown}"
+  echo "CANONICAL_SCRIPT_SHA256=$(sha256_file "$CANONICAL_SCRIPT")"
+  echo "CANONICAL_UNIT_SHA256=$(sha256_file "$CANONICAL_UNIT")"
+  echo "LAST_BACKUP_DIR=$last_backup"
+  if [ "$start_mode" = "manual" ] \
+    && [ "$script_parity" = "1" ] \
+    && [ "$unit_parity" = "1" ] \
+    && [ "$pg_failed" != "failed" ] \
+    && [ "$startup_failed" != "failed" ]; then
+    echo "HOST188_HYGIENE_READY=1"
+  else
+    echo "HOST188_HYGIENE_READY=0"
+  fi
+}
+
+create_backup() {
+  local backup_dir="${BACKUP_ROOT}/${RUN_ID}-before-reconcile"
+  install -d -m 0700 "$backup_dir"
+  cp -a "$POSTGRES_START_CONF" "$backup_dir/start.conf"
+  cp -a "$LIVE_SCRIPT" "$backup_dir/awoooi-startup.sh"
+  cp -a "$LIVE_UNIT" "$backup_dir/awoooi-startup.service"
+  printf '%s\n' "$backup_dir"
+}
+
+restore_backup() {
+  local backup_dir="$1"
+  case "$backup_dir" in
+    "$BACKUP_ROOT"/*) ;;
+    *) block "hygiene_rollback_path_invalid" ;;
+  esac
+  [ -f "$backup_dir/start.conf" ] \
+    && [ -f "$backup_dir/awoooi-startup.sh" ] \
+    && [ -f "$backup_dir/awoooi-startup.service" ] \
+    || block "hygiene_rollback_payload_missing"
+  install -m 0644 "$backup_dir/start.conf" "$POSTGRES_START_CONF"
+  install -m 0755 "$backup_dir/awoooi-startup.sh" "$LIVE_SCRIPT"
+  install -m 0644 "$backup_dir/awoooi-startup.service" "$LIVE_UNIT"
+  systemctl daemon-reload
+}
+
+write_last_backup() {
+  local tmp="${LAST_BACKUP_FILE}.tmp.$$"
+  printf '%s\n' "$1" >"$tmp"
+  install -m 0600 "$tmp" "$LAST_BACKUP_FILE"
+  rm -f "$tmp"
+}
+
+reconcile() {
+  local backup_dir start_tmp
+  assert_recovery_runtime
+  backup_dir="$(create_backup)"
+  start_tmp="${POSTGRES_START_CONF}.candidate.$$"
+  printf '# Managed by awoooi-host188-hygiene; recovery container owns automatic startup.\nmanual\n' >"$start_tmp"
+  install -m 0644 "$start_tmp" "$POSTGRES_START_CONF"
+  rm -f "$start_tmp"
+  install -m 0755 "$CANONICAL_SCRIPT" "$LIVE_SCRIPT"
+  install -m 0644 "$CANONICAL_UNIT" "$LIVE_UNIT"
+  if ! systemd-analyze verify "$LIVE_UNIT"; then
+    restore_backup "$backup_dir"
+    block "startup_unit_verify_failed_rollback_restored"
+  fi
+  systemctl daemon-reload
+  systemctl reset-failed postgresql@14-main.service awoooi-startup.service
+  assert_recovery_runtime
+  if [ "$(postgres_start_mode)" != "manual" ] \
+    || [ "$(systemctl is-failed postgresql@14-main.service 2>/dev/null || true)" = "failed" ] \
+    || [ "$(systemctl is-failed awoooi-startup.service 2>/dev/null || true)" = "failed" ]; then
+    restore_backup "$backup_dir"
+    block "host188_hygiene_postverify_failed_rollback_restored"
+  fi
+  write_last_backup "$backup_dir"
+  audit "reconciled"
+  echo "HOST188_HYGIENE_EXECUTOR_RESULT=reconciled"
+  echo "RUN_ID=$RUN_ID"
+  echo "BACKUP_DIR=$backup_dir"
+  echo "DAEMON_RELOAD_DONE=1"
+  echo "FAILED_STATE_RESET_DONE=1"
+  echo "PRODUCT_SERVICE_RESTARTS=0"
+}
+
+rollback_latest() {
+  [ -f "$LAST_BACKUP_FILE" ] || block "hygiene_last_backup_missing"
+  restore_backup "$(cat "$LAST_BACKUP_FILE")"
+  systemctl reset-failed postgresql@14-main.service awoooi-startup.service
+  audit "rolled_back"
+  echo "HOST188_HYGIENE_EXECUTOR_RESULT=rolled_back"
+  echo "RUN_ID=$RUN_ID"
+  echo "PRODUCT_SERVICE_RESTARTS=0"
+}
+
+if [ "${EUID:-$(id -u)}" -ne 0 ]; then
+  echo "BLOCKER hygiene_executor_requires_root"
+  exit 77
+fi
+
+install -d -m 0700 "$STATE_DIR" "$BACKUP_ROOT"
+touch "$AUDIT_LOG"
+chmod 0600 "$AUDIT_LOG"
+assert_canonical_payload
+assert_recovery_runtime
+
+exec 9>"$LOCK_FILE"
+flock -w 30 9 || block "hygiene_executor_lock_timeout"
+
+case "$ACTION" in
+  check)
+    print_status
+    ;;
+  reconcile)
+    reconcile
+    print_status
+    ;;
+  rollback-latest)
+    rollback_latest
+    print_status
+    ;;
+  *)
+    block "unsupported_hygiene_executor_action"
+    ;;
+esac
--- a/ops/reboot-recovery/awoooi-host188-hygiene.sudoers
+++ b/ops/reboot-recovery/awoooi-host188-hygiene.sudoers
@@ -0,0 +1,2 @@
+Cmnd_Alias AWOOOI_HOST188_HYGIENE = /usr/local/sbin/awoooi-host188-hygiene check, /usr/local/sbin/awoooi-host188-hygiene reconcile, /usr/local/sbin/awoooi-host188-hygiene rollback-latest
+ollama ALL=(root) NOPASSWD: AWOOOI_HOST188_HYGIENE