fix(agent99): recover host188 edge services

2026-07-14 23:16:07 +08:00
parent 8862da3929
commit 1431abba54
18 changed files with 874 additions and 57 deletions
--- a/scripts/ops/ansible-validate.sh
+++ b/scripts/ops/ansible-validate.sh
@@ -31,6 +31,7 @@ python3 ops/runner/guard-gitea-runner-pressure.py --root "$ROOT_DIR"
 echo "== Shell 語法 =="
 bash -n \
  scripts/reboot-recovery/full-stack-cold-start-check.sh \
+  scripts/reboot-recovery/host188-edge-services-recover.sh \
  scripts/reboot-recovery/full-stack-recovery-scorecard.sh \
  scripts/reboot-recovery/dr-offsite-operator-checklist.sh \
  scripts/reboot-recovery/wait-dr-offsite-ready.sh \
--- a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh
+++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh
@@ -231,7 +231,9 @@ probe_public_routes() {
    "https://stock.wooo.work/api/v1/system/freshness" \
    "https://gitea.wooo.work/" \
    "https://registry.wooo.work/v2/" \
-    "https://harbor.wooo.work/"
+    "https://harbor.wooo.work/" \
+    "https://n8n.wooo.work/" \
+    "https://ollama.wooo.work/"
  do
    code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
    [ -n "$code" ] || code="000"
--- a/scripts/reboot-recovery/cold-start-textfile-exporter.sh
+++ b/scripts/reboot-recovery/cold-start-textfile-exporter.sh
@@ -14,7 +14,7 @@ CHECK_TIMEOUT_SECONDS="${CHECK_TIMEOUT_SECONDS:-240}"
 CHECK_WATCH_INTERVAL_SECONDS="${CHECK_WATCH_INTERVAL_SECONDS:-10}"
 CHECK_WATCH_MAX_ATTEMPTS="${CHECK_WATCH_MAX_ATTEMPTS:-3}"
 HOST_LABEL="${AIOPS_HOST_LABEL:-110}"
-SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_120_121_188}"
+SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"
 LOCK_FILE="${LOCK_FILE:-/tmp/awoooi-cold-start-textfile-exporter.lock}"

 escape_label() {
@@ -39,6 +39,8 @@ write_metric_file() {
  local public_route_tls_blocker="${15}"
  local host_120_unreachable_blocker="${16}"
  local backup_health_blocker="${17}"
+  local host_112_unreachable_blocker="${18}"
+  local host_112_guest_not_ready_blocker="${19}"
  local host scope
  host=$(escape_label "$HOST_LABEL")
  scope=$(escape_label "$SCOPE_LABEL")
@@ -80,6 +82,8 @@ awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="k3s_node_fi
 awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="public_route_tls_failure",target="public_https"} $public_route_tls_blocker
 awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="120"} $host_120_unreachable_blocker
 awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="backup_health_blocked",target="110"} $backup_health_blocker
+awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="host_unreachable",target="112"} $host_112_unreachable_blocker
+awoooi_cold_start_blocker_reason{host="$host",scope="$scope",reason="guest_readiness_failed",target="112"} $host_112_guest_not_ready_blocker
 METRICS
 }

@@ -102,7 +106,7 @@ if [ ! -x "$CHECK_SCRIPT" ]; then
  tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
  last_green=$(cat "$state_file" 2>/dev/null || echo 0)
  printf 'CHECK_SCRIPT not executable: %s\n' "$CHECK_SCRIPT" >"$log_file"
-  write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0
+  write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" 127 0 0 0 1 0 0 0 1 "$last_green" 0 0 0 0 0 0
  chmod 0644 "$tmp_metric"
  mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
  exit 0
@@ -131,6 +135,8 @@ k3s_node_fs_blocker=0
 public_route_tls_blocker=0
 host_120_unreachable_blocker=0
 backup_health_blocker=0
+host_112_unreachable_blocker=0
+host_112_guest_not_ready_blocker=0

 if [ -n "$summary_line" ]; then
  monitor_up=1
@@ -166,6 +172,14 @@ if grep -Eq 'BLOCKED 110 backup health has stale expected jobs' "$log_file"; the
  backup_health_blocker=1
 fi

+if grep -Eq 'BLOCKED (ping 192\.168\.0\.112|ssh port 192\.168\.0\.112:22|112 canonical guest readback unavailable)' "$log_file"; then
+  host_112_unreachable_blocker=1
+fi
+
+if grep -Eq 'BLOCKED 112 (canonical guest readback schema missing|systemd not running|guest readiness failed|console or required services not ready|Wazuh independent verifier failed|guest recovery timer not ready|cold-start probe write boundary violated)' "$log_file"; then
+  host_112_guest_not_ready_blocker=1
+fi
+
 end_ts=$(date +%s)
 if [ "$green" -eq 1 ]; then
  printf '%s\n' "$end_ts" >"$state_file"
@@ -173,6 +187,6 @@ fi
 last_green=$(cat "$state_file" 2>/dev/null || echo 0)

 tmp_metric=$(mktemp "$TEXTFILE_DIR/.cold_start_recovery.XXXXXX")
-write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker"
+write_metric_file "$tmp_metric" "$end_ts" "$((end_ts - start_ts))" "$exit_code" "$monitor_up" "$pass" "$warn" "$blocked" "$green" "$degraded" "$blocked_state" "$check_failed" "$last_green" "$k3s_node_fs_blocker" "$public_route_tls_blocker" "$host_120_unreachable_blocker" "$backup_health_blocker" "$host_112_unreachable_blocker" "$host_112_guest_not_ready_blocker"
 chmod 0644 "$tmp_metric"
 mv "$tmp_metric" "$TEXTFILE_DIR/$OUTPUT_NAME"
--- a/scripts/reboot-recovery/full-stack-cold-start-check.sh
+++ b/scripts/reboot-recovery/full-stack-cold-start-check.sh
@@ -246,6 +246,13 @@ registry_code_ready() {
  [ "$code" = "200" ] || [ "$code" = "401" ]
 }

+readback_value() {
+  local text="$1" key="$2" rest
+  rest="${text#*${key}=}"
+  [ "$rest" != "$text" ] || return 1
+  printf '%s' "${rest%% *}"
+}
+
 probe_tcp() {
  local host="$1"
  local port="$2"
@@ -254,11 +261,11 @@ probe_tcp() {

 print_neighbor_rows() {
  if command -v arp >/dev/null 2>&1; then
-    arp -an | grep -E '192\.168\.0\.(110|120|121|188)'
+    arp -an | grep -E '192\.168\.0\.(110|112|120|121|188)'
    return $?
  fi
  if command -v ip >/dev/null 2>&1; then
-    ip neigh show | grep -E '192\.168\.0\.(110|120|121|188)'
+    ip neigh show | grep -E '192\.168\.0\.(110|112|120|121|188)'
    return $?
  fi
  return 1
@@ -267,14 +274,14 @@ print_neighbor_rows() {
 print_header() {
  echo "AWOOOI full-stack cold-start check"
  date '+%Y-%m-%d %H:%M:%S %Z'
-  echo "Scope: 110 / 120 / 121 / 188. 112 Kali is intentionally skipped."
+  echo "Scope: 110 / 112 / 120 / 121 / 188."
  echo "Baseline: ops/reboot-recovery/full-stack-cold-start-baseline.yml"
 }

 check_network() {
  log_section "P0-NETWORK"
  local host
-  for host in 110 120 121 188; do
+  for host in 110 112 120 121 188; do
    if ping -c 1 -W 2 "192.168.0.$host" >/dev/null 2>&1; then
      ok "ping 192.168.0.$host"
    else
@@ -297,6 +304,50 @@ check_network() {
  fi
 }

+check_112() {
+  log_section "P0-112-SECURITY"
+  local out schema systemd_state guest_ready console_ready services_ready
+  local recovery_timer_ready manager_verifier_ready runtime_write_performed
+  if ! out=$(host_cmd "kali@192.168.0.112" \
+      '/usr/local/sbin/awoooi-host112-guest-readiness --check' 2>&1); then
+    fail "112 canonical guest readback unavailable"
+    echo "$out"
+    return
+  fi
+
+  schema="$(readback_value "$out" schema_version || true)"
+  systemd_state="$(readback_value "$out" systemd_state || true)"
+  guest_ready="$(readback_value "$out" guest_ready || true)"
+  console_ready="$(readback_value "$out" console_ready || true)"
+  services_ready="$(readback_value "$out" services_ready || true)"
+  recovery_timer_ready="$(readback_value "$out" recovery_timer_ready || true)"
+  manager_verifier_ready="$(readback_value "$out" manager_independent_verifier_ready || true)"
+  runtime_write_performed="$(readback_value "$out" runtime_write_performed || true)"
+  echo "HOST112_READBACK schema=${schema:-missing} systemd=${systemd_state:-missing} guest=${guest_ready:-missing} console=${console_ready:-missing} services=${services_ready:-missing} timer=${recovery_timer_ready:-missing} manager_verifier=${manager_verifier_ready:-missing} runtime_write=${runtime_write_performed:-missing}"
+
+  [ "$schema" = "host112_guest_recovery_v2" ] \
+    && ok "112 canonical guest readback schema verified" \
+    || fail "112 canonical guest readback schema missing"
+  [ "$systemd_state" = "running" ] \
+    && ok "112 systemd running" \
+    || fail "112 systemd not running"
+  [ "$guest_ready" = "1" ] \
+    && ok "112 guest readiness verified" \
+    || fail "112 guest readiness failed"
+  [ "$console_ready" = "1" ] && [ "$services_ready" = "1" ] \
+    && ok "112 console and required services ready" \
+    || fail "112 console or required services not ready"
+  [ "$manager_verifier_ready" = "1" ] \
+    && ok "112 Wazuh independent verifier ready" \
+    || fail "112 Wazuh independent verifier failed"
+  [ "$recovery_timer_ready" = "1" ] \
+    && ok "112 guest recovery timer ready" \
+    || fail "112 guest recovery timer not ready"
+  [ "$runtime_write_performed" = "0" ] \
+    && ok "112 cold-start probe remained read-only" \
+    || fail "112 cold-start probe write boundary violated"
+}
+
 check_188() {
  log_section "P0-188-DATA"
  local out
@@ -660,6 +711,8 @@ check_public_routes() {
    "langfuse|https://langfuse.wooo.work/"
    "bitan|https://bitan.wooo.work/"
    "aiops|https://aiops.wooo.work/"
+    "n8n|https://n8n.wooo.work/"
+    "open_webui|https://ollama.wooo.work/"
  )

  for item in "${routes[@]}"; do
@@ -1067,6 +1120,7 @@ fi

 print_header
 check_network
+check_112
 check_188
 check_110
 check_k3s
--- a/scripts/reboot-recovery/host188-edge-services-recover.sh
+++ b/scripts/reboot-recovery/host188-edge-services-recover.sh
@@ -0,0 +1,288 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODE="check"
+TRACE_ID=""
+RUN_ID=""
+WORK_ITEM_ID=""
+
+HOST_ID="192.168.0.188"
+N8N_IMAGE="harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:dbe732489e5b8941aa89ca71b320ccb7f80319fcccd0f2c9e05f561562d804e5"
+N8N_IMAGE_ID="sha256:19c1ad26c0c285fddd7fc1300ff670b7977afebf52a21de4dfb2a5a2b2a75e2b"
+OPEN_WEBUI_IMAGE="harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:a673af6e7fab29822a0d32d71184d3f0277347a693633c86372643590c921915"
+OPEN_WEBUI_IMAGE_ID="sha256:5bddf2d93bc307b99c1fde8b8d0c2dc3c563e0e24fc3eebb6488c539fa55d080"
+BACKUP_IMAGE="harbor.wooo.work/dockerhub-cache/library/alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc"
+N8N_DIR="/home/ollama/n8n"
+N8N_OVERRIDE="${N8N_DIR}/docker-compose.agent99.yml"
+OPEN_WEBUI_DIR="/home/ollama/open-webui"
+OPEN_WEBUI_COMPOSE="${OPEN_WEBUI_DIR}/docker-compose.yml"
+RECEIPT_ROOT="/home/ollama/agent99-runtime-recovery"
+
+usage() {
+  printf 'usage: %s --check | --apply --trace-id ID --run-id ID --work-item-id ID\n' "$0" >&2
+  exit 64
+}
+
+valid_id() {
+  [[ "$1" =~ ^[A-Za-z0-9._:-]+$ ]]
+}
+
+while (($#)); do
+  case "$1" in
+    --check)
+      MODE="check"
+      shift
+      ;;
+    --apply)
+      MODE="apply"
+      shift
+      ;;
+    --trace-id)
+      (($# >= 2)) || usage
+      TRACE_ID="$2"
+      shift 2
+      ;;
+    --run-id)
+      (($# >= 2)) || usage
+      RUN_ID="$2"
+      shift 2
+      ;;
+    --work-item-id)
+      (($# >= 2)) || usage
+      WORK_ITEM_ID="$2"
+      shift 2
+      ;;
+    *)
+      usage
+      ;;
+  esac
+done
+
+if [[ "$MODE" == "apply" ]]; then
+  [[ -n "$TRACE_ID" && -n "$RUN_ID" && -n "$WORK_ITEM_ID" ]] || usage
+  valid_id "$TRACE_ID" && valid_id "$RUN_ID" && valid_id "$WORK_ITEM_ID" || usage
+fi
+
+bool_json() {
+  if [[ "$1" == "true" ]]; then printf 'true'; else printf 'false'; fi
+}
+
+http_status() {
+  local url="$1"
+  local resolve_arg="${2:-}"
+  local status
+  if [[ -n "$resolve_arg" ]]; then
+    status="$(curl -k -sS -L -o /dev/null -w '%{http_code}' --resolve "$resolve_arg" --max-time 12 "$url" 2>/dev/null || true)"
+  else
+    status="$(curl -sS -L -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)"
+  fi
+  [[ "$status" =~ ^[0-9]{3}$ ]] || status="000"
+  printf '%s' "$status"
+}
+
+non_5xx() {
+  local status="$1"
+  [[ "$status" =~ ^[0-9]{3}$ ]] && ((10#$status >= 200 && 10#$status < 500))
+}
+
+container_value() {
+  local name="$1"
+  local format="$2"
+  docker inspect "$name" --format "$format" 2>/dev/null || true
+}
+
+collect_state() {
+  local runtime_write="${1:-false}"
+  local terminal="${2:-observed}"
+  local receipt_path="${3:-}"
+
+  local n8n_status n8n_image n8n_restart n8n_restarts n8n_local n8n_public
+  local webui_status webui_health webui_image webui_restart webui_restarts webui_local webui_public
+  local n8n_config_ok=false webui_config_ok=false n8n_ok=false webui_ok=false overall=false
+
+  n8n_status="$(container_value n8n '{{.State.Status}}')"
+  n8n_image="$(container_value n8n '{{.Image}}')"
+  n8n_restart="$(container_value n8n '{{.HostConfig.RestartPolicy.Name}}')"
+  n8n_restarts="$(container_value n8n '{{.RestartCount}}')"
+  [[ "$n8n_restarts" =~ ^[0-9]+$ ]] || n8n_restarts=0
+  n8n_local="$(http_status 'http://127.0.0.1:5678/')"
+  n8n_public="$(http_status 'https://n8n.wooo.work/' 'n8n.wooo.work:443:127.0.0.1')"
+
+  webui_status="$(container_value open-webui '{{.State.Status}}')"
+  webui_health="$(container_value open-webui '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}')"
+  webui_image="$(container_value open-webui '{{.Image}}')"
+  webui_restart="$(container_value open-webui '{{.HostConfig.RestartPolicy.Name}}')"
+  webui_restarts="$(container_value open-webui '{{.RestartCount}}')"
+  [[ "$webui_restarts" =~ ^[0-9]+$ ]] || webui_restarts=0
+  webui_local="$(http_status 'http://127.0.0.1:3010/')"
+  webui_public="$(http_status 'https://ollama.wooo.work/' 'ollama.wooo.work:443:127.0.0.1')"
+
+  if [[ -f "$N8N_OVERRIDE" ]] && grep -Fqx "    image: ${N8N_IMAGE}" "$N8N_OVERRIDE"; then
+    n8n_config_ok=true
+  fi
+  if [[ -f "$OPEN_WEBUI_COMPOSE" ]] && grep -Fqx "    image: ${OPEN_WEBUI_IMAGE}" "$OPEN_WEBUI_COMPOSE"; then
+    webui_config_ok=true
+  fi
+
+  if [[ "$n8n_config_ok" == true && "$n8n_status" == running && "$n8n_image" == "$N8N_IMAGE_ID" && "$n8n_restart" == always ]] && non_5xx "$n8n_local" && non_5xx "$n8n_public"; then
+    n8n_ok=true
+  fi
+  if [[ "$webui_config_ok" == true && "$webui_status" == running && "$webui_health" == healthy && "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" && "$webui_restart" == always ]] && non_5xx "$webui_local" && non_5xx "$webui_public"; then
+    webui_ok=true
+  fi
+  if [[ "$n8n_ok" == true && "$webui_ok" == true ]]; then
+    overall=true
+  fi
+
+  printf '{"schemaVersion":"agent99_host188_edge_services_v1","mode":"%s","traceId":"%s","runId":"%s","workItemId":"%s","host":"%s","overallHealthy":%s,"runtimeWritePerformed":%s,"secretValueRead":false,"rawDataRead":false,"terminal":"%s","receiptPath":"%s","services":[{"name":"n8n","publicUrl":"https://n8n.wooo.work","containerStatus":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s},{"name":"open-webui","publicUrl":"https://ollama.wooo.work","containerStatus":"%s","health":"%s","imagePinned":%s,"restartPolicy":"%s","restartCount":%s,"localStatus":%s,"publicStatus":%s,"configManaged":%s,"verified":%s}],"rollback":{"automatic":false,"snapshotPreserved":true,"procedure":"stop_changed_containers_then_restore_run_scoped_snapshot_after_owner_break_glass"},"prohibitedActions":["host_reboot","vm_power_change","firewall_change","database_content_read","secret_read","external_registry_direct_pull"]}\n' \
+    "$MODE" "$TRACE_ID" "$RUN_ID" "$WORK_ITEM_ID" "$HOST_ID" \
+    "$(bool_json "$overall")" "$(bool_json "$runtime_write")" "$terminal" "$receipt_path" \
+    "$n8n_status" "$(bool_json "$([[ "$n8n_image" == "$N8N_IMAGE_ID" ]] && echo true || echo false)")" "$n8n_restart" "$n8n_restarts" "$n8n_local" "$n8n_public" "$(bool_json "$n8n_config_ok")" "$(bool_json "$n8n_ok")" \
+    "$webui_status" "$webui_health" "$(bool_json "$([[ "$webui_image" == "$OPEN_WEBUI_IMAGE_ID" ]] && echo true || echo false)")" "$webui_restart" "$webui_restarts" "$webui_local" "$webui_public" "$(bool_json "$webui_config_ok")" "$(bool_json "$webui_ok")"
+
+  [[ "$overall" == true ]]
+}
+
+if [[ "$MODE" == "check" ]]; then
+  if collect_state false observed ""; then
+    exit 0
+  fi
+  exit 2
+fi
+
+exec 9>/tmp/agent99-host188-edge-services.lock
+if ! flock -w 30 9; then
+  collect_state false single_flight_lock_timeout "" || true
+  exit 75
+fi
+
+RECEIPT_DIR="${RECEIPT_ROOT}/${RUN_ID}"
+OPERATION_LOG="${RECEIPT_DIR}/operation.log"
+RECEIPT_PATH="${RECEIPT_DIR}/receipt.json"
+
+emit_apply_failure() {
+  local exit_code="$1"
+  local result
+  trap - ERR
+  set +e
+  result="$(collect_state true "apply_failed_exit_${exit_code}" "$RECEIPT_PATH" || true)"
+  printf '%s\n' "$result" | tee "$RECEIPT_PATH"
+  chmod 600 "$RECEIPT_PATH" 2>/dev/null || true
+  exit "$exit_code"
+}
+
+mkdir -p "$RECEIPT_DIR" "$N8N_DIR" "$OPEN_WEBUI_DIR"
+chmod 700 "$RECEIPT_DIR"
+: >"$OPERATION_LOG"
+chmod 600 "$OPERATION_LOG"
+trap 'emit_apply_failure $?' ERR
+
+for image in "$BACKUP_IMAGE" "$N8N_IMAGE" "$OPEN_WEBUI_IMAGE"; do
+  timeout 1200 docker pull "$image" >>"$OPERATION_LOG" 2>&1
+done
+
+snapshot_with_helper() {
+  local source_path="$1"
+  local archive_name="$2"
+  if [[ -s "${RECEIPT_DIR}/${archive_name}" ]]; then
+    return 0
+  fi
+  docker run --rm --pull never --userns=host \
+    -v "${source_path}:/data:ro" -v "${RECEIPT_DIR}:/backup" \
+    "$BACKUP_IMAGE" sh -c "umask 077; tar czf /backup/${archive_name} -C /data ." \
+    >>"$OPERATION_LOG" 2>&1
+}
+
+[[ -d "${N8N_DIR}/data" && -f "${N8N_DIR}/docker-compose.yml" ]] || {
+  collect_state false n8n_base_assets_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
+  exit 3
+}
+
+if ! docker volume inspect open-webui >/dev/null 2>&1; then
+  docker volume create open-webui >>"$OPERATION_LOG"
+fi
+
+snapshot_with_helper "${N8N_DIR}/data" "n8n-data-before.tgz"
+docker run --rm --pull never --userns=host \
+  -v open-webui:/data:ro -v "${RECEIPT_DIR}:/backup" \
+  "$BACKUP_IMAGE" sh -c 'umask 077; tar czf /backup/open-webui-volume-before.tgz -C /data .' \
+  >>"$OPERATION_LOG" 2>&1
+docker run --rm --pull never --userns=host -v "${RECEIPT_DIR}:/backup" "$BACKUP_IMAGE" \
+  chown -R "$(id -u):$(id -g)" /backup >>"$OPERATION_LOG" 2>&1
+chmod 600 "${RECEIPT_DIR}/n8n-data-before.tgz" "${RECEIPT_DIR}/open-webui-volume-before.tgz"
+
+tmp_n8n="$(mktemp "${N8N_OVERRIDE}.XXXXXX")"
+cat >"$tmp_n8n" <<EOF
+services:
+  n8n:
+    image: ${N8N_IMAGE}
+EOF
+chmod 600 "$tmp_n8n"
+mv "$tmp_n8n" "$N8N_OVERRIDE"
+
+tmp_webui="$(mktemp "${OPEN_WEBUI_COMPOSE}.XXXXXX")"
+cat >"$tmp_webui" <<EOF
+services:
+  open-webui:
+    image: ${OPEN_WEBUI_IMAGE}
+    container_name: open-webui
+    restart: always
+    ports:
+      - "3010:8080"
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    environment:
+      OLLAMA_BASE_URL: http://host.docker.internal:11434
+      OLLAMA_BASE_URLS: http://host.docker.internal:11434
+    volumes:
+      - open-webui:/app/backend/data
+
+volumes:
+  open-webui:
+    external: true
+EOF
+chmod 600 "$tmp_webui"
+mv "$tmp_webui" "$OPEN_WEBUI_COMPOSE"
+
+remap_uid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subuid 2>/dev/null || true)"
+remap_gid="$(awk -F: '$1 == "dockremap" { print $2; exit }' /etc/subgid 2>/dev/null || true)"
+if [[ "$remap_uid" =~ ^[0-9]+$ && "$remap_gid" =~ ^[0-9]+$ ]]; then
+  node_uid=$((remap_uid + 1000))
+  node_gid=$((remap_gid + 1000))
+  docker run --rm --pull never --userns=host -v "${N8N_DIR}/data:/data" "$BACKUP_IMAGE" \
+    chown -R "${node_uid}:${node_gid}" /data >>"$OPERATION_LOG" 2>&1
+else
+  collect_state true docker_userns_mapping_missing "$RECEIPT_PATH" | tee "$RECEIPT_PATH" || true
+  exit 4
+fi
+
+docker compose --project-directory "$N8N_DIR" \
+  -f "${N8N_DIR}/docker-compose.yml" -f "$N8N_OVERRIDE" \
+  up -d --pull never >>"$OPERATION_LOG" 2>&1
+webui_project="$(docker inspect open-webui --format '{{index .Config.Labels "com.docker.compose.project"}}' 2>/dev/null || true)"
+if [[ -n "$(docker ps -aq --filter name='^/open-webui$')" && "$webui_project" != "open-webui" ]]; then
+  docker rm -f open-webui >>"$OPERATION_LOG" 2>&1
+fi
+docker compose --project-directory "$OPEN_WEBUI_DIR" -f "$OPEN_WEBUI_COMPOSE" \
+  up -d --pull never >>"$OPERATION_LOG" 2>&1
+
+deadline=$((SECONDS + 360))
+while ((SECONDS < deadline)); do
+  if collect_state true verifying "$RECEIPT_PATH" >/dev/null; then
+    break
+  fi
+  sleep 5
+done
+
+if result="$(collect_state true deployed_verified "$RECEIPT_PATH")"; then
+  trap - ERR
+  printf '%s\n' "$result" | tee "$RECEIPT_PATH"
+  chmod 600 "$RECEIPT_PATH"
+  exit 0
+fi
+
+result="$(collect_state true verifier_failed_snapshot_preserved "$RECEIPT_PATH" || true)"
+trap - ERR
+printf '%s\n' "$result" | tee "$RECEIPT_PATH"
+chmod 600 "$RECEIPT_PATH"
+exit 5
--- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py
+++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py
@@ -63,6 +63,31 @@ def test_full_stack_cold_start_check_bounds_ssh_probes() -> None:
    assert "SSH_110_RECOVERY_PACKAGE_NEXT_ACTION verify_or_preinstall_local_recovery_package_from_console_before_harbor_repair_retry" in text


+def test_full_stack_cold_start_includes_host112_as_a_required_p0_gate() -> None:
+    text = COLD_START_CHECK.read_text(encoding="utf-8")
+    baseline = (ROOT / "ops" / "reboot-recovery" / "full-stack-cold-start-baseline.yml").read_text(
+        encoding="utf-8"
+    )
+    exporter = (ROOT / "scripts" / "reboot-recovery" / "cold-start-textfile-exporter.sh").read_text(
+        encoding="utf-8"
+    )
+    deploy_verifier = VERIFY_DEPLOY.read_text(encoding="utf-8")
+
+    assert "Scope: 110 / 112 / 120 / 121 / 188." in text
+    assert "for host in 110 112 120 121 188; do" in text
+    assert 'log_section "P0-112-SECURITY"' in text
+    assert "awoooi-host112-guest-readiness --check" in text
+    assert 'check_112\ncheck_188' in text
+    assert '"112": "Kali security, Wazuh, guest readiness and bounded recovery"' in baseline
+    assert "- id: P0-112-SECURITY" in baseline
+    assert "112 Kali is intentionally skipped" not in text
+    assert "not part of cold-start release gate" not in baseline
+    assert 'SCOPE_LABEL="${AIOPS_SCOPE_LABEL:-110_112_120_121_188}"' in exporter
+    assert 'reason="host_unreachable",target="112"' in exporter
+    assert 'reason="guest_readiness_failed",target="112"' in exporter
+    assert 'scope="110_112_120_121_188"' in deploy_verifier
+
+
 def test_cold_start_momo_current_month_handles_no_new_source_without_false_warn() -> None:
    text = COLD_START_CHECK.read_text(encoding="utf-8")

--- a/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py
+++ b/scripts/reboot-recovery/tests/test_host188_edge_service_recovery_contract.py
@@ -0,0 +1,79 @@
+import json
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def read(relative: str) -> str:
+    return (ROOT / relative).read_text(encoding="utf-8")
+
+
+def test_agent99_config_covers_host188_edge_services_and_public_routes() -> None:
+    for name in ("agent99.config.example.json", "agent99.config.99.example.json"):
+        config = json.loads(read(name))
+        edge = config["edgeServiceRecovery"]
+
+        assert edge["enabled"] is True
+        assert edge["host"] == "192.168.0.188"
+        assert edge["scriptPath"] == "/home/ollama/bin/agent99-host188-edge-services-recover.sh"
+        assert set(edge["requiredServices"]) == {"n8n", "open-webui"}
+        assert "https://n8n.wooo.work" in config["publicUrls"]
+        assert "https://ollama.wooo.work" in config["publicUrls"]
+
+
+def test_host188_recovery_uses_only_internal_pinned_images() -> None:
+    script = read("scripts/reboot-recovery/host188-edge-services-recover.sh")
+
+    assert "harbor.wooo.work/dockerhub-cache/n8nio/n8n@sha256:" in script
+    assert "harbor.wooo.work/ghcr-cache/open-webui/open-webui@sha256:" in script
+    assert "harbor.wooo.work/dockerhub-cache/library/alpine@sha256:" in script
+    assert "ghcr.io/" not in script
+    assert "docker.n8n.io/" not in script
+    assert ":latest" not in script
+    assert "external_registry_direct_pull" in script
+
+
+def test_host188_check_mode_is_read_only_and_apply_is_bounded() -> None:
+    script = read("scripts/reboot-recovery/host188-edge-services-recover.sh")
+    check_block = script.split('if [[ "$MODE" == "check" ]]', 1)[1].split(
+        "exec 9>", 1
+    )[0]
+
+    for forbidden in (
+        "docker pull",
+        "docker run",
+        "docker compose",
+        "mkdir",
+        "mv ",
+        "chown",
+        "chmod",
+        "tee ",
+    ):
+        assert forbidden not in check_block
+
+    assert "flock -w 30" in script
+    assert "snapshot_with_helper" in script
+    assert "--pull never" in script
+    assert '"secretValueRead":false' in script
+    assert '"rawDataRead":false' in script
+    assert '"automatic":false' in script
+    assert "host_reboot" in script
+    assert "vm_power_change" in script
+
+
+def test_agent99_recover_binds_check_apply_verify_and_completion() -> None:
+    control = read("agent99-control-plane.ps1")
+    deploy = read("agent99-deploy.ps1")
+    playbook = read("infra/ansible/playbooks/188-ai-web.yml")
+
+    assert "function Invoke-AgentHost188EdgeRecovery" in control
+    assert 'New-AgentRecoveryPhase "host188_edge_services"' in control
+    assert 'New-AgentOutcomeCheck "host188_edge_services_ready"' in control
+    assert "$host188EdgeRecovery.runtimeWritePerformed" in control
+    assert "$host188EdgeRecovery.verified" in control
+    assert "container_local_upstream_and_public_https" in control
+    assert 'name = "canonical_public_route_coverage"' in deploy
+    assert "agent99-host188-edge-services-recover.sh --check" in playbook
+    assert "--work-item-id AIA-P0-006-EDGE-502-188" in playbook
+    assert "not ansible_check_mode" in playbook
--- a/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
+++ b/scripts/reboot-recovery/verify-cold-start-monitor-deploy.sh
@@ -129,10 +129,15 @@ require_remote_pattern \
  "110 deployed check script carries SSH host-key policy"

 require_remote_pattern \
-  'awoooi_cold_start_monitor_up{host="110",scope="110_120_121_188",mode="read_only"} 1' \
+  'awoooi_cold_start_monitor_up{host="110",scope="110_112_120_121_188",mode="read_only"} 1' \
  "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \
  "110 cold-start monitor produced parseable metrics"

+require_remote_pattern \
+  'awoooi_cold_start_blocker_reason{host="110",scope="110_112_120_121_188",reason="guest_readiness_failed",target="112"}' \
+  "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom" \
+  "110 cold-start monitor exports host112 readiness blocker series"
+
 report_runtime_state
 report_cold_start_alerts