From 7aae60cdff33c8f0b644aa3fcb2476d3dfa701ff Mon Sep 17 00:00:00 2001
From: ogt <ogt@awoooi.local>
Date: Fri, 10 Jul 2026 02:39:52 +0800
Subject: [PATCH] fix(reboot): apply fd budget followup for host pressure

---
 .../ai-log-triage-auto-recovery.sh            | 74 +++++++++++++++++++
 .../test_reboot_p0_operational_contract.py    |  4 +
 2 files changed, 78 insertions(+)

diff --git a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh
index 643c628c0..8a253dcec 100755
--- a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh
+++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh
@@ -51,6 +51,8 @@ Options:
 Controlled apply allowlist:
   - 110: start docker if inactive; run harbor-watchdog --repair-once or the
     Harbor log-first compose recovery path when the registry is unhealthy.
+  - 110/188: when logs show too many open files, install systemd FD-budget
+    drop-ins for containerd/docker/nginx and daemon-reload without restarting.
   - 120/121: after registry readiness, delete only ImagePullBackOff/ErrImagePull
     pods in the AWOOOI namespace so Kubernetes recreates them.
   - 188: start nginx/docker only if inactive.
@@ -515,7 +517,79 @@ done
   emit_action "APPLY_188_EDGE_CORE_UNITS" "188" "rc_$rc" "$file"
 }
 
+apply_fd_budget_followup_for_host() {
+  local host_label="$1"
+  local user_host="$2"
+  local file="$ARTIFACT_DIR/action-fd-budget-${host_label}.log"
+  local rc=0
+  {
+    host_cmd "$user_host" '
+set -uo pipefail
+echo ACTION_FD_BUDGET_FOLLOWUP=1
+SUDO=""
+if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then
+  SUDO="sudo -n"
+fi
+if [ -z "$SUDO" ]; then
+  echo FD_BUDGET_SKIPPED=sudo_not_ready
+  echo SECRET_VALUE_READ=false
+  echo RESTART_PERFORMED=false
+  echo DOCKER_PRUNE_PERFORMED=false
+  exit 0
+fi
+for unit in containerd.service docker.service nginx.service; do
+  load_state=$(systemctl show "$unit" -p LoadState --value 2>/dev/null || echo not-found)
+  active_state=$(systemctl is-active "$unit" 2>/dev/null || true)
+  echo FD_BUDGET_PRECHECK unit=$unit load_state=${load_state:-unknown} active_state=${active_state:-unknown}
+  if [ "$load_state" = "not-found" ]; then
+    echo FD_BUDGET_SKIP unit=$unit reason=unit_not_found
+    continue
+  fi
+  dropin_dir="/etc/systemd/system/${unit}.d"
+  dropin_file="$dropin_dir/99-awoooi-fd-budget.conf"
+  tmp_file=$(mktemp)
+  {
+    echo "[Unit]"
+    echo "StartLimitIntervalSec=300"
+    echo "StartLimitBurst=20"
+    echo ""
+    echo "[Service]"
+    echo "LimitNOFILE=1048576"
+  } >"$tmp_file"
+  $SUDO mkdir -p "$dropin_dir"
+  $SUDO install -m 0644 "$tmp_file" "$dropin_file"
+  rm -f "$tmp_file"
+  echo FD_BUDGET_DROPIN_WRITTEN unit=$unit file=$dropin_file
+done
+$SUDO systemctl daemon-reload
+for unit in containerd.service docker.service nginx.service; do
+  load_state=$(systemctl show "$unit" -p LoadState --value 2>/dev/null || echo not-found)
+  [ "$load_state" = "not-found" ] && continue
+  limit_nofile=$(systemctl show "$unit" -p LimitNOFILE --value 2>/dev/null || true)
+  dropins=$(systemctl show "$unit" -p DropInPaths --value 2>/dev/null | tr " " "," || true)
+  echo FD_BUDGET_READBACK unit=$unit limit_nofile=${limit_nofile:-unknown} dropins=${dropins:-none}
+done
+echo DAEMON_RELOAD_PERFORMED=true
+echo SECRET_VALUE_READ=false
+echo RESTART_PERFORMED=false
+echo DOCKER_PRUNE_PERFORMED=false
+'
+  } >"$file" 2>&1 || rc=$?
+  emit_action "APPLY_FD_BUDGET_FOLLOWUP_${host_label}" "$host_label" "rc_$rc" "$file"
+}
+
+apply_fd_budget_followup() {
+  apply_fd_budget_followup_for_host "110" "$HOST_110"
+  apply_fd_budget_followup_for_host "188" "$HOST_188"
+}
+
 apply_actions() {
+  if contains 'TOO_MANY_OPEN_FILES' "$findings_file"; then
+    apply_fd_budget_followup
+  else
+    emit_action "APPLY_FD_BUDGET_FOLLOWUP" "110_188" "skipped_no_fd_budget_finding" "$findings_file"
+  fi
+
   if contains 'HARBOR_REGISTRY_NOT_READY|REGISTRY_PORT_5000_NOT_LISTENING|DOCKER_SCOPE_TIMEOUT|CONTAINERD_TASK_CLEANUP_STUCK' "$findings_file"; then
     apply_110_controlled
   else
diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
index 35b31b19b..7a844d84a 100644
--- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
+++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py
@@ -163,6 +163,10 @@ def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None:
     assert "MODE=\"check\"" in runner
     assert "--apply" in runner
     assert "TOO_MANY_OPEN_FILES" in runner
+    assert "APPLY_FD_BUDGET_FOLLOWUP" in runner
+    assert "LimitNOFILE=1048576" in runner
+    assert "systemctl daemon-reload" in runner
+    assert "RESTART_PERFORMED=false" in runner
     assert "SSH_COMMAND_SESSION_TIMEOUT" in runner
     assert "SSH_AUTH_FAILED" in runner
     assert "DOCKER_SCOPE_TIMEOUT" in runner