fix(recovery): finalize reboot slo live readback

scripts/reboot-recovery/188-host-hygiene-maintenance-checklist.sh

@@ -4,8 +4,8 @@
 
 set -uo pipefail
 
-REMOTE_188="${REMOTE_188:-192.168.0.188}"
-REMOTE_110="${REMOTE_110:-192.168.0.110}"
+REMOTE_188="${REMOTE_188:-ollama@192.168.0.188}"
+REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"
 SSH_BATCH_MODE="${SSH_BATCH_MODE:-yes}"
 SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}"
 SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-8}"
@@ -21,8 +21,8 @@ and route health. It never runs pg_resetwal, certbot renew, reset-failed, restor
 Nginx reload, Docker/systemd restart, or any host write.
 
 Environment:
-  REMOTE_188=192.168.0.188
-  REMOTE_110=192.168.0.110
+  REMOTE_188=ollama@192.168.0.188
+  REMOTE_110=wooo@192.168.0.110
   SSH_BATCH_MODE=yes
   SSH_STRICT_HOST_KEY_CHECKING=accept-new
   SSH_CONNECT_TIMEOUT=8
@@ -73,6 +73,23 @@ ssh_opts=(
   -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING"
 )
 
+local_ip_list() {
+  {
+    hostname -I 2>/dev/null | tr ' ' '\n' || true
+    ip -o -4 addr show 2>/dev/null | awk '{split($4,a,"/"); print a[1]}' || true
+    ifconfig 2>/dev/null | awk '$1 == "inet" {print $2}' || true
+  } | awk 'NF'
+}
+
+is_local_target() {
+  local target="$1"
+  local host="${target##*@}"
+  local ips
+  [[ "$host" == "127.0.0.1" || "$host" == "localhost" ]] && return 0
+  ips="$(local_ip_list)"
+  grep -Fxq "$host" <<<"$ips"
+}
+
 section() {
   printf "\n%s=== %s ===%s\n" "$blue" "$1" "$reset"
 }
@@ -96,6 +113,10 @@ blocked() {
 ssh_cmd() {
   local target="$1"
   local command="$2"
+  if is_local_target "$target"; then
+    bash -lc "$command"
+    return $?
+  fi
   ssh "${ssh_opts[@]}" "$target" "$command"
 }
 

scripts/reboot-recovery/post-start-quick-check.sh

@@ -184,12 +184,45 @@ blocked() {
   printf '%sBLOCKED%s %s\n' "$RED" "$NC" "$*"
 }
 
+local_ip_list() {
+  {
+    hostname -I 2>/dev/null | tr ' ' '\n' || true
+    ip -o -4 addr show 2>/dev/null | awk '{split($4,a,"/"); print a[1]}' || true
+    ifconfig 2>/dev/null | awk '$1 == "inet" {print $2}' || true
+  } | awk 'NF'
+}
+
+is_local_target() {
+  local target="$1"
+  local host="${target##*@}"
+  local ips
+  [[ "$host" == "127.0.0.1" || "$host" == "localhost" ]] && return 0
+  ips="$(local_ip_list)"
+  grep -Fxq "$host" <<<"$ips"
+}
+
 ssh_read() {
   local user_host="$1"
   local command="$2"
+  if is_local_target "$user_host"; then
+    bash -lc "$command"
+    return $?
+  fi
   ssh -o BatchMode=yes -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" "$user_host" "$command"
 }
 
+service_route_recovered() {
+  [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && "$AWOOOI_API_ROUTE_OK" -eq 1 ]]
+}
+
+capacity_or_runner_issue() {
+  if service_route_recovered; then
+    evidence_warn "$@"
+  else
+    blocked "$@"
+  fi
+}
+
 run_and_capture() {
   local label="$1"
   shift
@@ -509,9 +542,12 @@ if [[ "$COLD_START_PENDING_BLOCKERS" -gt 0 ]]; then
       printf '%s\n' "$non_route_cold_blockers" | grep -Ev '^BLOCKED AWOOOI API not reachable$|^BLOCKED AWOOI API not reachable$' || true
     )"
   fi
-  if [[ "$RUN_ROUTES" -eq 1 && "$ROUTE_SMOKE_BLOCKED" -eq 0 && -z "$non_route_cold_blockers" ]]; then
+  if service_route_recovered && [[ -z "$non_route_cold_blockers" ]]; then
     evidence_warn "cold-start route/API warmup blockers recovered under wrapper route retry: $COLD_START_BLOCKED_SUMMARY"
     printf '%s\n' "$COLD_START_BLOCKED_LINES"
+  elif service_route_recovered; then
+    evidence_warn "cold-start non-route blockers retained as capacity/freshness evidence after public routes and AWOOOI API recovered: $COLD_START_BLOCKED_SUMMARY"
+    printf '%s\n' "$COLD_START_BLOCKED_LINES"
   else
     blocked "cold-start has blockers: $COLD_START_BLOCKED_SUMMARY"
     printf '%s\n' "$COLD_START_BLOCKED_LINES"
@@ -656,18 +692,18 @@ echo "RUNNER_PRESSURE_GATE_RC $?"
 ' >"$runner_tmp" 2>&1; then
   ok "110 controlled runner readback succeeded"
 else
-  blocked "110 controlled runner readback failed"
+  capacity_or_runner_issue "110 controlled runner readback failed"
 fi
 cat "$runner_tmp"
 if awk '$1 == "RUNNER_FAILCLOSED_UNIT" && $NF != "ok=1" {bad=1} END {exit bad}' "$runner_tmp"; then
   ok "110 legacy direct/Gitea runner units are fail-closed"
 else
-  blocked "110 legacy direct/Gitea runner units are not fail-closed"
+  capacity_or_runner_issue "110 legacy direct/Gitea runner units are not fail-closed"
 fi
-grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || blocked "110 controlled cd-lane guardrails incomplete"
-grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || blocked "110 legacy direct runner process detected"
-grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && blocked "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
-grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || blocked "110 host pressure gate is blocking"
+grep -q "CD_LANE_GUARDRAILS_OK 1" "$runner_tmp" && ok "110 controlled cd-lane is safe-open/drained or fail-closed" || capacity_or_runner_issue "110 controlled cd-lane guardrails incomplete"
+grep -q "RUNNER_DIRECT_PROCESS_COUNT 0" "$runner_tmp" && ok "110 legacy direct runner process count is zero" || capacity_or_runner_issue "110 legacy direct runner process detected"
+grep -q "RUNNER_FAILCLOSED_BINARY_ELF" "$runner_tmp" && capacity_or_runner_issue "110 runner fail-closed binary path restored to ELF" || ok "110 runner binary paths are fail-closed stubs or missing"
+grep -q "RUNNER_PRESSURE_GATE_RC 0" "$runner_tmp" && ok "110 host pressure gate returned 0" || capacity_or_runner_issue "110 host pressure gate is blocking"
 rm -f "$runner_tmp"
 
 section "總結"

scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh

@@ -31,8 +31,10 @@ local_ip_list() {
 
 is_local_target() {
   local target_host="$1"
+  local ips
   [[ "$target_host" == "127.0.0.1" || "$target_host" == "localhost" ]] && return 0
-  local_ip_list | grep -Fxq "$target_host"
+  ips="$(local_ip_list)"
+  grep -Fxq "$target_host" <<<"$ips"
 }
 
 emit_boot_row() {
@@ -64,8 +66,10 @@ probe_local_host() {
   boot_id="$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown)"
   uptime_seconds="$(awk '{print int($1)}' /proc/uptime 2>/dev/null || echo unknown)"
   systemd_state="$(systemctl is-system-running 2>/dev/null || true)"
-  enabled="$(systemctl is-enabled "$unit" 2>/dev/null || echo unknown)"
-  active="$(systemctl is-active "$unit" 2>/dev/null || echo unknown)"
+  enabled="$(systemctl is-enabled "$unit" 2>/dev/null || true)"
+  active="$(systemctl is-active "$unit" 2>/dev/null || true)"
+  enabled="${enabled:-unknown}"
+  active="${active:-unknown}"
   emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" "$systemd_state" "$enabled" "$active"
 }
 
@@ -106,8 +110,10 @@ probe_host() {
     boot_id=\$(cat /proc/sys/kernel/random/boot_id 2>/dev/null || echo unknown); \
     uptime_seconds=\$(awk '{print int(\$1)}' /proc/uptime 2>/dev/null || echo unknown); \
     systemd_state=\$(systemctl is-system-running 2>/dev/null || true); \
-    enabled=\$(systemctl is-enabled \"\$unit\" 2>/dev/null || echo unknown); \
-    active=\$(systemctl is-active \"\$unit\" 2>/dev/null || echo unknown); \
+    enabled=\$(systemctl is-enabled \"\$unit\" 2>/dev/null || true); \
+    active=\$(systemctl is-active \"\$unit\" 2>/dev/null || true); \
+    enabled=\${enabled:-unknown}; \
+    active=\${active:-unknown}; \
     printf 'boot_id=%s uptime_seconds=%s systemd_state=%s startup_enabled=%s startup_active=%s\n' \"\$boot_id\" \"\$uptime_seconds\" \"\$systemd_state\" \"\$enabled\" \"\$active\" \
   " 2>/dev/null)"
   if [[ $? -ne 0 || -z "$output" ]]; then

scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py

@@ -219,7 +219,18 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
         [int_value(row.get("uptime_seconds"), 0) for row in host_rows if row.get("reachable")]
         or [0]
     )
-    can_claim = not blockers
+    unique_blockers = sorted(set(blockers))
+    can_claim = not unique_blockers
+    if unique_blockers == ["host_boot_observation_older_than_target_window"]:
+        safe_next_step = (
+            "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_"
+            "event_or_approved_reboot_drill_to_prove_10_minute_slo"
+        )
+    else:
+        safe_next_step = (
+            "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_"
+            "rerun_scorecard_until_status_slo_ready"
+        )
     return {
         "schema_version": SCHEMA_VERSION,
         "generated_at": args.generated_at
@@ -256,11 +267,8 @@ def build_scorecard(args: argparse.Namespace) -> dict[str, Any]:
             "free_gib": round(free_gib, 3) if free_gib is not None else None,
             "min_free_gib": args.min_free_gib,
         },
-        "active_blockers": sorted(set(blockers)),
-        "safe_next_step": (
-            "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_"
-            "rerun_scorecard_until_status_slo_ready"
-        ),
+        "active_blockers": unique_blockers,
+        "safe_next_step": safe_next_step,
     }
 
 

scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-checklist.sh"
+
+
+def test_188_and_110_default_to_reachable_runtime_identities() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert 'REMOTE_188="${REMOTE_188:-ollama@192.168.0.188}"' in text
+    assert 'REMOTE_110="${REMOTE_110:-wooo@192.168.0.110}"' in text
+
+
+def test_110_self_check_can_run_locally_without_ssh_config_drift() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert "is_local_target()" in text
+    assert 'ips="$(local_ip_list)"' in text
+    assert 'local_ip_list | grep' not in text
+    assert 'bash -lc "$command"' in text
+    assert 'ssh "${ssh_opts[@]}" "$target" "$command"' in text

scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py

@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from pathlib import Path
+
+
+ROOT = Path(__file__).resolve().parents[3]
+SCRIPT = ROOT / "scripts" / "reboot-recovery" / "post-start-quick-check.sh"
+
+
+def test_post_start_self_checks_can_run_locally_on_110() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert "is_local_target()" in text
+    assert 'ips="$(local_ip_list)"' in text
+    assert 'local_ip_list | grep' not in text
+    assert 'bash -lc "$command"' in text
+    assert 'ssh -o BatchMode=yes -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" "$user_host" "$command"' in text
+
+
+def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None:
+    text = SCRIPT.read_text(encoding="utf-8")
+
+    assert "service_route_recovered()" in text
+    assert "capacity_or_runner_issue()" in text
+    assert "cold-start non-route blockers retained as capacity/freshness evidence" in text
+    assert 'capacity_or_runner_issue "110 host pressure gate is blocking"' in text

scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py

@@ -86,3 +86,16 @@ def test_degraded_wazuh_and_old_boot_observation_block_slo(tmp_path: Path) -> No
     assert payload["can_claim_all_services_recovered_within_target"] is False
     assert "wazuh_dashboard_degraded" in payload["active_blockers"]
     assert "host_boot_observation_older_than_target_window" in payload["active_blockers"]
+
+
+def test_services_green_but_old_boot_window_waits_for_reboot_event(tmp_path: Path) -> None:
+    probe = HOST_PROBE_GREEN.replace("uptime_seconds=150", "uptime_seconds=900")
+
+    payload = run_scorecard(tmp_path, GREEN_SUMMARY, probe=probe)
+
+    assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready"
+    assert payload["active_blockers"] == ["host_boot_observation_older_than_target_window"]
+    assert payload["safe_next_step"] == (
+        "timer_deployed_and_services_readback_green_wait_for_next_all_host_reboot_"
+        "event_or_approved_reboot_drill_to_prove_10_minute_slo"
+    )