From 4481f18aa163713a4ad17fb499bba995d3c33713 Mon Sep 17 00:00:00 2001 From: ogt Date: Thu, 9 Jul 2026 16:21:19 +0800 Subject: [PATCH] feat(reboot): add ai log triage auto recovery --- docs/runbooks/FULL-STACK-COLD-START-SOP.md | 6 +- docs/runbooks/REBOOT-RECOVERY-SOP.md | 12 +- .../ai-log-triage-auto-recovery.sh | 574 ++++++++++++++++++ ...oooi-full-host-reboot-orchestrator.service | 18 + ...awoooi-full-host-reboot-orchestrator.timer | 12 + .../awoooi-reboot-auto-recovery-slo.service | 3 + .../reboot-recovery/awoooi-telegram-notify.sh | 29 + .../full-host-reboot-orchestrator.sh | 302 +++++++++ .../install-full-host-reboot-orchestrator.sh | 148 +++++ .../reboot-auto-recovery-slo-exporter.sh | 39 +- .../reboot-auto-recovery-slo-scorecard.py | 54 ++ .../test_reboot_p0_operational_contract.py | 80 +++ .../windows-99/configure-vmware-autostart.ps1 | 89 +++ ...gure-windows-update-no-surprise-reboot.ps1 | 26 + 14 files changed, 1386 insertions(+), 6 deletions(-) create mode 100755 scripts/reboot-recovery/ai-log-triage-auto-recovery.sh create mode 100644 scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service create mode 100644 scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.timer create mode 100755 scripts/reboot-recovery/awoooi-telegram-notify.sh create mode 100755 scripts/reboot-recovery/full-host-reboot-orchestrator.sh create mode 100755 scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh create mode 100644 scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 create mode 100644 scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1 diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 2909e8863..270a7c2a7 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.130 -> Last updated: 2026-07-03 Asia/Taipei +> Version: v1.131 +> Last updated: 2026-07-09 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號,但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal;此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,8 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天,必須先重跑 live check,再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +v1.131 AI log triage controlled auto-recovery rule:每次全主機重啟後,10 分鐘 SLO verifier 不得只跑 read-only summary。`awoooi-reboot-auto-recovery-slo.service` 必須以 `AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply` 先執行 `scripts/reboot-recovery/ai-log-triage-auto-recovery.sh`,bounded 收集 99 / 110 / 120 / 121 / 188 logs、K3s events、VMware no-secret readback 與 public route status,自動輸出 findings / actions / summary,再接 host probe、post-reboot summary、scorecard 與 textfile metrics。2026-07-09 16:17 live check artifact `/tmp/awoooi-ai-log-triage-codex-20260709-161759` 的目前 truth 是 `FINDING_COUNT=4`、`FINDING_P0_COUNT=3`:`TOO_MANY_OPEN_FILES host=110_or_188`、`STOCK_UPSTREAM_NOT_READY host=110_188`、`EDGE_502_UPSTREAM_REFUSED host=188_edge`,另有 `HOST188_POSTGRES_SYSTEMD_FAILED` P2 hygiene;public route readback 為 AWOOOI health `200`、AWOOOI web `307`、Stock freshness `200`、Gitea `200`、registry `/v2/` `401`、Harbor `200`。這些不得再被當成噪音或人工 SOP;必須進 AI findings、Prometheus/Telegram blocker 與下一步 action matrix。允許的 `--apply` 只限低風險 allowlist:start inactive docker/nginx、Harbor watchdog log-first repair、registry ready 後刪 failed-pull pods 讓 K3s 重建。仍禁止 reboot、node drain、firewall cutover、DB destructive action、Docker prune、secret/session/.env/raw-volume reads。 + v1.130 Windows99 VMX backup/source search package rule:2026-07-03 18:19 最新 via-110 no-secret locator 已把目前 truth 從早先 `candidate_source_count=6` 修正為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`,且 `raw_path_output=false`、`path_fingerprint_only=true`、`secret_value_read=false`、`remote_write_performed=false`、`vm_power_change_performed=false`。因此 P0-4 下一步不得再引用 generic Ubuntu candidate;必須讀 `/api/v1/agents/windows99-vmx-source-backup-search-package` 與 priority readback 的 `windows99_vmx_source_backup_search_*` 欄位。當前 backup/source search receipt 顯示 120 / 121 / local 均未找到 verified 111 source,99 alias 無 Linux backup host mapping,110 / 188 / 112 SSH BatchMode 不可用,`candidate_source_count=0`、`blocked_host_count=4`、`apply_allowed_by_this_package=false`。固定下一步是離線備份 inventory 或授權 console source path,再取得 verified source path 後建立 scoped relink dry-run;仍不得 power on VM、restore、讀 VMX 內容、讀 secret、重啟 Windows / VMware service / host / Docker / Nginx / K3s / DB / firewall。 若只是重啟後要快速判斷能不能宣稱恢復,先跑機器可讀摘要:`scripts/reboot-recovery/post-reboot-readiness-summary.sh --no-color`。此腳本會呼叫一頁式總檢查、188 host hygiene checklist 與 Wazuh no-false-green repo gates,並把 delegated logs 和可重放的 `summary.txt` 留在 `/tmp/awoooi-post-reboot-readiness-*`。v1.75 起,同一輪驗收後續步驟必須吃同一個 `$ARTIFACT_DIR/summary.txt`,例如 `scripts/reboot-recovery/post-reboot-declaration-guard.py --summary-file "$ARTIFACT_DIR/summary.txt" --no-color` 與 `scripts/reboot-recovery/post-reboot-next-gate-dispatch.sh --summary-file "$ARTIFACT_DIR/summary.txt" --no-color`;不得在同一輪 evidence chain 反覆重跑 live probes 後混用不同時間點結論。v1.76 起,delegated cold-start 若在 K3s rollout / CD 替換瞬間出現單次 `BLOCKED AWOOOI API not reachable`,但 wrapper 自己的 public `https://awoooi.wooo.work/api/v1/health` route retry 已回 2xx,該 blocker 只列為 route/API warmup evidence warning;public API 仍失敗、其他 non-route blocker、或 retry 後未恢復時,仍維持 hard blocked。宣告 guard 會把 summary 轉成 allowed / forbidden declaration,避免把服務綠誤報成 DR complete、188 host hygiene、Wazuh registry recovered 或 runtime authorized。若 summary 顯示 `SERVICE_GREEN=1` 但 `NEXT_REQUIRED_GATES` 仍非空,再由 dispatch checklist 把尚未完成的 blocker 轉成 owner / evidence / forbidden-action checklist;需要機器可讀 intake 時,再跑 `scripts/reboot-recovery/post-reboot-next-gate-owner-packets.py --summary-file "$ARTIFACT_DIR/summary.txt" --output /tmp/awoooi-post-reboot-owner-packets.json` 產生 `awoooi_post_reboot_next_gate_owner_packets_v1` JSON,立刻跑 `scripts/reboot-recovery/post-reboot-owner-packet-contract-guard.py --packet-file /tmp/awoooi-post-reboot-owner-packets.json`,並用 `scripts/reboot-recovery/post-reboot-owner-response-template.py --owner-packet-file /tmp/awoooi-post-reboot-owner-packets.json --output /tmp/awoooi-post-reboot-owner-response-template.json` 產生只含 active gates 的 placeholder response。dispatch / packet / guard / template 均固定 `DISPATCH_AUTHORIZED=0`、`REQUEST_SENT_COUNT=0`、`OWNER_RESPONSE_ACCEPTED=0`、`HOST_WRITE_AUTHORIZED=0`、`SECRET_VALUE_COLLECTION_ALLOWED=0`、`RUNTIME_GATE=0`;guard 未通過時不得送 owner request、不得寫 escrow marker、不得進維護窗口、不得宣稱 DR / Wazuh registry complete。v1.74 起,任何 owner response JSON 還必須經過 `scripts/reboot-recovery/post-reboot-owner-response-preflight.py --no-color --owner-packet-file --response-file `:空模板、placeholder、secret payload、runtime action request、credential marker write、Wazuh active response / re-enroll / restart、Kali active scan 或缺少 active gate evidence 都必須 fail-closed;preflight 通過也只表示可進入獨立 reviewer acceptance,不是 runtime 授權。需要人工展開時,再跑 `scripts/reboot-recovery/post-start-quick-check.sh --no-color` 並以 `docs/runbooks/REBOOT-POST-START-QUICK-CHECK.md` 作為 fallback。長 SOP 保留完整背景、例外處理與 Plan B;短版 wrapper / checklist 負責每次 T+10 分鐘內的固定判定。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index 57fc34f37..da28bcf34 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.21 -> **最後更新**: 2026-07-03 (台北時間) +> **版本**: v5.22 +> **最後更新**: 2026-07-09 (台北時間) > **更新者**: Codex > **觸發事件**: 2026-07-03 18:19 Windows99 VMX backup/source blocked-host truth 上卷 @@ -21,6 +21,14 @@ ## 架構概覽與依賴圖 +### 2026-07-09 P0 AI Log Triage / Controlled Auto-Recovery 覆蓋 + +主機重啟後不得再靠人工逐台看 log 才判斷異常。固定第一步改為執行 `scripts/reboot-recovery/ai-log-triage-auto-recovery.sh --apply`;若只允許讀取,則用 `--check`,但 scorecard / 告警仍必須吃同一份 artifact。此 runner 會 bounded 收集 110 Docker / containerd / Harbor / Nginx、120 / 121 K3s events、188 edge Nginx / Docker / systemd、99 VMware no-secret readback 與 public route HTTP status,然後輸出 `findings.tsv`、`actions.tsv`、`summary.txt`。 + +本輪 2026-07-09 16:17 live evidence `/tmp/awoooi-ai-log-triage-codex-20260709-161759` 顯示 `FINDING_COUNT=4`、`FINDING_P0_COUNT=3`,P0 依序為 `TOO_MANY_OPEN_FILES host=110_or_188`、`STOCK_UPSTREAM_NOT_READY host=110_188`、`EDGE_502_UPSTREAM_REFUSED host=188_edge`,另有 `HOST188_POSTGRES_SYSTEMD_FAILED` P2 hygiene。public route readback 為 AWOOOI health `200`、AWOOOI web `307`、Stock freshness `200`、Gitea `200`、registry `/v2/` `401`、Harbor `200`;因此不得沿用稍早 Harbor 5000 refused / K3s ImagePullBackOff 舊 artifact 當作目前 truth。若後續 registry 再度不 ready,K3s pod delete 只能被自動化標為 `skipped_registry_not_ready`,不得宣稱 502 已修復。 + +`awoooi-reboot-auto-recovery-slo.service` 必須設定 `AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply`,並由 `reboot-auto-recovery-slo-exporter.sh` 在 host probe / readiness summary 前先跑 AI log triage。允許的 controlled apply 僅限:110 Docker inactive 時 start、Harbor watchdog `--repair-once` / harbor-log-first recovery、registry ready 後只刪 `ImagePullBackOff` / `ErrImagePull` pods 讓 Kubernetes 重建、188 nginx/docker inactive 時 start。仍禁止 reboot、node drain、firewall cutover、DB restore/drop/truncate、Docker prune、secret/session/.env/raw-volume reads。 + ### 2026-07-03 18:19 P0-4 VMX Backup / Source Search 覆蓋 最新 via-110 no-secret locator 已成功收斂為 `candidate_source_count=0`、`source_locator_status=blocked_missing_expected_vmx_source_no_candidate_found`,早先 `candidate_source_count=6` / generic Ubuntu candidate 僅保留為歷史 receipt,不再作為目前 relink 候選。`docs/operations/windows99-vmx-source-locator-receipt.snapshot.json` 已更新為 17:57 no-candidate truth。 diff --git a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh new file mode 100755 index 000000000..643c628c0 --- /dev/null +++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh @@ -0,0 +1,574 @@ +#!/usr/bin/env bash +# AWOOOI AI log triage and controlled auto-recovery runner. +# +# This runner turns post-reboot logs into machine-readable findings and applies +# only bounded, reversible recovery actions from an allowlist. It never reboots +# hosts, drains nodes, changes firewall rules, restores databases, prunes Docker, +# reads secrets, or reads raw runtime volumes. + +set -uo pipefail + +MODE="check" +ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-ai-log-triage-$(date '+%Y%m%d-%H%M%S')}" +SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-180}" +WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}" +WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS="${WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS:-210}" +SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}" +HOST_110="${HOST_110:-wooo@192.168.0.110}" +HOST_120="${HOST_120:-wooo@192.168.0.120}" +HOST_121="${HOST_121:-wooo@192.168.0.121}" +HOST_188="${HOST_188:-ollama@192.168.0.188}" +WINDOWS99_VIA_HOST="${WINDOWS99_VIA_HOST:-$HOST_110}" +AWOOOI_NAMESPACE="${AWOOOI_NAMESPACE:-awoooi-prod}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout=8 + -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=1 + -o NumberOfPasswordPrompts=0 + -o StrictHostKeyChecking="$SSH_STRICT_HOST_KEY_CHECKING" +) + +usage() { + cat <<'USAGE' +Usage: ai-log-triage-auto-recovery.sh [--check|--apply] [options] + +Options: + --check Collect logs, classify abnormalities, emit repair plan. + --apply Run the same triage, then apply allowlisted repairs. + --artifact-dir DIR Evidence directory. Default: /tmp/awoooi-ai-log-triage-... + --host-110 HOST SSH target for 110. Default: wooo@192.168.0.110 + --host-120 HOST SSH target for 120. Default: wooo@192.168.0.120 + --host-121 HOST SSH target for 121. Default: wooo@192.168.0.121 + --host-188 HOST SSH target for 188. Default: ollama@192.168.0.188 + --windows99-via-host HOST + SSH jump/control host for the no-secret Windows 99 probe. + -h, --help Show this help. + +Controlled apply allowlist: + - 110: start docker if inactive; run harbor-watchdog --repair-once or the + Harbor log-first compose recovery path when the registry is unhealthy. + - 120/121: after registry readiness, delete only ImagePullBackOff/ErrImagePull + pods in the AWOOOI namespace so Kubernetes recreates them. + - 188: start nginx/docker only if inactive. + +Forbidden even in --apply: + reboot, node drain, firewall cutover, DB restore/drop/truncate, Docker prune, + repo/ref destructive operations, secret/session/.env/raw-volume reads. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + --artifact-dir) + ARTIFACT_DIR="${2:?--artifact-dir requires a directory}" + shift + ;; + --host-110) + HOST_110="${2:?--host-110 requires a host}" + shift + ;; + --host-120) + HOST_120="${2:?--host-120 requires a host}" + shift + ;; + --host-121) + HOST_121="${2:?--host-121 requires a host}" + shift + ;; + --host-188) + HOST_188="${2:?--host-188 requires a host}" + shift + ;; + --windows99-via-host) + WINDOWS99_VIA_HOST="${2:?--windows99-via-host requires a host}" + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +mkdir -p "$ARTIFACT_DIR" +summary_file="$ARTIFACT_DIR/summary.txt" +findings_file="$ARTIFACT_DIR/findings.tsv" +actions_file="$ARTIFACT_DIR/actions.tsv" +: >"$summary_file" +: >"$findings_file" +: >"$actions_file" + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S %Z')" "$*" | tee -a "$summary_file" +} + +run_timeout() { + local seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout "$seconds" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "$seconds" "$@" + else + "$@" + fi +} + +host_has_ip() { + local expected_ip="$1" + if command -v ip >/dev/null 2>&1; then + ip -o -4 addr show 2>/dev/null | awk '{print $4}' | grep -q "^${expected_ip}/" && return 0 + fi + hostname -I 2>/dev/null | tr ' ' '\n' | grep -qx "$expected_ip" +} + +ssh_cmd() { + local user_host="$1" + local cmd="$2" + local quoted_cmd + printf -v quoted_cmd '%q' "$cmd" + ssh "${SSH_OPTS[@]}" "$user_host" \ + "if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_cmd}; else bash -lc ${quoted_cmd}; fi" +} + +host_cmd() { + local user_host="$1" + local cmd="$2" + case "$user_host" in + *@192.168.0.110) + if host_has_ip "192.168.0.110"; then + bash -lc "$cmd" + return + fi + ;; + *@192.168.0.120) + if host_has_ip "192.168.0.120"; then + bash -lc "$cmd" + return + fi + ;; + *@192.168.0.121) + if host_has_ip "192.168.0.121"; then + bash -lc "$cmd" + return + fi + ;; + *@192.168.0.188) + if host_has_ip "192.168.0.188"; then + bash -lc "$cmd" + return + fi + ;; + esac + ssh_cmd "$user_host" "$cmd" +} + +collect_host() { + local label="$1" + local user_host="$2" + local cmd="$3" + local file="$ARTIFACT_DIR/${label}.log" + local rc=0 + { + printf 'AWOOOI_AI_LOG_TRIAGE_COLLECT label=%s host=%s mode=%s\n' "$label" "$user_host" "$MODE" + host_cmd "$user_host" "$cmd" + } >"$file" 2>&1 || rc=$? + printf 'COLLECT label=%s rc=%s file=%s\n' "$label" "$rc" "$file" | tee -a "$summary_file" + return 0 +} + +contains() { + local pattern="$1" + shift + grep -Eiq "$pattern" "$@" 2>/dev/null +} + +emit_finding() { + local id="$1" + local severity="$2" + local host="$3" + local evidence="$4" + local next_action="$5" + printf '%s\t%s\t%s\t%s\t%s\n' "$id" "$severity" "$host" "$evidence" "$next_action" >>"$findings_file" + printf 'FINDING id=%s severity=%s host=%s evidence=%s next_action=%s\n' \ + "$id" "$severity" "$host" "$evidence" "$next_action" | tee -a "$summary_file" +} + +emit_action() { + local id="$1" + local host="$2" + local result="$3" + local evidence="$4" + printf '%s\t%s\t%s\t%s\n' "$id" "$host" "$result" "$evidence" >>"$actions_file" + printf 'ACTION id=%s host=%s result=%s evidence=%s\n' "$id" "$host" "$result" "$evidence" | tee -a "$summary_file" +} + +probe_public_routes() { + local file="$ARTIFACT_DIR/public-routes.log" + local url code + : >"$file" + for url in \ + "https://awoooi.wooo.work/api/v1/health" \ + "https://awoooi.wooo.work/" \ + "https://stock.wooo.work/api/v1/system/freshness" \ + "https://gitea.wooo.work/" \ + "https://registry.wooo.work/v2/" \ + "https://harbor.wooo.work/" + do + code="$(curl -k -sS -o /dev/null -w '%{http_code}' --max-time 12 "$url" 2>/dev/null || true)" + [ -n "$code" ] || code="000" + printf 'PUBLIC_ROUTE url=%s http_status=%s\n' "$url" "$code" | tee -a "$file" + done + printf 'COLLECT label=public-routes rc=0 file=%s\n' "$file" | tee -a "$summary_file" +} + +collect_windows99() { + local file="$ARTIFACT_DIR/windows99-vmware.log" + local rc=0 + if [ ! -x "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" ]; then + printf 'WINDOWS99_COLLECTOR_MISSING=1\n' >"$file" + printf 'COLLECT label=windows99-vmware rc=0 file=%s\n' "$file" | tee -a "$summary_file" + return 0 + fi + WINDOWS99_REMOTE_VERIFY_TIMEOUT="$WINDOWS99_REMOTE_VERIFY_TIMEOUT" \ + run_timeout "$WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" bash "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" \ + --check --via-host "$WINDOWS99_VIA_HOST" --users wooo --timeout 8 --max-auth-users 1 \ + >"$file" 2>&1 || rc=$? + printf 'COLLECT label=windows99-vmware rc=%s file=%s\n' "$rc" "$file" | tee -a "$summary_file" +} + +collect_all() { + collect_host "110" "$HOST_110" ' +set -uo pipefail +echo HOST=110 +date +uptime +echo DOCKER_STATE=$(systemctl is-active docker 2>/dev/null || true) +echo CONTAINERD_STATE=$(systemctl is-active containerd 2>/dev/null || true) +echo NGINX_STATE=$(systemctl is-active nginx 2>/dev/null || true) +echo HARBOR_WATCHDOG_STATE=$(systemctl is-active harbor-watchdog 2>/dev/null || true) +echo SYSTEMD_STATE=$(systemctl is-system-running 2>/dev/null || true) +curl -s -o /dev/null -w "HARBOR_LOCAL_V2_HTTP_STATUS=%{http_code}\n" --max-time 6 http://127.0.0.1:5000/v2/ 2>/dev/null || echo HARBOR_LOCAL_V2_HTTP_STATUS=000 +curl -k -s -o /dev/null -w "HARBOR_HOST_V2_HTTP_STATUS=%{http_code}\n" --max-time 6 https://192.168.0.110:5000/v2/ 2>/dev/null || echo HARBOR_HOST_V2_HTTP_STATUS=000 +curl -s -o /dev/null -w "STOCK_LOCAL_HTTP_STATUS=%{http_code}\n" --max-time 6 http://127.0.0.1:31235/api/v1/system/freshness 2>/dev/null || echo STOCK_LOCAL_HTTP_STATUS=000 +ss -ltn 2>/dev/null | awk "/:(5000|31235|3001|9090|9093|80|443) / {print \"LISTEN \" \$0}" || true +docker ps -a --format "CONTAINER name={{.Names}} status={{.Status}} ports={{.Ports}}" 2>/dev/null | grep -Ei "harbor|registry|stockplatform|gitea|prometheus|alertmanager|nginx" || true +journalctl -u docker -u containerd -u harbor-watchdog -u nginx --since "-45 min" --no-pager 2>/dev/null | tail -360 || true +' + + collect_host "120-k3s" "$HOST_120" " +set -uo pipefail +echo HOST=120 +date +uptime +echo K3S_STATE=\$(systemctl is-active k3s 2>/dev/null || true) +sudo -n kubectl get nodes -o wide 2>/dev/null || true +sudo -n kubectl -n $AWOOOI_NAMESPACE get pods -o wide 2>/dev/null || true +sudo -n kubectl -n $AWOOOI_NAMESPACE get events --sort-by=.lastTimestamp 2>/dev/null | tail -160 || true +journalctl -u k3s --since '-45 min' --no-pager 2>/dev/null | tail -180 || true +" + + collect_host "121-k3s" "$HOST_121" " +set -uo pipefail +echo HOST=121 +date +uptime +echo K3S_STATE=\$(systemctl is-active k3s 2>/dev/null || true) +sudo -n kubectl get nodes -o wide 2>/dev/null || true +sudo -n kubectl -A get pods -o wide 2>/dev/null | grep -Ei 'awoooi|ImagePullBackOff|ErrImagePull|CrashLoopBackOff|Pending' || true +sudo -n kubectl -A get events --sort-by=.lastTimestamp 2>/dev/null | tail -160 || true +journalctl -u k3s --since '-45 min' --no-pager 2>/dev/null | tail -180 || true +" + + collect_host "188-edge" "$HOST_188" ' +set -uo pipefail +echo HOST=188 +date +uptime +echo DOCKER_STATE=$(systemctl is-active docker 2>/dev/null || true) +echo NGINX_STATE=$(systemctl is-active nginx 2>/dev/null || true) +echo SYSTEMD_STATE=$(systemctl is-system-running 2>/dev/null || true) +systemctl --failed --no-legend 2>/dev/null || true +for target in \ + momo=http://127.0.0.1:5003/health \ + vibework=http://127.0.0.1:32336/ \ + fifa_web=http://127.0.0.1:3108/ \ + fifa_api=http://127.0.0.1:8000/health \ + tsenyang=http://127.0.0.1:3000/ +do + name=${target%%=*} + url=${target#*=} + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 6 "$url" 2>/dev/null || true) + [ -n "$code" ] || code=000 + echo LOCAL_ROUTE name=$name http_status=$code url=$url +done +ss -ltn 2>/dev/null | awk "/:(80|443|5003|32336|3108|8000|3000) / {print \"LISTEN \" \$0}" || true +docker ps --format "CONTAINER name={{.Names}} status={{.Status}} ports={{.Ports}}" 2>/dev/null | grep -Ei "momo|vibework|fifa|tsenyang|nginx|stock|awoooi" || true +journalctl -u nginx -u docker --since "-45 min" --no-pager 2>/dev/null | tail -360 || true +if [ -s /var/log/nginx/error.log ]; then tail -220 /var/log/nginx/error.log; fi +' + + collect_windows99 + probe_public_routes +} + +classify_findings() { + local log110="$ARTIFACT_DIR/110.log" + local log120="$ARTIFACT_DIR/120-k3s.log" + local log121="$ARTIFACT_DIR/121-k3s.log" + local log188="$ARTIFACT_DIR/188-edge.log" + local log99="$ARTIFACT_DIR/windows99-vmware.log" + local public="$ARTIFACT_DIR/public-routes.log" + local registry_public_ready=0 + local registry_110_readback_missing=0 + + contains 'PUBLIC_ROUTE url=https://registry\.wooo\.work/v2/ http_status=(200|401)' "$public" \ + && registry_public_ready=1 + contains 'Timeout, server .* not responding|Connection timed out|Operation timed out|Permission denied' "$log110" \ + && registry_110_readback_missing=1 + + contains 'failed to create inotify fd: too many open files|too many open files' "$log110" "$log188" \ + && emit_finding "TOO_MANY_OPEN_FILES" "P0" "110_or_188" "containerd_or_nginx_fd_budget_exhausted" "apply_fd_budget_followup_and_reduce_noisy_restart_loop" + + contains 'Timeout, server .* not responding|Connection timed out|Operation timed out' "$log110" "$ARTIFACT_DIR"/action-110*.log "$ARTIFACT_DIR"/action-registry-readback.log \ + && emit_finding "SSH_COMMAND_SESSION_TIMEOUT" "P0" "110" "ssh_auth_or_tcp_reachable_but_command_session_timeout" "repair_110_sshd_session_control_path_or_use_console" + + contains 'Permission denied \(publickey,password\)|Permission denied \(publickey\)' "$log188" \ + && emit_finding "SSH_AUTH_FAILED" "P0" "188" "ssh_publickey_auth_failed_for_configured_user" "fix_host_target_user_or_authorized_keys" + + contains 'Timeout waiting for systemd to create docker-[0-9a-f]+\.scope|unable to apply cgroup configuration' "$log110" \ + && emit_finding "DOCKER_SCOPE_TIMEOUT" "P0" "110" "systemd_cgroup_scope_creation_timeout" "run_harbor_log_first_recovery_and_keep_docker_restart_bounded" + + contains 'failed to delete task|failed to delete container from containerd|RWLayer of container .* unexpectedly nil|No such container' "$log110" \ + && emit_finding "CONTAINERD_TASK_CLEANUP_STUCK" "P0" "110" "containerd_task_cleanup_or_rwlayer_stuck" "avoid_prune_recover_registry_then_scorecard" + + if [ "$registry_public_ready" -ne 1 ] \ + && [ "$registry_110_readback_missing" -ne 1 ] \ + && ! contains 'HARBOR_(LOCAL|HOST)_V2_HTTP_STATUS=(200|401)' "$log110"; then + emit_finding "HARBOR_REGISTRY_NOT_READY" "P0" "110" "127.0.0.1_or_192.168.0.110_5000_not_ready" "apply_harbor_watchdog_repair_once" + fi + + if [ "$registry_public_ready" -ne 1 ] \ + && [ "$registry_110_readback_missing" -ne 1 ] \ + && ! contains 'LISTEN .*:5000 ' "$log110"; then + emit_finding "REGISTRY_PORT_5000_NOT_LISTENING" "P0" "110" "ss_missing_5000_listener" "apply_harbor_watchdog_repair_once" + fi + + if contains 'STOCK_LOCAL_HTTP_STATUS=(000|502|503|504)|connect\(\) failed.*31235|upstream: "http://192\.168\.0\.110:31235|PUBLIC_ROUTE url=https://stock\.wooo\.work/api/v1/system/freshness http_status=(502|503|504|000)' "$log110" "$log188" "$public"; then + emit_finding "STOCK_UPSTREAM_NOT_READY" "P0" "110_188" "stock_31235_or_edge_upstream_unhealthy" "verify_stock_container_and_public_freshness" + fi + + if contains '^awoooi-[^[:space:]]+[[:space:]]+[0-9]/[0-9][[:space:]]+(ImagePullBackOff|ErrImagePull)' "$log120" "$log121"; then + emit_finding "K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY" "P0" "120_121" "awoooi_pods_cannot_pull_from_110_registry" "after_registry_ready_delete_only_failed_pods" + fi + + if contains 'connect\(\) failed .* while connecting to upstream|upstream.*(502|refused)|PUBLIC_ROUTE .* http_status=502' "$log188" "$public"; then + emit_finding "EDGE_502_UPSTREAM_REFUSED" "P0" "188_edge" "nginx_alive_but_upstreams_refusing_or_raw_502" "repair_upstreams_then_public_route_readback" + fi + + if contains 'VMWARE_VMX_PROCESS_MATCH alias=112 count=1|KALI_LOCK_DIAGNOSIS=in_use_by_running_vmware_vmx_process|exclusive lock' "$log99"; then + emit_finding "WINDOWS99_KALI_LOCK_IN_USE" "P1" "99" "kali_vmx_lock_owned_by_running_vm" "do_not_delete_lock_attach_existing_console" + fi + + if contains 'postgresql@14-main\.service.*failed|postgresql@14-main\.service failed' "$log188"; then + emit_finding "HOST188_POSTGRES_SYSTEMD_FAILED" "P2" "188" "system_postgresql_failed_after_reboot" "keep_product_containers_green_schedule_hygiene_fix" + fi +} + +apply_110_controlled() { + local file="$ARTIFACT_DIR/action-110.log" + local rc=0 + collect_host "action-110-pre" "$HOST_110" ' +set -uo pipefail +echo ACTION_110_PRECHECK=1 +echo DOCKER_STATE=$(systemctl is-active docker 2>/dev/null || true) +curl -s -o /dev/null -w "HARBOR_LOCAL_V2_HTTP_STATUS=%{http_code}\n" --max-time 6 http://127.0.0.1:5000/v2/ 2>/dev/null || echo HARBOR_LOCAL_V2_HTTP_STATUS=000 +' + + { + host_cmd "$HOST_110" ' +set -uo pipefail +echo ACTION_110_CONTROLLED_APPLY=1 +date +SUDO="" +if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then + SUDO="sudo -n" +fi +docker_state=$(systemctl is-active docker 2>/dev/null || true) +echo BEFORE_DOCKER_STATE=${docker_state:-unknown} +if [ "$docker_state" != "active" ]; then + echo START_DOCKER=1 + $SUDO systemctl start docker.socket docker.service || true +fi +for i in $(seq 1 18); do + docker_state=$(systemctl is-active docker 2>/dev/null || true) + echo DOCKER_WAIT attempt=$i state=${docker_state:-unknown} + [ "$docker_state" = "active" ] && break + sleep 5 +done +if command -v /usr/local/bin/harbor-watchdog.sh >/dev/null 2>&1; then + echo HARBOR_REPAIR_METHOD=harbor_watchdog + $SUDO /usr/local/bin/harbor-watchdog.sh --repair-once || true +elif [ -f /home/wooo/harbor/harbor/docker-compose.yml ]; then + echo HARBOR_REPAIR_METHOD=compose_log_first + cd /home/wooo/harbor/harbor + docker compose up -d harbor-log || true + for i in $(seq 1 18); do + status=$(docker inspect --format="{{.State.Health.Status}}" harbor-log 2>/dev/null || echo missing) + echo HARBOR_LOG_WAIT attempt=$i status=$status + [ "$status" = "healthy" ] && break + sleep 5 + done + docker compose up -d || true +else + echo HARBOR_REPAIR_METHOD=missing +fi +curl -s -o /dev/null -w "AFTER_HARBOR_LOCAL_V2_HTTP_STATUS=%{http_code}\n" --max-time 8 http://127.0.0.1:5000/v2/ 2>/dev/null || echo AFTER_HARBOR_LOCAL_V2_HTTP_STATUS=000 +curl -k -s -o /dev/null -w "AFTER_HARBOR_HOST_V2_HTTP_STATUS=%{http_code}\n" --max-time 8 https://192.168.0.110:5000/v2/ 2>/dev/null || echo AFTER_HARBOR_HOST_V2_HTTP_STATUS=000 +ss -ltn 2>/dev/null | awk "/:5000 / {print \"AFTER_LISTEN_5000 \" \$0}" || true +' + } >"$file" 2>&1 || rc=$? + emit_action "APPLY_110_REGISTRY_HARBOR" "110" "rc_$rc" "$file" +} + +registry_ready_from_110() { + local file="$ARTIFACT_DIR/action-registry-readback.log" + collect_host "action-registry-readback" "$HOST_110" ' +set -uo pipefail +curl -s -o /dev/null -w "HARBOR_LOCAL_V2_HTTP_STATUS=%{http_code}\n" --max-time 8 http://127.0.0.1:5000/v2/ 2>/dev/null || echo HARBOR_LOCAL_V2_HTTP_STATUS=000 +curl -k -s -o /dev/null -w "HARBOR_HOST_V2_HTTP_STATUS=%{http_code}\n" --max-time 8 https://192.168.0.110:5000/v2/ 2>/dev/null || echo HARBOR_HOST_V2_HTTP_STATUS=000 +ss -ltn 2>/dev/null | awk "/:5000 / {print \"LISTEN_5000 \" \$0}" || true +' + contains 'HARBOR_(LOCAL|HOST)_V2_HTTP_STATUS=(200|401)|LISTEN_5000' "$file" "$ARTIFACT_DIR/action-registry-readback.log" +} + +apply_k3s_failed_pods() { + local host_label="$1" + local user_host="$2" + local file="$ARTIFACT_DIR/action-${host_label}.log" + local rc=0 + { + host_cmd "$user_host" " +set -uo pipefail +echo ACTION_K3S_FAILED_PODS host=$host_label namespace=$AWOOOI_NAMESPACE +code=\$(curl -k -s -o /dev/null -w '%{http_code}' --max-time 8 https://192.168.0.110:5000/v2/ 2>/dev/null || true) +[ -n \"\$code\" ] || code=000 +echo REGISTRY_HTTP_STATUS=\$code +case \"\$code\" in + 200|401) ;; + *) + echo SKIP_K3S_POD_DELETE=registry_not_ready + exit 0 + ;; +esac +pods=\$(sudo -n kubectl -n $AWOOOI_NAMESPACE get pods --no-headers 2>/dev/null | awk '\$0 ~ /ImagePullBackOff|ErrImagePull/ {print \$1}' || true) +if [ -z \"\$pods\" ]; then + echo FAILED_PULL_PODS=none + exit 0 +fi +printf 'FAILED_PULL_PODS=%s\n' \"\$(echo \"\$pods\" | tr '\n' ',' | sed 's/,\$//')\" +echo \"\$pods\" | xargs -r sudo -n kubectl -n $AWOOOI_NAMESPACE delete pod --wait=false +sudo -n kubectl -n $AWOOOI_NAMESPACE get pods -o wide || true +" + } >"$file" 2>&1 || rc=$? + emit_action "APPLY_K3S_FAILED_PULL_PODS" "$host_label" "rc_$rc" "$file" +} + +apply_188_controlled() { + local file="$ARTIFACT_DIR/action-188.log" + local rc=0 + { + host_cmd "$HOST_188" ' +set -uo pipefail +echo ACTION_188_CONTROLLED_APPLY=1 +SUDO="" +if command -v sudo >/dev/null 2>&1 && sudo -n true >/dev/null 2>&1; then + SUDO="sudo -n" +fi +for unit in docker nginx; do + state=$(systemctl is-active "$unit" 2>/dev/null || true) + echo BEFORE_${unit}_STATE=${state:-unknown} + if [ "$state" != "active" ]; then + echo START_UNIT=$unit + $SUDO systemctl start "$unit" || true + fi + echo AFTER_${unit}_STATE=$(systemctl is-active "$unit" 2>/dev/null || true) +done +' + } >"$file" 2>&1 || rc=$? + emit_action "APPLY_188_EDGE_CORE_UNITS" "188" "rc_$rc" "$file" +} + +apply_actions() { + if contains 'HARBOR_REGISTRY_NOT_READY|REGISTRY_PORT_5000_NOT_LISTENING|DOCKER_SCOPE_TIMEOUT|CONTAINERD_TASK_CLEANUP_STUCK' "$findings_file"; then + apply_110_controlled + else + emit_action "APPLY_110_REGISTRY_HARBOR" "110" "skipped_no_registry_finding" "$findings_file" + fi + + if registry_ready_from_110 && contains 'K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY' "$findings_file"; then + apply_k3s_failed_pods "120" "$HOST_120" + apply_k3s_failed_pods "121" "$HOST_121" + elif contains 'K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY' "$findings_file"; then + emit_action "APPLY_K3S_FAILED_PULL_PODS" "120_121" "skipped_registry_not_ready" "$ARTIFACT_DIR/action-registry-readback.log" + else + emit_action "APPLY_K3S_FAILED_PULL_PODS" "120_121" "skipped_no_image_pull_finding" "$findings_file" + fi + + if contains 'EDGE_502_UPSTREAM_REFUSED' "$findings_file"; then + apply_188_controlled + else + emit_action "APPLY_188_EDGE_CORE_UNITS" "188" "skipped_no_edge_finding" "$findings_file" + fi +} + +write_final_summary() { + local finding_count action_count p0_count p1_count p2_count + finding_count="$(wc -l <"$findings_file" | tr -d ' ')" + action_count="$(wc -l <"$actions_file" | tr -d ' ')" + p0_count="$(awk -F'\t' '$2 == "P0" {count++} END {print count+0}' "$findings_file")" + p1_count="$(awk -F'\t' '$2 == "P1" {count++} END {print count+0}' "$findings_file")" + p2_count="$(awk -F'\t' '$2 == "P2" {count++} END {print count+0}' "$findings_file")" + { + printf 'AWOOOI_AI_LOG_TRIAGE_AUTO_RECOVERY=1\n' + printf 'MODE=%s\n' "$MODE" + printf 'ARTIFACT_DIR=%s\n' "$ARTIFACT_DIR" + printf 'FINDING_COUNT=%s\n' "$finding_count" + printf 'FINDING_P0_COUNT=%s\n' "$p0_count" + printf 'FINDING_P1_COUNT=%s\n' "$p1_count" + printf 'FINDING_P2_COUNT=%s\n' "$p2_count" + printf 'ACTION_COUNT=%s\n' "$action_count" + printf 'FINDINGS_FILE=%s\n' "$findings_file" + printf 'ACTIONS_FILE=%s\n' "$actions_file" + } | tee -a "$summary_file" +} + +log "AI log triage started mode=$MODE artifact_dir=$ARTIFACT_DIR" +collect_all +classify_findings +if [ "$MODE" = "apply" ]; then + apply_actions +fi +probe_public_routes +write_final_summary + +if awk -F'\t' '$2 == "P0" {found=1} END {exit found ? 0 : 1}' "$findings_file"; then + exit 2 +fi +exit 0 diff --git a/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service b/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service new file mode 100644 index 000000000..46ee1cf5f --- /dev/null +++ b/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service @@ -0,0 +1,18 @@ +[Unit] +Description=AWOOOI full-host reboot recovery orchestrator +Documentation=file:/opt/awoooi/reboot-recovery/docs/runbooks/REBOOT-RECOVERY-SOP.md +Wants=network-online.target +After=network-online.target + +[Service] +Type=oneshot +WorkingDirectory=/opt/awoooi/reboot-recovery +Environment=ARTIFACT_ROOT=/var/log/awoooi/reboot-recovery +Environment=RECOVERY_SLO_SECONDS=600 +EnvironmentFile=-/etc/awoooi/reboot-recovery.env +ConditionPathExists=/opt/awoooi/reboot-recovery/scripts/reboot-recovery/full-host-reboot-orchestrator.sh +ExecStart=/usr/bin/env bash /opt/awoooi/reboot-recovery/scripts/reboot-recovery/full-host-reboot-orchestrator.sh --apply +TimeoutStartSec=900 + +[Install] +WantedBy=multi-user.target diff --git a/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.timer b/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.timer new file mode 100644 index 000000000..a35b3eeab --- /dev/null +++ b/scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.timer @@ -0,0 +1,12 @@ +[Unit] +Description=Run AWOOOI reboot recovery orchestrator during boot and every minute until healthy + +[Timer] +OnBootSec=45s +OnUnitActiveSec=60s +AccuracySec=10s +Unit=awoooi-full-host-reboot-orchestrator.service +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service index 722b6d8fe..8d646eb7c 100644 --- a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service +++ b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service @@ -14,6 +14,9 @@ Environment=ROOT_DIR=/home/wooo/awoooi-reboot-recovery-slo Environment=TEXTFILE_DIR=/home/wooo/node_exporter_textfiles Environment=LOG_DIR=/home/wooo/reboot-recovery Environment=TARGET_MINUTES=10 +Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply +Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420 +Environment=AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS=20 ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh TimeoutStartSec=600 StandardOutput=journal diff --git a/scripts/reboot-recovery/awoooi-telegram-notify.sh b/scripts/reboot-recovery/awoooi-telegram-notify.sh new file mode 100755 index 000000000..4eee1609e --- /dev/null +++ b/scripts/reboot-recovery/awoooi-telegram-notify.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +PHASE="" +STATUS="" +ARTIFACT_DIR="" +MESSAGE="" + +while [[ $# -gt 0 ]]; do + case "$1" in + --phase) PHASE="$2"; shift 2 ;; + --status) STATUS="$2"; shift 2 ;; + --artifact-dir) ARTIFACT_DIR="$2"; shift 2 ;; + --message) MESSAGE="$2"; shift 2 ;; + *) echo "Unknown argument: $1" >&2; exit 2 ;; + esac +done + +: "${TELEGRAM_BOT_TOKEN:?TELEGRAM_BOT_TOKEN is required in host-local env}" +: "${TELEGRAM_CHAT_ID:?TELEGRAM_CHAT_ID is required in host-local env}" + +TEXT="[AWOOOI reboot-recovery] phase=${PHASE} status=${STATUS} +${MESSAGE} +artifact=${ARTIFACT_DIR}" + +curl --fail --silent --show-error --max-time 10 \ + --request POST "https://api.telegram.org/bot${TELEGRAM_BOT_TOKEN}/sendMessage" \ + --data-urlencode "chat_id=${TELEGRAM_CHAT_ID}" \ + --data-urlencode "text=${TEXT}" >/dev/null diff --git a/scripts/reboot-recovery/full-host-reboot-orchestrator.sh b/scripts/reboot-recovery/full-host-reboot-orchestrator.sh new file mode 100755 index 000000000..6dccbb139 --- /dev/null +++ b/scripts/reboot-recovery/full-host-reboot-orchestrator.sh @@ -0,0 +1,302 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# Full-host reboot recovery orchestrator. +# This is intentionally a coordinator: irreversible actions stay out of scope, +# while bounded service-start/recovery hooks are delegated to audited scripts. + +MODE="check" +ARTIFACT_ROOT="${ARTIFACT_ROOT:-/tmp/awoooi-full-host-reboot-recovery}" +RUN_ID="${RUN_ID:-$(date -u +%Y%m%dT%H%M%SZ)}" +HOSTS="${HOSTS:-99 110 111 112 120 121 188}" +PUBLIC_ROUTES="${PUBLIC_ROUTES:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/healthz https://stock.wooo.work/api/healthz https://gitea.wooo.work/ https://harbor.wooo.work/ https://registry.wooo.work/v2/}" +SSH_TIMEOUT_SECONDS="${SSH_TIMEOUT_SECONDS:-12}" +RECOVERY_SLO_SECONDS="${RECOVERY_SLO_SECONDS:-600}" +TELEGRAM_NOTIFY_SCRIPT="${TELEGRAM_NOTIFY_SCRIPT:-}" +DISABLE_TELEGRAM="${DISABLE_TELEGRAM:-0}" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ARTIFACT_DIR="${ARTIFACT_ROOT%/}/${RUN_ID}" +EVENTS_TSV="${ARTIFACT_DIR}/events.tsv" +FINDINGS_TSV="${ARTIFACT_DIR}/findings.tsv" +ACTIONS_TSV="${ARTIFACT_DIR}/actions.tsv" +SUMMARY_TXT="${ARTIFACT_DIR}/summary.txt" + +usage() { + cat <<'USAGE' +Usage: + full-host-reboot-orchestrator.sh [--check|--apply] [--artifact-dir DIR] + +Modes: + --check Read-only orchestration and evidence collection. + --apply Run controlled apply hooks that are already allowlisted. + +Environment: + HOSTS Space-separated host list, default: 99 110 111 112 120 121 188 + PUBLIC_ROUTES Space-separated route readback list. + SSH_TIMEOUT_SECONDS SSH connect/command timeout, default: 12. + RECOVERY_SLO_SECONDS Full recovery target, default: 600. + TELEGRAM_NOTIFY_SCRIPT Optional no-secret wrapper script for Telegram notification. + DISABLE_TELEGRAM=1 Suppress Telegram wrapper even when configured. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --check) + MODE="check" + shift + ;; + --apply) + MODE="apply" + shift + ;; + --artifact-dir) + ARTIFACT_DIR="$2" + ARTIFACT_ROOT="$(dirname "$2")" + RUN_ID="$(basename "$2")" + shift 2 + ;; + --help|-h) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac +done + +mkdir -p "$ARTIFACT_DIR" +: >"$EVENTS_TSV" +: >"$FINDINGS_TSV" +: >"$ACTIONS_TSV" + +event() { + local level="$1" component="$2" message="$3" + printf '%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$level" "$component" "$message" | tee -a "$EVENTS_TSV" >/dev/null +} + +finding() { + local severity="$1" component="$2" code="$3" message="$4" + printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$severity" "$component" "$code" "$message" | tee -a "$FINDINGS_TSV" >/dev/null +} + +action() { + local mode="$1" component="$2" code="$3" message="$4" + printf '%s\t%s\t%s\t%s\t%s\n' "$(date -u +%Y-%m-%dT%H:%M:%SZ)" "$mode" "$component" "$code" "$message" | tee -a "$ACTIONS_TSV" >/dev/null +} + +notify() { + local phase="$1" status="$2" message="$3" + if [[ "$DISABLE_TELEGRAM" == "1" || -z "$TELEGRAM_NOTIFY_SCRIPT" ]]; then + event "INFO" "telegram" "notify skipped phase=${phase} status=${status}: ${message}" + return 0 + fi + if [[ ! -x "$TELEGRAM_NOTIFY_SCRIPT" ]]; then + finding "P1" "telegram" "TELEGRAM_WRAPPER_NOT_EXECUTABLE" "$TELEGRAM_NOTIFY_SCRIPT" + return 0 + fi + "$TELEGRAM_NOTIFY_SCRIPT" --phase "$phase" --status "$status" --artifact-dir "$ARTIFACT_DIR" --message "$message" || { + finding "P1" "telegram" "TELEGRAM_NOTIFY_FAILED" "phase=${phase} status=${status}" + } +} + +host_ip() { + case "$1" in + 99) printf '192.168.0.99' ;; + 110) printf '192.168.0.110' ;; + 111) printf '192.168.0.111' ;; + 112) printf '192.168.0.112' ;; + 120) printf '192.168.0.120' ;; + 121) printf '192.168.0.121' ;; + 188) printf '192.168.0.188' ;; + *) printf '%s' "$1" ;; + esac +} + +ssh_user() { + case "$1" in + 188) printf 'ollama' ;; + *) printf 'wooo' ;; + esac +} + +probe_host_tcp() { + local host="$1" ip + ip="$(host_ip "$host")" + if nc -z -w "$SSH_TIMEOUT_SECONDS" "$ip" 22 >/dev/null 2>&1; then + event "INFO" "host-${host}" "ssh tcp open ${ip}:22" + return 0 + fi + finding "P0" "host-${host}" "HOST_SSH_PORT_DOWN" "${ip}:22 not reachable" + return 1 +} + +probe_host_command() { + local host="$1" ip user output_file + ip="$(host_ip "$host")" + user="$(ssh_user "$host")" + output_file="${ARTIFACT_DIR}/host-${host}-uptime.txt" + if timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'printf "host=%s\n" "$(hostname)"; uptime -s; uptime' >"$output_file" 2>&1; then + event "INFO" "host-${host}" "ssh command ok ${user}@${ip}" + return 0 + fi + if grep -Eqi 'Permission denied|publickey|authentication' "$output_file"; then + finding "P0" "host-${host}" "SSH_AUTH_FAILED" "${user}@${ip}" + elif grep -Eqi 'timed out|Timeout|not responding' "$output_file"; then + finding "P0" "host-${host}" "SSH_COMMAND_SESSION_TIMEOUT" "${user}@${ip}" + else + finding "P0" "host-${host}" "SSH_COMMAND_FAILED" "${user}@${ip}; see ${output_file}" + fi + return 1 +} + +detect_recent_reboot() { + local host="$1" ip user output_file boot_epoch now_epoch age + ip="$(host_ip "$host")" + user="$(ssh_user "$host")" + output_file="${ARTIFACT_DIR}/host-${host}-boot.txt" + if ! timeout "$((SSH_TIMEOUT_SECONDS + 5))" ssh -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ConnectTimeout="$SSH_TIMEOUT_SECONDS" "${user}@${ip}" 'date +%s; cat /proc/stat | awk "/btime/ {print \$2}"' >"$output_file" 2>&1; then + event "WARN" "host-${host}" "recent reboot detection unavailable" + return 0 + fi + now_epoch="$(sed -n '1p' "$output_file")" + boot_epoch="$(sed -n '2p' "$output_file")" + if [[ "$now_epoch" =~ ^[0-9]+$ && "$boot_epoch" =~ ^[0-9]+$ ]]; then + age=$((now_epoch - boot_epoch)) + printf '%s\t%s\t%s\n' "$host" "$boot_epoch" "$age" >>"${ARTIFACT_DIR}/boot-age.tsv" + if (( age <= RECOVERY_SLO_SECONDS )); then + event "INFO" "host-${host}" "recent reboot detected age_seconds=${age}" + else + event "INFO" "host-${host}" "host already past reboot window age_seconds=${age}" + fi + fi +} + +probe_public_routes() { + local route status output_file safe_name + for route in $PUBLIC_ROUTES; do + safe_name="$(printf '%s' "$route" | sed 's#[^A-Za-z0-9._-]#_#g')" + output_file="${ARTIFACT_DIR}/route-${safe_name}.txt" + status="$(curl -k -L --max-time 12 -o "$output_file" -w '%{http_code}' "$route" 2>"${output_file}.err" || true)" + printf '%s\t%s\n' "$route" "$status" >>"${ARTIFACT_DIR}/public-routes.tsv" + case "$status" in + 200|204|301|302|307|308|401) + event "INFO" "route" "${route} status=${status}" + ;; + 502|503|504) + finding "P0" "route" "PUBLIC_ROUTE_UPSTREAM_FAILED" "${route} status=${status}" + ;; + 000) + finding "P0" "route" "PUBLIC_ROUTE_UNREACHABLE" "${route} status=${status}" + ;; + *) + finding "P1" "route" "PUBLIC_ROUTE_UNEXPECTED_STATUS" "${route} status=${status}" + ;; + esac + done +} + +run_ai_log_triage() { + local triage_dir="${ARTIFACT_DIR}/ai-log-triage" + if [[ ! -x "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" ]]; then + finding "P0" "recovery" "AI_LOG_TRIAGE_SCRIPT_MISSING" "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" + return 1 + fi + action "$MODE" "ai-log-triage" "RUN_AI_LOG_TRIAGE" "artifact_dir=${triage_dir}" + "${SCRIPT_DIR}/ai-log-triage-auto-recovery.sh" "--${MODE}" --artifact-dir "$triage_dir" || { + finding "P0" "ai-log-triage" "AI_LOG_TRIAGE_FAILED" "see ${triage_dir}" + return 1 + } + if [[ -s "${triage_dir}/findings.tsv" ]]; then + awk -v now="$(date -u +%Y-%m-%dT%H:%M:%SZ)" -F '\t' \ + '{print now "\t" $2 "\tai-log-triage\t" $1 "\thost=" $3 " evidence=" $4 " next_action=" $5}' \ + "${triage_dir}/findings.tsv" >>"$FINDINGS_TSV" || true + fi +} + +run_scorecard() { + local scorecard_file="${ARTIFACT_DIR}/scorecard.txt" + if [[ ! -f "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" ]]; then + finding "P0" "scorecard" "SCORECARD_SCRIPT_MISSING" "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" + return 1 + fi + action "check" "scorecard" "RUN_REBOOT_SCORECARD" "$scorecard_file" + python3 "${SCRIPT_DIR}/reboot-auto-recovery-slo-scorecard.py" >"$scorecard_file" 2>&1 || { + finding "P0" "scorecard" "SCORECARD_FAILED" "see ${scorecard_file}" + return 1 + } +} + +run_stock_upstream_gate_hint() { + local stock_gate="/Users/ogt/codex-workspaces/gitea-restored-worktrees-20260701/stockplatform-v2/scripts/ops/stockplatform-api-upstream-recovery-gate.sh" + if [[ ! -x "$stock_gate" ]]; then + event "WARN" "stockplatform" "stock upstream recovery gate not executable or missing: ${stock_gate}" + return 0 + fi + action "check" "stockplatform" "VERIFY_STOCK_API_UPSTREAM_GATE" "$stock_gate" + STOCKPLATFORM_DIRECT_EDGE_URL="${STOCKPLATFORM_DIRECT_EDGE_URL:-http://127.0.0.1:31235}" \ + STOCKPLATFORM_PUBLIC_WEB_URL="${STOCKPLATFORM_PUBLIC_WEB_URL:-https://stock.wooo.work}" \ + "$stock_gate" --verify-only >"${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt" 2>&1 || { + finding "P0" "stockplatform" "STOCK_API_UPSTREAM_GATE_BLOCKED" "see ${ARTIFACT_DIR}/stockplatform-api-upstream-gate.txt" + } +} + +write_summary() { + local p0_count p1_count elapsed status + elapsed=$(( $(date +%s) - START_EPOCH )) + p0_count="$(awk -F '\t' '$2=="P0"{c++} END{print c+0}' "$FINDINGS_TSV")" + p1_count="$(awk -F '\t' '$2=="P1"{c++} END{print c+0}' "$FINDINGS_TSV")" + if (( p0_count == 0 )); then + status="RECOVERY_READY_FOR_RUNTIME_CONFIRMATION" + else + status="RECOVERY_BLOCKED_P0" + fi + { + printf 'run_id=%s\n' "$RUN_ID" + printf 'mode=%s\n' "$MODE" + printf 'status=%s\n' "$status" + printf 'elapsed_seconds=%s\n' "$elapsed" + printf 'slo_seconds=%s\n' "$RECOVERY_SLO_SECONDS" + printf 'p0_count=%s\n' "$p0_count" + printf 'p1_count=%s\n' "$p1_count" + printf 'artifact_dir=%s\n' "$ARTIFACT_DIR" + printf '\n[P0 findings]\n' + awk -F '\t' '$2=="P0"{print}' "$FINDINGS_TSV" || true + printf '\n[Actions]\n' + cat "$ACTIONS_TSV" + } >"$SUMMARY_TXT" + cat "$SUMMARY_TXT" +} + +START_EPOCH="$(date +%s)" + +event "INFO" "orchestrator" "start mode=${MODE} run_id=${RUN_ID}" +notify "start" "running" "full-host reboot recovery started mode=${MODE}" + +for host in $HOSTS; do + probe_host_tcp "$host" || true +done + +for host in $HOSTS; do + probe_host_command "$host" || true + detect_recent_reboot "$host" || true +done + +probe_public_routes +run_ai_log_triage || true +run_scorecard || true +run_stock_upstream_gate_hint || true + +write_summary + +if awk -F '\t' '$2=="P0"{found=1} END{exit found ? 0 : 1}' "$FINDINGS_TSV"; then + notify "finish" "blocked" "full-host reboot recovery has P0 blockers; artifact=${ARTIFACT_DIR}" + exit 1 +fi + +notify "finish" "ok" "full-host reboot recovery has no P0 blockers; artifact=${ARTIFACT_DIR}" +exit 0 diff --git a/scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh b/scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh new file mode 100755 index 000000000..0d00a1e10 --- /dev/null +++ b/scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)" +UNIT_NAME="awoooi-full-host-reboot-orchestrator" +SYSTEMD_DIR="${SYSTEMD_DIR:-/etc/systemd/system}" +ENV_DIR="${ENV_DIR:-/etc/awoooi}" +LOG_DIR="${LOG_DIR:-/var/log/awoooi/reboot-recovery}" +INSTALL_ROOT="${INSTALL_ROOT:-/opt/awoooi/reboot-recovery}" +MODE="dry-run" +ENABLE_NOW=0 + +usage() { + cat <<'USAGE' +Usage: install-full-host-reboot-orchestrator.sh [--dry-run|--apply|--verify-only|--rollback] [--enable-now] + +Installs the AWOOOI full-host reboot recovery orchestrator source and systemd +units. The default is --dry-run. Runtime scheduling requires both --apply and +--enable-now so a source deploy cannot accidentally start host automation. + +Options: + --dry-run Print the install plan only. This is the default. + --apply Stage source files and systemd units, then daemon-reload. + --verify-only Read back systemd timer/service/source status without writing. + --rollback Disable and remove only this orchestrator unit/timer. + --enable-now With --apply, enable and start only the orchestrator timer. + -h, --help Show this help. + +Forbidden here: reboot, node drain, firewall changes, DB restore/drop/truncate, +Docker prune, secret/session/.env/raw-volume reads, repo/ref destructive ops. +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + --dry-run) + MODE="dry-run" + ;; + --apply) + MODE="apply" + ;; + --verify-only) + MODE="verify" + ;; + --rollback) + MODE="rollback" + ;; + --enable-now) + ENABLE_NOW=1 + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + usage >&2 + exit 2 + ;; + esac + shift +done + +cat </dev/null || true) +FULL_HOST_ORCHESTRATOR_TIMER_ACTIVE=$(systemctl is-active "${UNIT_NAME}.timer" 2>/dev/null || true) +FULL_HOST_ORCHESTRATOR_SERVICE_LOAD=$(systemctl show "${UNIT_NAME}.service" -p LoadState --value 2>/dev/null || true) +FULL_HOST_ORCHESTRATOR_SERVICE_RESULT=$(systemctl show "${UNIT_NAME}.service" -p Result --value 2>/dev/null || true) +FULL_HOST_ORCHESTRATOR_SERVICE_LAST_STATUS=$(systemctl show "${UNIT_NAME}.service" -p ExecMainStatus --value 2>/dev/null || true) +FULL_HOST_ORCHESTRATOR_INSTALL_ROOT_PRESENT=$([ -x "${INSTALL_ROOT}/scripts/reboot-recovery/full-host-reboot-orchestrator.sh" ] && echo 1 || echo 0) +FULL_HOST_ORCHESTRATOR_AI_TRIAGE_PRESENT=$([ -x "${INSTALL_ROOT}/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh" ] && echo 1 || echo 0) +FULL_HOST_ORCHESTRATOR_LATEST_ARTIFACT_DIR=$(find "$LOG_DIR" -maxdepth 1 -type d -name '20*' 2>/dev/null | sort | tail -n 1 || true) +STATUS +} + +if [[ "$MODE" != "apply" ]]; then + case "$MODE" in + dry-run) + echo "DRY_RUN=1" + exit 0 + ;; + verify) + verify_status + exit 0 + ;; + rollback) + systemctl disable --now "${UNIT_NAME}.timer" >/dev/null 2>&1 || true + systemctl stop "${UNIT_NAME}.service" >/dev/null 2>&1 || true + rm -f "${SYSTEMD_DIR}/${UNIT_NAME}.timer" "${SYSTEMD_DIR}/${UNIT_NAME}.service" + systemctl daemon-reload + echo "FULL_HOST_ORCHESTRATOR_ROLLBACK_DONE=1" + verify_status + exit 0 + ;; + *) + printf 'Unsupported mode: %s\n' "$MODE" >&2 + exit 2 + ;; + esac +fi + +install -d "$SYSTEMD_DIR" "$ENV_DIR" "$LOG_DIR" "$INSTALL_ROOT" +rsync -a \ + --exclude '.git' \ + --exclude '__pycache__' \ + --exclude '.pytest_cache' \ + "${ROOT_DIR}/" "${INSTALL_ROOT}/" +install -m 0644 "${SCRIPT_DIR}/${UNIT_NAME}.service" "${SYSTEMD_DIR}/${UNIT_NAME}.service" +install -m 0644 "${SCRIPT_DIR}/${UNIT_NAME}.timer" "${SYSTEMD_DIR}/${UNIT_NAME}.timer" + +if [[ ! -f "${ENV_DIR}/reboot-recovery.env" ]]; then + install -m 0600 /dev/null "${ENV_DIR}/reboot-recovery.env" + cat >"${ENV_DIR}/reboot-recovery.env" <<'ENV' +# Optional no-secret runtime knobs for full-host reboot recovery. +# Do not place raw tokens in source control. Put host-local secrets only in this root-owned file. +HOSTS="99 110 111 112 120 121 188" +RECOVERY_SLO_SECONDS=600 +SSH_TIMEOUT_SECONDS=12 +DISABLE_TELEGRAM=1 +# TELEGRAM_NOTIFY_SCRIPT=/usr/local/sbin/awoooi-telegram-notify +ENV +fi + +systemctl daemon-reload +if [[ "$ENABLE_NOW" -eq 1 ]]; then + systemctl enable --now "${UNIT_NAME}.timer" +else + echo "ENABLE_NOW_SKIPPED=1" +fi +verify_status diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 31eed664b..2e37d60c3 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -1,8 +1,9 @@ #!/usr/bin/env bash # Boot-triggerable 10-minute reboot recovery SLO exporter. # -# It is read-only: it probes boot state, runs the existing post-reboot summary, -# evaluates the SLO scorecard, and writes node-exporter textfile metrics. +# It first runs the controlled AI log triage auto-recovery runner, then probes +# boot state, runs the post-reboot summary, evaluates the SLO scorecard, and +# writes node-exporter textfile metrics. set -uo pipefail @@ -22,6 +23,9 @@ PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}" WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-111 188 120 121 112}" POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS="${POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS:-900}" +AI_LOG_TRIAGE_AUTO_RECOVERY_MODE="${AI_LOG_TRIAGE_AUTO_RECOVERY_MODE:-check}" +AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-420}" +AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS:-20}" mkdir -p "$TEXTFILE_DIR" "$LOG_DIR" @@ -46,6 +50,7 @@ public_maintenance_prom="$artifact_dir/public-maintenance-fallback.prom" public_maintenance_edge_file="$artifact_dir/public-maintenance-edge-fallback.txt" windows99_management_file="$artifact_dir/windows99-management-channel.json" windows99_vmware_file="$artifact_dir/windows99-vmware-verify.txt" +ai_log_triage_file="$artifact_dir/ai-log-triage-auto-recovery.txt" reboot_event_state_file="${REBOOT_EVENT_STATE_FILE:-${LOG_DIR}/reboot-event-state.json}" extract_summary_number() { @@ -151,6 +156,36 @@ NEXT_REQUIRED_GATES=post_reboot_readiness_full_summary_timeout SUMMARY } +case "$AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" in + check|apply) + ai_log_triage_args=( + bash "$ROOT_DIR/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh" + "--$AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" + --artifact-dir "$artifact_dir/ai-log-triage" + ) + if command -v timeout >/dev/null 2>&1; then + timeout "${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS}s" "${ai_log_triage_args[@]}" \ + >"$ai_log_triage_file" 2>&1 || true + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout "${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS}s" "${ai_log_triage_args[@]}" \ + >"$ai_log_triage_file" 2>&1 || true + else + "${ai_log_triage_args[@]}" >"$ai_log_triage_file" 2>&1 || true + fi + if [ "$AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" = "apply" ] \ + && [[ "$AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS" =~ ^[0-9]+$ ]] \ + && [ "$AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS" -gt 0 ]; then + sleep "$AI_LOG_TRIAGE_AUTO_RECOVERY_POST_APPLY_SETTLE_SECONDS" + fi + ;; + off|disabled|0) + printf 'AI_LOG_TRIAGE_AUTO_RECOVERY_SKIPPED=1 mode=%s\n' "$AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" >"$ai_log_triage_file" + ;; + *) + printf 'AI_LOG_TRIAGE_AUTO_RECOVERY_INVALID_MODE=%s\n' "$AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" >"$ai_log_triage_file" + ;; +esac + bash "$ROOT_DIR/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh" >"$host_probe" 2>&1 || true python3 "$ROOT_DIR/scripts/reboot-recovery/reboot-event-detector.py" \ --host-probe-file "$host_probe" \ diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index ffb5db885..6f8f1b049 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -741,10 +741,64 @@ def source_controls() -> dict[str, bool]: "slo_exporter_source_present": source_file( "scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh" ).exists(), + "ai_log_triage_auto_recovery_source_present": file_contains( + source_file("scripts/reboot-recovery/ai-log-triage-auto-recovery.sh"), + "AWOOOI_AI_LOG_TRIAGE_AUTO_RECOVERY=1", + "SSH_COMMAND_SESSION_TIMEOUT", + "HARBOR_REGISTRY_NOT_READY", + "K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY", + "EDGE_502_UPSTREAM_REFUSED", + "Forbidden even in --apply", + ), + "slo_exporter_runs_ai_log_triage_source_present": file_contains( + source_file("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh"), + "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE", + "ai-log-triage-auto-recovery.sh", + ), "slo_systemd_service_source_present": file_contains( source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service"), "ExecStart=/usr/local/bin/awoooi-reboot-auto-recovery-slo.sh", ), + "slo_systemd_service_ai_log_triage_apply_source_present": file_contains( + source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service"), + "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply", + "AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS=420", + ), + "full_host_reboot_orchestrator_source_present": file_contains( + source_file("scripts/reboot-recovery/full-host-reboot-orchestrator.sh"), + "run_ai_log_triage", + "run_scorecard", + "run_stock_upstream_gate_hint", + "TELEGRAM_NOTIFY_SCRIPT", + ), + "full_host_reboot_orchestrator_systemd_source_present": file_contains( + source_file( + "scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service" + ), + "WorkingDirectory=/opt/awoooi/reboot-recovery", + ( + "ExecStart=/usr/bin/env bash /opt/awoooi/reboot-recovery/scripts/" + "reboot-recovery/full-host-reboot-orchestrator.sh --apply" + ), + ), + "full_host_reboot_orchestrator_timer_source_present": file_contains( + source_file( + "scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.timer" + ), + "OnBootSec=45s", + "OnUnitActiveSec=60s", + "Unit=awoooi-full-host-reboot-orchestrator.service", + ), + "full_host_reboot_orchestrator_installer_source_present": file_contains( + source_file( + "scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh" + ), + "--dry-run", + "--verify-only", + "--rollback", + 'UNIT_NAME="awoooi-full-host-reboot-orchestrator"', + 'systemctl enable --now "${UNIT_NAME}.timer"', + ), "slo_systemd_timer_source_present": file_contains( source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer"), "OnBootSec=", diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 6411163be..0f1337dc4 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -140,6 +140,86 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert "$labels.blocker" in alerts +def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None: + runner = read("scripts/reboot-recovery/ai-log-triage-auto-recovery.sh") + exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") + service = read("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.service") + full_host_orchestrator = read( + "scripts/reboot-recovery/full-host-reboot-orchestrator.sh" + ) + full_host_service = read( + "scripts/reboot-recovery/awoooi-full-host-reboot-orchestrator.service" + ) + full_host_installer = read( + "scripts/reboot-recovery/install-full-host-reboot-orchestrator.sh" + ) + vmware_autostart = read( + "scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1" + ) + windows_update = read( + "scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1" + ) + + assert "MODE=\"check\"" in runner + assert "--apply" in runner + assert "TOO_MANY_OPEN_FILES" in runner + assert "SSH_COMMAND_SESSION_TIMEOUT" in runner + assert "SSH_AUTH_FAILED" in runner + assert "DOCKER_SCOPE_TIMEOUT" in runner + assert "CONTAINERD_TASK_CLEANUP_STUCK" in runner + assert "HARBOR_REGISTRY_NOT_READY" in runner + assert "registry_public_ready" in runner + assert "registry_110_readback_missing" in runner + assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}"' in runner + assert "WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" in runner + assert "https://registry\\.wooo\\.work/v2/" in runner + assert "STOCK_UPSTREAM_NOT_READY" in runner + assert "https://stock\\.wooo\\.work/api/v1/system/freshness" in runner + assert "K3S_IMAGE_PULL_BLOCKED_BY_REGISTRY" in runner + assert "^awoooi-[^[:space:]]+" in runner + assert "EDGE_502_UPSTREAM_REFUSED" in runner + assert "WINDOWS99_KALI_LOCK_IN_USE" in runner + assert "harbor-watchdog.sh --repair-once" in runner + assert "delete pod --wait=false" in runner + assert "systemctl start" in runner + assert "Forbidden even in --apply" in runner + assert "node drain" in runner + assert "Docker prune" in runner + + assert "ai-log-triage-auto-recovery.sh" in exporter + assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE" in exporter + assert "AI_LOG_TRIAGE_AUTO_RECOVERY_MODE=apply" in service + + assert "run_ai_log_triage" in full_host_orchestrator + assert "run_scorecard" in full_host_orchestrator + assert "run_stock_upstream_gate_hint" in full_host_orchestrator + assert "TELEGRAM_NOTIFY_SCRIPT" in full_host_orchestrator + assert 'host=" $3 " evidence=" $4 " next_action=" $5' in full_host_orchestrator + assert "/Users/ogt/" not in full_host_service + assert "WorkingDirectory=/opt/awoooi/reboot-recovery" in full_host_service + assert ( + "/opt/awoooi/reboot-recovery/scripts/reboot-recovery/" + "full-host-reboot-orchestrator.sh --apply" + in full_host_service + ) + assert 'MODE="dry-run"' in full_host_installer + assert 'UNIT_NAME="awoooi-full-host-reboot-orchestrator"' in full_host_installer + assert "ENABLE_NOW=0" in full_host_installer + assert "--enable-now" in full_host_installer + assert "--verify-only" in full_host_installer + assert "--rollback" in full_host_installer + assert "DRY_RUN=1" in full_host_installer + assert 'systemctl enable --now "${UNIT_NAME}.timer"' in full_host_installer + assert "--delete" not in full_host_installer + assert "awooooi-full-host-reboot-orchestrator" not in full_host_installer + assert "awooi-full-host-reboot-orchestrator" not in full_host_installer + + assert "[switch]$WhatIfOnly" in vmware_autostart + assert "if ($WhatIfOnly)" in vmware_autostart + assert "[switch]$WhatIfOnly" in windows_update + assert "WHATIF windows_update_policy" in windows_update + + def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> None: for path in [ "ops/nginx/awoooi.wooo.work.conf", diff --git a/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 new file mode 100644 index 000000000..aef865ec0 --- /dev/null +++ b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 @@ -0,0 +1,89 @@ +param( + [string[]]$VmNameAllowList = @("111", "188", "120", "121", "112"), + [string[]]$SearchRoots = @("C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"), + [string]$TaskName = "AWOOOI VMware VM AutoStart", + [switch]$WhatIfOnly +) + +$ErrorActionPreference = "Stop" + +function Find-Vmrun { + $candidates = @( + "${env:ProgramFiles(x86)}\VMware\VMware Workstation\vmrun.exe", + "${env:ProgramFiles}\VMware\VMware Workstation\vmrun.exe" + ) + foreach ($candidate in $candidates) { + if (Test-Path $candidate) { return $candidate } + } + throw "vmrun.exe not found. VMware Workstation must be installed before VM autostart can be configured." +} + +function Read-VmxDisplayName { + param([string]$Path) + $line = Select-String -Path $Path -Pattern '^\s*displayName\s*=' -ErrorAction SilentlyContinue | Select-Object -First 1 + if (-not $line) { return [IO.Path]::GetFileNameWithoutExtension($Path) } + return ($line.Line -replace '^\s*displayName\s*=\s*', '' -replace '"', '').Trim() +} + +$vmrun = Find-Vmrun +$vmxFiles = @() +foreach ($root in $SearchRoots) { + if (Test-Path $root) { + $vmxFiles += Get-ChildItem -Path $root -Recurse -Filter "*.vmx" -ErrorAction SilentlyContinue + } +} + +$selected = @() +foreach ($vmx in $vmxFiles) { + $displayName = Read-VmxDisplayName -Path $vmx.FullName + foreach ($allowed in $VmNameAllowList) { + if ($displayName -eq $allowed -or $displayName -like "*$allowed*" -or $vmx.FullName -like "*$allowed*") { + $selected += [PSCustomObject]@{ Name = $displayName; Path = $vmx.FullName } + break + } + } +} + +$selected = $selected | Sort-Object Name, Path -Unique +if ($selected.Count -lt $VmNameAllowList.Count) { + $found = ($selected | ForEach-Object { $_.Name }) -join ", " + throw "Not all required VMX files were found. found=[$found] required=[$($VmNameAllowList -join ', ')]" +} + +Set-Service -Name "VMAuthdService" -StartupType Automatic +Set-Service -Name "VMUSBArbService" -StartupType Automatic -ErrorAction SilentlyContinue +Start-Service -Name "VMAuthdService" + +$runner = "C:\ProgramData\AWOOOI\Start-AwoooIVms.ps1" +New-Item -ItemType Directory -Force -Path (Split-Path $runner) | Out-Null + +$vmLines = $selected | ForEach-Object { '"' + ($_.Path -replace '"', '\"') + '"' } +$script = @" +`$ErrorActionPreference = "Continue" +`$vmrun = "$vmrun" +`$vms = @( + $($vmLines -join ",`n ") +) +Start-Sleep -Seconds 30 +foreach (`$vmx in `$vms) { + & `$vmrun -T ws start `$vmx nogui | Out-File -Append C:\ProgramData\AWOOOI\vmware-autostart.log + Start-Sleep -Seconds 15 +} +"@ + +if ($WhatIfOnly) { + Write-Output "WHATIF task=$TaskName runner=$runner" + $selected | Format-Table -AutoSize + exit 0 +} + +Set-Content -Path $runner -Value $script -Encoding UTF8 + +$action = New-ScheduledTaskAction -Execute "powershell.exe" -Argument "-NoProfile -ExecutionPolicy Bypass -File `"$runner`"" +$trigger = New-ScheduledTaskTrigger -AtStartup +$principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -RunLevel Highest +$settings = New-ScheduledTaskSettingsSet -AllowStartIfOnBatteries -DisallowStartIfOnBatteries:$false -MultipleInstances IgnoreNew -ExecutionTimeLimit (New-TimeSpan -Minutes 10) +Register-ScheduledTask -TaskName $TaskName -Action $action -Trigger $trigger -Principal $principal -Settings $settings -Force | Out-Null + +Write-Output "Configured VMware autostart task: $TaskName" +$selected | Format-Table -AutoSize diff --git a/scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1 b/scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1 new file mode 100644 index 000000000..c9e960295 --- /dev/null +++ b/scripts/reboot-recovery/windows-99/configure-windows-update-no-surprise-reboot.ps1 @@ -0,0 +1,26 @@ +param( + [int]$ActiveHoursStart = 8, + [int]$ActiveHoursEnd = 23, + [switch]$WhatIfOnly +) + +$ErrorActionPreference = "Stop" +$wuPath = "HKLM:\SOFTWARE\Policies\Microsoft\Windows\WindowsUpdate" +$auPath = "$wuPath\AU" + +if ($WhatIfOnly) { + Write-Output "WHATIF windows_update_policy=NoAutoRebootWithLoggedOnUsers" + Write-Output "WHATIF active_hours=${ActiveHoursStart}-${ActiveHoursEnd}" + exit 0 +} + +New-Item -Path $wuPath -Force | Out-Null +New-Item -Path $auPath -Force | Out-Null + +New-ItemProperty -Path $auPath -Name NoAutoRebootWithLoggedOnUsers -Value 1 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPath -Name AUOptions -Value 3 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $wuPath -Name SetActiveHours -Value 1 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $wuPath -Name ActiveHoursStart -Value $ActiveHoursStart -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $wuPath -Name ActiveHoursEnd -Value $ActiveHoursEnd -PropertyType DWord -Force | Out-Null + +Write-Output "Windows Update surprise reboot prevention configured. active_hours=${ActiveHoursStart}-${ActiveHoursEnd}"