From 276410fbb6dbd43a18df238dfdac184dd74d1ef2 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 15:08:27 +0800 Subject: [PATCH] fix(recovery): close cold-start runtime blockers --- agent99-windows-update-policy.ps1 | 1 + .../reboot_auto_recovery_slo_scorecard.py | 4 +- ..._reboot_auto_recovery_slo_scorecard_api.py | 3 +- docs/runbooks/FULL-STACK-COLD-START-SOP.md | 17 +- .../HOST-188-HYGIENE-MAINTENANCE-RUNBOOK.md | 35 ++- docs/runbooks/REBOOT-RECOVERY-SOP.md | 18 +- ops/nginx/awoooi.wooo.work.conf | 14 + .../awoooi-edge-maintenance-helper.sh | 273 ++++++++++++++++++ .../awoooi-edge-maintenance.sudoers | 2 + .../awoooi-host188-hygiene-helper.sh | 220 ++++++++++++++ .../awoooi-host188-hygiene.sudoers | 2 + .../ai-log-triage-auto-recovery.sh | 6 +- .../awoooi-reboot-auto-recovery-slo.timer | 2 +- scripts/reboot-recovery/awoooi-startup.sh | 47 ++- .../bootstrap-edge-maintenance-executor.sh | 178 ++++++++++++ .../bootstrap-host188-hygiene-executor.sh | 171 +++++++++++ .../collect-windows99-vmware-verify.sh | 18 +- .../enforce-110-runner-failclosed.sh | 24 +- .../install-macos111-host-boot-readback.sh | 152 ++++++++++ .../install-reboot-auto-recovery-slo-110.sh | 3 +- .../macos-host-boot-readback.sh | 33 +++ .../post-reboot-readiness-summary.sh | 26 +- .../reboot-recovery/post-start-quick-check.sh | 20 +- .../public-maintenance-edge-fallback-apply.sh | 132 +++++---- .../reboot-auto-recovery-host-probe.sh | 46 ++- .../reboot-auto-recovery-slo-exporter.sh | 9 +- .../reboot-auto-recovery-slo-scorecard.py | 6 +- .../tests/test_188_host_hygiene_checklist.py | 37 +++ .../test_cold_start_monitor_bounded_probes.py | 20 ++ ..._post_reboot_readiness_summary_contract.py | 5 + .../test_post_start_quick_check_contract.py | 3 + ...test_reboot_auto_recovery_slo_installer.py | 18 +- .../test_reboot_p0_operational_contract.py | 60 +++- ...ows99_console_verify_artifact_validator.py | 21 +- .../test_windows99_vmware_verify_collector.py | 6 + ...idate-windows99-console-verify-artifact.py | 2 +- .../windows-99/configure-vmware-autostart.ps1 | 2 +- .../windows99-vmware-autostart.ps1 | 3 +- 38 files changed, 1494 insertions(+), 145 deletions(-) create mode 100755 ops/reboot-recovery/awoooi-edge-maintenance-helper.sh create mode 100644 ops/reboot-recovery/awoooi-edge-maintenance.sudoers create mode 100755 ops/reboot-recovery/awoooi-host188-hygiene-helper.sh create mode 100644 ops/reboot-recovery/awoooi-host188-hygiene.sudoers create mode 100755 scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh create mode 100755 scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh create mode 100755 scripts/reboot-recovery/install-macos111-host-boot-readback.sh create mode 100755 scripts/reboot-recovery/macos-host-boot-readback.sh diff --git a/agent99-windows-update-policy.ps1 b/agent99-windows-update-policy.ps1 index 7f20e6a31..3824143cd 100644 --- a/agent99-windows-update-policy.ps1 +++ b/agent99-windows-update-policy.ps1 @@ -52,6 +52,7 @@ New-ItemProperty -Path $auPolicy -Name "AlwaysAutoRebootAtScheduledTime" -Value New-ItemProperty -Path $auPolicy -Name "AUOptions" -Value 3 -PropertyType DWord -Force | Out-Null New-ItemProperty -Path $auPolicy -Name "ScheduledInstallDay" -Value 0 -PropertyType DWord -Force | Out-Null New-ItemProperty -Path $auPolicy -Name "ScheduledInstallTime" -Value 3 -PropertyType DWord -Force | Out-Null +New-ItemProperty -Path $auPolicy -Name "AUPowerManagement" -Value 0 -PropertyType DWord -Force | Out-Null # Make the interactive active-hours UI reflect an ops-safe window. New-ItemProperty -Path $uxSettings -Name "ActiveHoursStart" -Value 0 -PropertyType DWord -Force | Out-Null diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 40c6fb39a..2a636b14e 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -143,7 +143,9 @@ _WINDOWS99_POWERED_OFF_NEXT_SAFE_ACTION = ( "run_windows99_vmware_autostart_check_mode_package_for_powered_off_aliases_" "then_rerun_no_secret_collector_no_vm_power_change" ) -_WINDOWS99_REQUIRED_VM_ALIASES = ["111", "188", "120", "121", "112"] +# Host 111 remains in whole-host reboot detection but is a physical MacBook, +# not a VMware guest on Windows 99. +_WINDOWS99_REQUIRED_VM_ALIASES = ["110", "188", "120", "121", "112"] _WINDOWS99_EXPECTED_VMX_CANDIDATES = { "110": [ r"D:\Documents\Virtual Machines\Ubuntu 64-bit (3)\Ubuntu 64-bit (3).vmx", diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index d57792c4e..6d56ce6b9 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -1722,7 +1722,8 @@ def _assert_reboot_slo_payload(payload: dict): assert "manual_db_update" in service_data_backup["forbidden_actions"] windows99 = payload["windows99_vmware_autostart"] assert windows99["readback_present"] is False - assert windows99["required_vm_aliases"] == ["111", "188", "120", "121", "112"] + assert windows99["required_vm_aliases"] == ["110", "188", "120", "121", "112"] + assert "111" in payload["host_boot_detection"]["required_hosts"] assert windows99["blockers"] == ["windows99_vmware_autostart_readback_missing"] collection = payload["windows99_verify_collection"] assert collection["schema_version"] == ( diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 7a6b90937..496573d83 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.132 -> Last updated: 2026-07-10 Asia/Taipei +> Version: v1.133 +> Last updated: 2026-07-11 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,19 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.133 2026-07-11 Cold-start runtime convergence + +2026-07-11 14:58 最新完成 artifact 為 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-145616`。`OVERALL_DECLARATION=GREEN_WITH_EVIDENCE_WARNINGS`、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`BACKUP_CORE_GREEN=1`、`POST_START_BLOCKED=0`、`HOST_188_HYGIENE_BLOCKED=0`；scorecard readiness `87%`。active blockers 只剩 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`，三者都只能由下一次真實全主機 reboot drill 在同一 10 分鐘 window 驗收；不得靠重跑 verifier 或為了清分數擅自 reboot。 + +- 110 CPU incident 根因是 runner fail-closed cron 每分鐘無 `flock` 重複 `--apply`，多個 enforcer 重疊 mask runner 與 `systemctl daemon-reload`，使 PID 1 長時間高 CPU。現行合約為 `flock` + `timeout 180` + `--ensure`，只有 drift 才 apply；post-verifier 為 `APPLY_PERFORMED=0`、`RECENT_APPLY_COUNT=0`、`SYSTEMD_RELOADS_SINCE_APPLY=0`、PID 1 CPU `0%`。runner 仍 fail-closed，token 未讀取或變更。 +- 188 edge Nginx 已透過 root-owned `/usr/local/sbin/awoooi-edge-maintenance` 固定 `check/apply/rollback-latest` executor 完成 backup、`nginx -t`、reload 與 SHA readback。live config / maintenance SHA 分別為 `577d37af17cf096719991d1acfb5e3cfc77ed4a197994817e656f8438d95127b` / `9807d067143527ae2c77e6c09e9d1659134384525e9cb4aa099db0950fcb48b2`，`EDGE_FALLBACK_READY=1`；rollback 在 `/var/lib/awoooi-edge-maintenance/backups/edge-maintenance-20260711T141908+0800-190071-before-apply`。canonical config 必須保留 `/api/sentry-tunnel -> awoooi_web`；public GET 回 `status=ok,target_configured=true`，6/6 public route 2xx、raw 5xx `0`。 +- 111 的 canonical identity 是 M1 Pro MacBook Local Ollama fallback，不是 Windows99 VM。Windows99 五個 required guest 是 `110/188/120/121/112`。110 透過 forced-command public key 只能執行 `/Users/ooo/.local/bin/awoooi-host-boot-readback`，禁止 shell / PTY / forwarding；最新回讀 `boot_id=darwin_1783696973`、LaunchAgent enabled、Ollama active。scorecard 已觀測 99/110/111/112/120/121/188 全部 uptime，`UNKNOWN_UPTIME_HOSTS=[]`。 +- SLO timer 已從 `OnUnitActiveSec=2min` 改為 `OnBootSec=2min` + `OnUnitInactiveSec=5min`，避免 2-3 分鐘 oneshot 因逾期而背靠背執行。 +- 188 PostgreSQL `14/main` 的 production runtime 已證實由 `k3s-postgres-recovery` 以 host network、同一 `/var/lib/postgresql/14/main` 與 `unless-stopped` 提供。host `start.conf` 已改為 `manual`，startup source/unit 已用 fixed-action hygiene executor 對齊並 reset stale failed state，沒有重啟產品。independent checklist 回 `SYSTEMD_STATE=running`、failed units `0`、`HOST_HYGIENE_BLOCKED=0`、public routes 4/4 200；rollback 在 `/var/lib/awoooi-host188-hygiene/backups/host188-hygiene-20260711T145537+0800-334605-before-reconcile`。 +- StockPlatform 2026-07-10 無資料來自 TWSE 官方颱風休市，不是 ingestion 中斷。交易日曆修正的 durable receipt 為 `raw.source_runs.source_run_id=3856`；freshness / ingestion 皆 `status=ok`、latest trading date `2026-07-09`、四類 coverage 100%。 + +source regression：reboot P0 / installer / scorecard / API focused tests `59 passed`，188 hygiene tests `7 passed`，`git diff --check` 通過。目前只是 local feature worktree，依單一 pusher 鎖不 push main；production runtime receipt 不得被未 push source 狀態覆蓋。 + ### v1.132 2026-07-10 備份與 Gitea 可還原性閉環 重啟恢復不得再以「備份檔存在」或「Gitea HTTP 200」宣告備份正常。固定證據鏈為：backup exporter/status freshness -> Gitea full dump isolated restore -> private bundle complete-set sync -> sample clone/fsck -> Prometheus rule/alert readback。任一層缺失都只能標示 degraded，不能標示 restore-ready。 diff --git a/docs/runbooks/HOST-188-HYGIENE-MAINTENANCE-RUNBOOK.md b/docs/runbooks/HOST-188-HYGIENE-MAINTENANCE-RUNBOOK.md index 4966e785c..3ba1e3296 100644 --- a/docs/runbooks/HOST-188-HYGIENE-MAINTENANCE-RUNBOOK.md +++ b/docs/runbooks/HOST-188-HYGIENE-MAINTENANCE-RUNBOOK.md @@ -1,6 +1,6 @@ # 188 Host Hygiene 維護窗口 Runbook -> 版本：2026-06-26.v2 +> 版本：2026-07-11.v3 > 適用範圍：`192.168.0.188` host PostgreSQL `14/main`、`awoooi-startup.service`、certbot / ACME renewal hygiene。 > 狀態：`HOST_188_HYGIENE_GREEN` @@ -16,18 +16,20 @@ ## 2. 最新只讀證據 -2026-06-26 12:02-12:13 Asia/Taipei live evidence： +2026-07-11 14:55 Asia/Taipei live evidence 取代舊 2026-06-26 baseline： | 層級 | 證據 | 判定 | |------|------|------| | Host systemd | `systemctl is-system-running` 為 `running`，failed units `0 loaded units listed` | host hygiene green | | 產品容器 | MOMO、VibeWork、2026FIFA、ClawBot、MinIO、exporters、MOMO DB 等容器 healthy / up | 產品服務 green | | PostgreSQL runtime | `k3s-postgres-recovery` container 為 `Running=true`、`NetworkMode=host`、restart policy `unless-stopped`，`pg_isready -h localhost -p 5432` accepting connections | production DB runtime green | -| Host PostgreSQL unit | `postgresql@14-main.service` reset 後 `Result=success / inactive`；`pg_lsclusters` 仍可能因 recovery container PID model 顯示 host cluster down | 不再作為 service blocker；由 runtime-ready 判斷 | -| Startup unit | `/usr/local/bin/awoooi-startup.sh` 已改為 `postgres_runtime_ready()`，接受 active host unit 或 `k3s-postgres-recovery` host-network runtime；不再自動執行 `pg_resetwal` | fail-closed 修復完成 | +| Host PostgreSQL unit | `/etc/postgresql/14/main/start.conf=manual`，`postgresql@14-main.service` 為 disabled / inactive；recovery container 同一 data/config mount 仍 accepting connections | 避免 host/container 雙啟動競爭 | +| Startup unit | source/live SHA 為 `d9b099382fbc618280241d398a2a3d2c89f004804ab793c177a25a43505ebe8d`，unit SHA 為 `995705578f4ba50f02100f7f60f432fa86d3f37e13aaa6f1d93c1d8685a8cafb`；metadata repair、kine write、ClawBot rebuild、188 SignOz 均預設 opt-in | fail-closed 修復完成 | | Nginx / ACME route | `sentry.wooo.work`、`gitea.wooo.work`、`langfuse.wooo.work`、`signoz.wooo.work` HTTP-01 self-test token 均由 `/var/www/html` 正確回應 | challenge route green | | certbot timers | duplicate apt `certbot.timer` 已停用；snap `snap.certbot.renew.timer` enabled / active / waiting；`certbot.service` 與 `snap.certbot.renew.service` `Result=success / inactive` | renewal runner hygiene green | -| public cert | shared `sentry.wooo.work` certificate valid until `2026-07-09 16:03:40 UTC` | 尚未宣稱已 renew；等待 snap timer / ACME window | +| public cert | `sentry.wooo.work` certificate `notAfter=2026-09-24 13:44:28 UTC`，snap renew timer enabled / waiting | certificate readback green | + +controlled reconcile receipt：`/usr/local/sbin/awoooi-host188-hygiene reconcile` 回 `PRODUCT_SERVICE_RESTARTS=0`，rollback 為 `/var/lib/awoooi-host188-hygiene/backups/host188-hygiene-20260711T145537+0800-334605-before-reconcile`。獨立 checklist 回 `PASS=17 WARN=1 BLOCKED=0`、`HOST_HYGIENE_BLOCKED=0`；warning 只說明 `pg_lsclusters` 看不到 container-owned PID model，不是 DB 不可用。 --- @@ -138,7 +140,16 @@ docker ps --format '{{.Names}}\t{{.Status}}\t{{.Ports}}' ### Phase C：受控修復 -只在 maintenance window 內由 operator 執行；此文件不提供自動 apply。 +先安裝固定動作 executor，再分開執行 check / reconcile： + +```bash +scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh --check +scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh --apply +ssh ollama@192.168.0.188 'sudo -n /usr/local/sbin/awoooi-host188-hygiene check' +ssh ollama@192.168.0.188 'sudo -n /usr/local/sbin/awoooi-host188-hygiene reconcile' +``` + +executor 只能安裝 canonical startup source/unit、將 PG start mode 設為 manual、daemon-reload 與 reset stale failed state；不能 start/restart/stop 產品、Docker、Nginx 或 PostgreSQL。回滾只允許 `sudo -n /usr/local/sbin/awoooi-host188-hygiene rollback-latest`。 ### Phase D：post-check @@ -170,16 +181,16 @@ curl -fsS https://awoooi.wooo.work/api/v1/health | 服務恢復 | `100%` | routes、containers、cold-start、backup core 已 green | | 188 host hygiene repair | `100%` | startup fail-closed、PostgreSQL runtime-ready 判斷、systemd failed units、ACME route 與 certbot timer hygiene 已收斂 | | PostgreSQL runtime source-of-truth | `100%` | production DB runtime 以 `k3s-postgres-recovery` host-network container + `pg_isready` 判斷，不用 host `pg_lsclusters` 假紅 | -| certbot / ACME route hygiene | `95%` | HTTP-01 route 與 timer split 已修；正式 renew 成功仍等待 snap timer / ACME window | -| DR escrow | `BLOCKED` | `escrow_missing=5`，不可用 188 host green 替代 | -| Wazuh registry | `BLOCKED` | manager registry accepted `0`，不可用 route `200` 或 transport count 替代 | +| certbot / ACME route hygiene | `100%` | snap renew timer active，certificate 有效到 2026-09-24；續依正常 timer 監控 | +| DR escrow | `100%` | 2026-07-11 checklist 回 `escrow_missing=0`，但仍與 host hygiene 分開記錄 | +| Wazuh registry | `NOT_EVALUATED_THIS_RUN` | 本輪 188 checklist 未取得 fresh manager registry acceptance，不得沿用舊值作目前結論 | --- ## 8. 不得宣稱 -- 不得宣稱 DR complete；credential escrow evidence 仍缺 `5`。 -- 不得宣稱 Wazuh registry recovered；manager registry accepted 仍為 `0`。 -- 不得宣稱 certbot certificate 已完成正式 renew；目前完成的是 HTTP-01 route / timer hygiene / failed units 清除。 +- 不得用 188 host green 單獨宣稱全平台 DR complete；DR 必須讀同一輪 backup/escrow scorecard。 +- 不得宣稱 Wazuh registry recovered；本輪沒有 fresh manager registry acceptance readback。 +- 不得因 certificate 目前有效就停用 snap renew timer / expiry 告警。 - 不得把 `reset-failed` 單獨當成修復完成；本次完成是因 startup source、Nginx ACME route、timer split 與 postcheck 同時收斂。 - 不得宣稱這份文件批准任何新的 runtime action。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index aeab166b4..d94548763 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.26 -> **最後更新**: 2026-07-10 (台北時間) +> **版本**: v5.27 +> **最後更新**: 2026-07-11 (台北時間) > **更新者**: Codex > **觸發事件**: 2026-07-11 Agent99 多排程 SSH contention、執行身份與 false-down readback 收斂 @@ -21,6 +21,20 @@ ## 架構概覽與依賴圖 +### 2026-07-11 P0 Cold-start 實機收旂 + +最新完成 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-145616` 回報 `GREEN_WITH_EVIDENCE_WARNINGS`、readiness `87%`、service/data/backup/post-start/188 hygiene 皆綠。唯三 blockers 是 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`；它們只能在下一次真實全主機 reboot 後同一 10 分鐘 window 驗收，未取得執行當下的 break-glass 指令前不得為了清分數重啟。 + +- 99 VMware required guest set 是 `110/188/120/121/112`。111 是 M1 Pro MacBook Local Ollama，已用 110 public key + forced command 回傳 Darwin boot/uptime/Ollama status，不授權 shell。舊版把 111 當 Windows99 VMX 的敘述依 live inventory 標記為 superseded。 +- 110 runner fail-closed authority 只能使用 `flock` + `timeout` + `--ensure`，不得每分鐘無條件 mask + daemon-reload。完成條件是 apply/reload count 為 0 且 PID 1 CPU 收旂。 +- edge fallback 只走 `/usr/local/sbin/awoooi-edge-maintenance check|apply|rollback-latest`；canonical/live SHA parity、backup、`nginx -t`、reload、public route verifier 缺一不可完成。`/api/sentry-tunnel` 必須在 generic `/api/` 前且 proxy 到 web。 +- 111 host probe 只讀 `boot_id=darwin_` / uptime / LaunchAgent / local Ollama；`UNKNOWN_UPTIME_HOSTS=[]` 才可進 reboot-event detector。 +- SLO timer 使用 `OnBootSec=2min` + `OnUnitInactiveSec=5min`；禁止恢復 `OnUnitActiveSec=2min`，否則 2-3 分鐘 oneshot 會連續排程並製造 CPU / 告警噪音。 +- 188 production PG 由 `k3s-postgres-recovery` host-network container 管理，host `start.conf=manual`。`awoooi-host188-hygiene reconcile` 只安裝 startup source/unit、daemon-reload、reset stale failed state，禁止產品 restart。startup 內 containerd/Docker metadata 刪除、kine write、ClawBot rebuild、188 legacy SignOz 均預設 fail-closed/opt-in。 +- 2026-07-10 Stock gap 來自 TWSE 颱風休市，不是 ingestion 中斷。交易日曆修正後 freshness/ingestion 皆 green；不得對休市日無限重跑 ETL。 + +下一次 reboot drill 必須在 T+0 由 boot-id 變更自動觸發 Agent99/110 SLO lane，T+10 前產生同一 run 的 VM power、host uptime、services/routes、Stock freshness、backup、Telegram lifecycle 與 KM writeback receipt。若任一主機不在 window，不得宣稱 10 分鐘 SLA 完成。 + ### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋 99 的 Agent99 是全主機恢復控制節點。Windows 開機後由既有 Agent99 scheduled tasks 啟動；`Status` 讀到 boot identity 變更時，只能用 single-flight queue 觸發一次 `Recover`，並以 10 分鐘 cooldown 避免併發恢復。每次執行必須寫入 `bootState`、`recoveryTrigger` 與 `recoverySlo` evidence。只有實際全主機重啟事件、五主機 reachability、必要服務、public routes 與 freshness 全部在同一個 10 分鐘 window 通過，才可宣稱 reboot recovery SLO 達標。 diff --git a/ops/nginx/awoooi.wooo.work.conf b/ops/nginx/awoooi.wooo.work.conf index 4961cbbdc..a41f5d134 100644 --- a/ops/nginx/awoooi.wooo.work.conf +++ b/ops/nginx/awoooi.wooo.work.conf @@ -93,6 +93,20 @@ server { # 避免主機或 K3s rollout 期間把 Nginx raw 5xx 直接給使用者。 proxy_intercept_errors on; + # ========================================================================= + # Next.js API routes + # ========================================================================= + # Keep this before /api/ so browser telemetry reaches the web route handler. + location /api/sentry-tunnel { + proxy_pass http://awoooi_web; + proxy_intercept_errors on; + error_page 502 503 504 =503 /__awoooi-maintenance.html; + proxy_read_timeout 30s; + proxy_send_timeout 30s; + + proxy_set_header Connection "keep-alive"; + } + # ========================================================================= # API 路由 (/api/) # ========================================================================= diff --git a/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh b/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh new file mode 100755 index 000000000..3b0e84e76 --- /dev/null +++ b/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh @@ -0,0 +1,273 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 077 + +ACTION="${1:-check}" +if [ "$#" -ne 1 ]; then + echo "BLOCKER invalid_executor_arguments" + exit 64 +fi + +CANONICAL_DIR="/usr/local/share/awoooi-edge-maintenance" +CANONICAL_CONF="${CANONICAL_DIR}/awoooi.wooo.work.conf" +CANONICAL_MAINT="${CANONICAL_DIR}/maintenance.html" +LIVE_CONF_LINK="/etc/nginx/sites-enabled/awoooi.wooo.work.conf" +EXPECTED_LIVE_CONF="/etc/nginx/sites-available/awoooi.wooo.work.conf" +LIVE_MAINT="/var/www/maintenance/maintenance.html" +STATE_DIR="/var/lib/awoooi-edge-maintenance" +BACKUP_ROOT="${STATE_DIR}/backups" +LAST_BACKUP_FILE="${STATE_DIR}/last-backup" +AUDIT_LOG="/var/log/awoooi-edge-maintenance.log" +LOCK_FILE="/run/lock/awoooi-edge-maintenance.lock" +RUN_ID="edge-maintenance-$(date +%Y%m%dT%H%M%S%z)-$$" + +audit() { + local result="$1" + printf '%s run_id=%s action=%s result=%s\n' \ + "$(date --iso-8601=seconds)" "$RUN_ID" "$ACTION" "$result" >>"$AUDIT_LOG" +} + +block() { + local reason="$1" + echo "EDGE_EXECUTOR_RESULT=blocked" + echo "RUN_ID=$RUN_ID" + echo "BLOCKER $reason" + audit "blocked:${reason}" || true + exit 75 +} + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +resolve_live_conf() { + local resolved + resolved="$(readlink -f "$LIVE_CONF_LINK" 2>/dev/null || true)" + if [ -z "$resolved" ] && [ ! -e "$LIVE_CONF_LINK" ]; then + resolved="$EXPECTED_LIVE_CONF" + fi + case "$resolved" in + "$EXPECTED_LIVE_CONF"|"$LIVE_CONF_LINK") + printf '%s\n' "$resolved" + ;; + *) + block "unexpected_live_config_target" + ;; + esac +} + +assert_root_payload() { + local file owner + for file in "$CANONICAL_CONF" "$CANONICAL_MAINT"; do + [ -f "$file" ] || block "canonical_payload_missing" + owner="$(stat -c '%u' "$file")" + [ "$owner" = "0" ] || block "canonical_payload_not_root_owned" + if find "$file" -prune -perm /022 -print -quit | grep -q .; then + block "canonical_payload_is_mutable" + fi + done + + grep -q 'server_name awoooi.wooo.work;' "$CANONICAL_CONF" \ + || block "canonical_server_name_missing" + grep -q '192.168.0.125:32334' "$CANONICAL_CONF" \ + || block "canonical_api_vip_missing" + grep -q '192.168.0.125:32335' "$CANONICAL_CONF" \ + || block "canonical_web_vip_missing" + grep -q 'location /api/sentry-tunnel {' "$CANONICAL_CONF" \ + || block "canonical_sentry_tunnel_missing" + grep -q 'proxy_intercept_errors on;' "$CANONICAL_CONF" \ + || block "canonical_intercept_missing" + grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$CANONICAL_CONF" \ + || block "canonical_error_page_missing" + grep -q 'X-AWOOOI-Fallback "local-maintenance"' "$CANONICAL_CONF" \ + || block "canonical_fallback_header_missing" + grep -q 'AWOOOI 維護模式' "$CANONICAL_MAINT" \ + || block "canonical_maintenance_marker_missing" + if grep -Ev '^[[:space:]]*#' "$CANONICAL_CONF" \ + | grep -Eq '(^|[^0-9])(31234|31235)([^0-9]|$)'; then + block "canonical_legacy_port_forbidden" + fi + + local tunnel_line api_line + tunnel_line="$(grep -n -m1 'location /api/sentry-tunnel {' "$CANONICAL_CONF" | cut -d: -f1)" + api_line="$(grep -n -m1 'location /api/ {' "$CANONICAL_CONF" | cut -d: -f1)" + [ "$tunnel_line" -lt "$api_line" ] || block "canonical_route_order_invalid" +} + +print_status() { + local live_conf last_backup="" + live_conf="$(resolve_live_conf)" + [ -f "$LAST_BACKUP_FILE" ] && last_backup="$(cat "$LAST_BACKUP_FILE")" + + echo "AWOOOI_EDGE_MAINTENANCE_EXECUTOR=1" + echo "EDGE_EXECUTOR_ACTION=check" + echo "EDGE_EXECUTOR_READY=1" + echo "RUN_ID=$RUN_ID" + echo "CANONICAL_CONF_SHA256=$(sha256_file "$CANONICAL_CONF")" + echo "CANONICAL_MAINT_SHA256=$(sha256_file "$CANONICAL_MAINT")" + echo "LIVE_CONF_TARGET=$live_conf" + if [ -f "$live_conf" ]; then + echo "LIVE_CONF_SHA256=$(sha256_file "$live_conf")" + else + echo "LIVE_CONF_SHA256=" + fi + if [ -f "$LIVE_MAINT" ]; then + echo "LIVE_MAINT_SHA256=$(sha256_file "$LIVE_MAINT")" + else + echo "LIVE_MAINT_SHA256=" + fi + echo "LAST_BACKUP_DIR=$last_backup" +} + +create_backup() { + local label="$1" live_conf backup_dir + live_conf="$(resolve_live_conf)" + backup_dir="${BACKUP_ROOT}/${RUN_ID}-${label}" + install -d -m 0700 "$backup_dir" + if [ -f "$live_conf" ]; then + cp -a "$live_conf" "$backup_dir/live.conf" + else + : >"$backup_dir/live.conf.missing" + fi + if [ -f "$LIVE_MAINT" ]; then + cp -a "$LIVE_MAINT" "$backup_dir/maintenance.html" + else + : >"$backup_dir/maintenance.html.missing" + fi + printf '%s\n' "$live_conf" >"$backup_dir/live-conf-target" + printf '%s\n' "$backup_dir" +} + +restore_backup() { + local backup_dir="$1" live_conf + case "$backup_dir" in + "$BACKUP_ROOT"/*) ;; + *) block "rollback_path_outside_backup_root" ;; + esac + [ -d "$backup_dir" ] || block "rollback_backup_missing" + live_conf="$(resolve_live_conf)" + + if [ -f "$backup_dir/live.conf" ]; then + install -m 0644 "$backup_dir/live.conf" "$live_conf" + elif [ -f "$backup_dir/live.conf.missing" ]; then + rm -f "$live_conf" + else + block "rollback_live_config_receipt_missing" + fi + + if [ -f "$backup_dir/maintenance.html" ]; then + install -d -m 0755 "$(dirname "$LIVE_MAINT")" + install -m 0644 "$backup_dir/maintenance.html" "$LIVE_MAINT" + elif [ -f "$backup_dir/maintenance.html.missing" ]; then + rm -f "$LIVE_MAINT" + else + block "rollback_maintenance_receipt_missing" + fi +} + +write_last_backup() { + local backup_dir="$1" tmp + tmp="${LAST_BACKUP_FILE}.tmp.$$" + printf '%s\n' "$backup_dir" >"$tmp" + install -m 0600 "$tmp" "$LAST_BACKUP_FILE" + rm -f "$tmp" +} + +apply_payload() { + local live_conf backup_dir + live_conf="$(resolve_live_conf)" + nginx -t + backup_dir="$(create_backup before-apply)" + + install -d -m 0755 "$(dirname "$live_conf")" "$(dirname "$LIVE_MAINT")" + install -m 0644 "$CANONICAL_CONF" "$live_conf" + install -m 0644 "$CANONICAL_MAINT" "$LIVE_MAINT" + + if ! nginx -t; then + restore_backup "$backup_dir" + nginx -t || true + echo "BACKUP_DIR=$backup_dir" + block "nginx_test_failed_rollback_restored" + fi + if ! systemctl reload nginx; then + restore_backup "$backup_dir" + nginx -t || true + systemctl reload nginx || true + echo "BACKUP_DIR=$backup_dir" + block "nginx_reload_failed_rollback_restored" + fi + if ! systemctl is-active --quiet nginx; then + restore_backup "$backup_dir" + nginx -t || true + systemctl reload nginx || true + echo "BACKUP_DIR=$backup_dir" + block "nginx_inactive_after_reload_rollback_restored" + fi + + write_last_backup "$backup_dir" + audit "applied" + echo "EDGE_EXECUTOR_RESULT=applied" + echo "RUN_ID=$RUN_ID" + echo "BACKUP_DIR=$backup_dir" + echo "BACKUP_CONF=$backup_dir/live.conf" + echo "NGINX_TEST_OK=1" + echo "NGINX_RELOAD_DONE=1" +} + +rollback_latest() { + local rollback_dir safety_backup + [ -f "$LAST_BACKUP_FILE" ] || block "last_backup_receipt_missing" + rollback_dir="$(cat "$LAST_BACKUP_FILE")" + case "$rollback_dir" in + "$BACKUP_ROOT"/*) ;; + *) block "last_backup_path_invalid" ;; + esac + safety_backup="$(create_backup before-rollback)" + restore_backup "$rollback_dir" + if ! nginx -t || ! systemctl reload nginx || ! systemctl is-active --quiet nginx; then + restore_backup "$safety_backup" + nginx -t || true + systemctl reload nginx || true + echo "SAFETY_BACKUP_DIR=$safety_backup" + block "rollback_verifier_failed_original_restored" + fi + write_last_backup "$safety_backup" + audit "rolled_back" + echo "EDGE_EXECUTOR_RESULT=rolled_back" + echo "RUN_ID=$RUN_ID" + echo "RESTORED_BACKUP_DIR=$rollback_dir" + echo "SAFETY_BACKUP_DIR=$safety_backup" + echo "NGINX_TEST_OK=1" + echo "NGINX_RELOAD_DONE=1" +} + +if [ "${EUID:-$(id -u)}" -ne 0 ]; then + echo "BLOCKER executor_requires_root" + exit 77 +fi + +install -d -m 0700 "$STATE_DIR" "$BACKUP_ROOT" +touch "$AUDIT_LOG" +chmod 0600 "$AUDIT_LOG" +assert_root_payload + +exec 9>"$LOCK_FILE" +flock -w 30 9 || block "executor_lock_timeout" + +case "$ACTION" in + check) + print_status + ;; + apply) + apply_payload + print_status + ;; + rollback-latest) + rollback_latest + print_status + ;; + *) + block "unsupported_executor_action" + ;; +esac diff --git a/ops/reboot-recovery/awoooi-edge-maintenance.sudoers b/ops/reboot-recovery/awoooi-edge-maintenance.sudoers new file mode 100644 index 000000000..14d78c8f0 --- /dev/null +++ b/ops/reboot-recovery/awoooi-edge-maintenance.sudoers @@ -0,0 +1,2 @@ +Cmnd_Alias AWOOOI_EDGE_MAINTENANCE = /usr/local/sbin/awoooi-edge-maintenance check, /usr/local/sbin/awoooi-edge-maintenance apply, /usr/local/sbin/awoooi-edge-maintenance rollback-latest +ollama ALL=(root) NOPASSWD: AWOOOI_EDGE_MAINTENANCE diff --git a/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh b/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh new file mode 100755 index 000000000..1d7a305f2 --- /dev/null +++ b/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh @@ -0,0 +1,220 @@ +#!/usr/bin/env bash +set -euo pipefail + +umask 077 + +ACTION="${1:-check}" +if [ "$#" -ne 1 ]; then + echo "BLOCKER invalid_hygiene_executor_arguments" + exit 64 +fi + +CANONICAL_DIR="/usr/local/share/awoooi-host188-hygiene" +CANONICAL_SCRIPT="${CANONICAL_DIR}/awoooi-startup.sh" +CANONICAL_UNIT="${CANONICAL_DIR}/awoooi-startup.service" +LIVE_SCRIPT="/usr/local/bin/awoooi-startup.sh" +LIVE_UNIT="/etc/systemd/system/awoooi-startup.service" +POSTGRES_START_CONF="/etc/postgresql/14/main/start.conf" +STATE_DIR="/var/lib/awoooi-host188-hygiene" +BACKUP_ROOT="${STATE_DIR}/backups" +LAST_BACKUP_FILE="${STATE_DIR}/last-backup" +AUDIT_LOG="/var/log/awoooi-host188-hygiene.log" +LOCK_FILE="/run/lock/awoooi-host188-hygiene.lock" +RUN_ID="host188-hygiene-$(date +%Y%m%dT%H%M%S%z)-$$" + +audit() { + printf '%s run_id=%s action=%s result=%s\n' \ + "$(date --iso-8601=seconds)" "$RUN_ID" "$ACTION" "$1" >>"$AUDIT_LOG" +} + +block() { + echo "HOST188_HYGIENE_EXECUTOR_RESULT=blocked" + echo "RUN_ID=$RUN_ID" + echo "BLOCKER $1" + audit "blocked:$1" || true + exit 75 +} + +sha256_file() { + sha256sum "$1" | awk '{print $1}' +} + +postgres_start_mode() { + awk '!/^[[:space:]]*#/ && NF {print $1; exit}' "$POSTGRES_START_CONF" 2>/dev/null || true +} + +assert_canonical_payload() { + local file + for file in "$CANONICAL_SCRIPT" "$CANONICAL_UNIT"; do + [ -f "$file" ] || block "canonical_hygiene_payload_missing" + [ "$(stat -c '%u' "$file")" = "0" ] || block "canonical_hygiene_payload_not_root_owned" + if find "$file" -prune -perm /022 -print -quit | grep -q .; then + block "canonical_hygiene_payload_is_mutable" + fi + done + grep -q 'CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \ + || block "canonical_clawbot_rebuild_guard_missing" + grep -q 'CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \ + || block "canonical_containerd_repair_guard_missing" + grep -q 'DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \ + || block "canonical_docker_repair_guard_missing" + grep -q 'KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \ + || block "canonical_kine_write_guard_missing" + grep -q 'SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"' "$CANONICAL_SCRIPT" \ + || block "canonical_signoz_owner_guard_missing" + grep -q '^TimeoutStartSec=600$' "$CANONICAL_UNIT" \ + || block "canonical_startup_timeout_invalid" +} + +assert_recovery_runtime() { + local state mount_ok=0 + state="$(docker inspect -f '{{.State.Running}} {{.HostConfig.NetworkMode}}' k3s-postgres-recovery 2>/dev/null || true)" + [ "$state" = "true host" ] || block "postgres_recovery_container_not_host_running" + if docker inspect -f '{{range .Mounts}}{{println .Source .Destination}}{{end}}' k3s-postgres-recovery 2>/dev/null \ + | awk '$2 == "/var/lib/postgresql/14/main" && $1 == "/var/lib/postgresql/14/main" {found=1} END {exit !found}'; then + mount_ok=1 + fi + [ "$mount_ok" = "1" ] || block "postgres_recovery_data_mount_mismatch" + pg_isready -h localhost -p 5432 >/dev/null 2>&1 \ + || block "postgres_recovery_not_accepting_connections" +} + +print_status() { + local start_mode script_parity=0 unit_parity=0 pg_failed startup_failed last_backup="" + start_mode="$(postgres_start_mode)" + [ -f "$LIVE_SCRIPT" ] && [ "$(sha256_file "$LIVE_SCRIPT")" = "$(sha256_file "$CANONICAL_SCRIPT")" ] \ + && script_parity=1 + [ -f "$LIVE_UNIT" ] && [ "$(sha256_file "$LIVE_UNIT")" = "$(sha256_file "$CANONICAL_UNIT")" ] \ + && unit_parity=1 + pg_failed="$(systemctl is-failed postgresql@14-main.service 2>/dev/null || true)" + startup_failed="$(systemctl is-failed awoooi-startup.service 2>/dev/null || true)" + [ -f "$LAST_BACKUP_FILE" ] && last_backup="$(cat "$LAST_BACKUP_FILE")" + + echo "AWOOOI_HOST188_HYGIENE_EXECUTOR=1" + echo "HOST188_HYGIENE_EXECUTOR_READY=1" + echo "RUN_ID=$RUN_ID" + echo "POSTGRES_RECOVERY_READY=1" + echo "POSTGRES_START_MODE=${start_mode:-unknown}" + echo "STARTUP_SCRIPT_PARITY=$script_parity" + echo "STARTUP_UNIT_PARITY=$unit_parity" + echo "POSTGRES_UNIT_STATE=${pg_failed:-unknown}" + echo "STARTUP_UNIT_STATE=${startup_failed:-unknown}" + echo "CANONICAL_SCRIPT_SHA256=$(sha256_file "$CANONICAL_SCRIPT")" + echo "CANONICAL_UNIT_SHA256=$(sha256_file "$CANONICAL_UNIT")" + echo "LAST_BACKUP_DIR=$last_backup" + if [ "$start_mode" = "manual" ] \ + && [ "$script_parity" = "1" ] \ + && [ "$unit_parity" = "1" ] \ + && [ "$pg_failed" != "failed" ] \ + && [ "$startup_failed" != "failed" ]; then + echo "HOST188_HYGIENE_READY=1" + else + echo "HOST188_HYGIENE_READY=0" + fi +} + +create_backup() { + local backup_dir="${BACKUP_ROOT}/${RUN_ID}-before-reconcile" + install -d -m 0700 "$backup_dir" + cp -a "$POSTGRES_START_CONF" "$backup_dir/start.conf" + cp -a "$LIVE_SCRIPT" "$backup_dir/awoooi-startup.sh" + cp -a "$LIVE_UNIT" "$backup_dir/awoooi-startup.service" + printf '%s\n' "$backup_dir" +} + +restore_backup() { + local backup_dir="$1" + case "$backup_dir" in + "$BACKUP_ROOT"/*) ;; + *) block "hygiene_rollback_path_invalid" ;; + esac + [ -f "$backup_dir/start.conf" ] \ + && [ -f "$backup_dir/awoooi-startup.sh" ] \ + && [ -f "$backup_dir/awoooi-startup.service" ] \ + || block "hygiene_rollback_payload_missing" + install -m 0644 "$backup_dir/start.conf" "$POSTGRES_START_CONF" + install -m 0755 "$backup_dir/awoooi-startup.sh" "$LIVE_SCRIPT" + install -m 0644 "$backup_dir/awoooi-startup.service" "$LIVE_UNIT" + systemctl daemon-reload +} + +write_last_backup() { + local tmp="${LAST_BACKUP_FILE}.tmp.$$" + printf '%s\n' "$1" >"$tmp" + install -m 0600 "$tmp" "$LAST_BACKUP_FILE" + rm -f "$tmp" +} + +reconcile() { + local backup_dir start_tmp + assert_recovery_runtime + backup_dir="$(create_backup)" + start_tmp="${POSTGRES_START_CONF}.candidate.$$" + printf '# Managed by awoooi-host188-hygiene; recovery container owns automatic startup.\nmanual\n' >"$start_tmp" + install -m 0644 "$start_tmp" "$POSTGRES_START_CONF" + rm -f "$start_tmp" + install -m 0755 "$CANONICAL_SCRIPT" "$LIVE_SCRIPT" + install -m 0644 "$CANONICAL_UNIT" "$LIVE_UNIT" + if ! systemd-analyze verify "$LIVE_UNIT"; then + restore_backup "$backup_dir" + block "startup_unit_verify_failed_rollback_restored" + fi + systemctl daemon-reload + systemctl reset-failed postgresql@14-main.service awoooi-startup.service + assert_recovery_runtime + if [ "$(postgres_start_mode)" != "manual" ] \ + || [ "$(systemctl is-failed postgresql@14-main.service 2>/dev/null || true)" = "failed" ] \ + || [ "$(systemctl is-failed awoooi-startup.service 2>/dev/null || true)" = "failed" ]; then + restore_backup "$backup_dir" + block "host188_hygiene_postverify_failed_rollback_restored" + fi + write_last_backup "$backup_dir" + audit "reconciled" + echo "HOST188_HYGIENE_EXECUTOR_RESULT=reconciled" + echo "RUN_ID=$RUN_ID" + echo "BACKUP_DIR=$backup_dir" + echo "DAEMON_RELOAD_DONE=1" + echo "FAILED_STATE_RESET_DONE=1" + echo "PRODUCT_SERVICE_RESTARTS=0" +} + +rollback_latest() { + [ -f "$LAST_BACKUP_FILE" ] || block "hygiene_last_backup_missing" + restore_backup "$(cat "$LAST_BACKUP_FILE")" + systemctl reset-failed postgresql@14-main.service awoooi-startup.service + audit "rolled_back" + echo "HOST188_HYGIENE_EXECUTOR_RESULT=rolled_back" + echo "RUN_ID=$RUN_ID" + echo "PRODUCT_SERVICE_RESTARTS=0" +} + +if [ "${EUID:-$(id -u)}" -ne 0 ]; then + echo "BLOCKER hygiene_executor_requires_root" + exit 77 +fi + +install -d -m 0700 "$STATE_DIR" "$BACKUP_ROOT" +touch "$AUDIT_LOG" +chmod 0600 "$AUDIT_LOG" +assert_canonical_payload +assert_recovery_runtime + +exec 9>"$LOCK_FILE" +flock -w 30 9 || block "hygiene_executor_lock_timeout" + +case "$ACTION" in + check) + print_status + ;; + reconcile) + reconcile + print_status + ;; + rollback-latest) + rollback_latest + print_status + ;; + *) + block "unsupported_hygiene_executor_action" + ;; +esac diff --git a/ops/reboot-recovery/awoooi-host188-hygiene.sudoers b/ops/reboot-recovery/awoooi-host188-hygiene.sudoers new file mode 100644 index 000000000..15fca75d5 --- /dev/null +++ b/ops/reboot-recovery/awoooi-host188-hygiene.sudoers @@ -0,0 +1,2 @@ +Cmnd_Alias AWOOOI_HOST188_HYGIENE = /usr/local/sbin/awoooi-host188-hygiene check, /usr/local/sbin/awoooi-host188-hygiene reconcile, /usr/local/sbin/awoooi-host188-hygiene rollback-latest +ollama ALL=(root) NOPASSWD: AWOOOI_HOST188_HYGIENE diff --git a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh index 8a253dcec..a330ef026 100755 --- a/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh +++ b/scripts/reboot-recovery/ai-log-triage-auto-recovery.sh @@ -11,8 +11,8 @@ set -uo pipefail MODE="check" ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-ai-log-triage-$(date '+%Y%m%d-%H%M%S')}" SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-180}" -WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}" -WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS="${WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS:-210}" +WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}" +WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS="${WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS:-60}" SSH_STRICT_HOST_KEY_CHECKING="${SSH_STRICT_HOST_KEY_CHECKING:-accept-new}" HOST_110="${HOST_110:-wooo@192.168.0.110}" HOST_120="${HOST_120:-wooo@192.168.0.120}" @@ -250,7 +250,7 @@ collect_windows99() { fi WINDOWS99_REMOTE_VERIFY_TIMEOUT="$WINDOWS99_REMOTE_VERIFY_TIMEOUT" \ run_timeout "$WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" bash "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" \ - --check --via-host "$WINDOWS99_VIA_HOST" --users wooo --timeout 8 --max-auth-users 1 \ + --check --via-host "$WINDOWS99_VIA_HOST" --users Administrator --timeout 8 --max-auth-users 1 \ >"$file" 2>&1 || rc=$? printf 'COLLECT label=windows99-vmware rc=%s file=%s\n' "$rc" "$file" | tee -a "$summary_file" } diff --git a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer index a50db82fa..4d9f5c67f 100644 --- a/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer +++ b/scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer @@ -3,7 +3,7 @@ Description=Run AWOOOI reboot auto-recovery SLO verifier after boot and until st [Timer] OnBootSec=2min -OnUnitActiveSec=2min +OnUnitInactiveSec=5min AccuracySec=15s Persistent=true Unit=awoooi-reboot-auto-recovery-slo.service diff --git a/scripts/reboot-recovery/awoooi-startup.sh b/scripts/reboot-recovery/awoooi-startup.sh index 03fd9d7d1..2ffe6ec27 100644 --- a/scripts/reboot-recovery/awoooi-startup.sh +++ b/scripts/reboot-recovery/awoooi-startup.sh @@ -19,6 +19,10 @@ log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*"; } CLAWBOT_UP_TIMEOUT_SECONDS="${CLAWBOT_UP_TIMEOUT_SECONDS:-90}" CLAWBOT_BUILD_TIMEOUT_SECONDS="${CLAWBOT_BUILD_TIMEOUT_SECONDS:-120}" CLAWBOT_STARTUP_REBUILD_ALLOWED="${CLAWBOT_STARTUP_REBUILD_ALLOWED:-0}" +CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}" +DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}" +KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}" +SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}" run_bounded() { seconds="$1" @@ -63,7 +67,11 @@ if ! systemctl is-active containerd >/dev/null 2>&1; then log "containerd 啟動失敗，檢查 BoltDB 損壞..." BOLT_DB="/var/lib/containerd/io.containerd.metadata.v1.bolt/meta.db" if [ -f "$BOLT_DB" ]; then - log "備份並刪除損壞的 meta.db..." + if [ "$CONTAINERD_METADATA_REPAIR_ALLOWED" != "1" ]; then + log "❌ containerd metadata repair 未授權；禁止自動備份/刪除 meta.db" + exit 1 + fi + log "containerd metadata repair 已授權；先備份再移除損壞的 meta.db..." cp "$BOLT_DB" "${BOLT_DB}.bak.$(date +%Y%m%d%H%M%S)" rm -f "$BOLT_DB" fi @@ -89,7 +97,11 @@ if ! systemctl is-active docker >/dev/null 2>&1; then log "Docker 啟動失敗，修復 network BoltDB..." NETWORK_DB="/var/lib/docker/network/files/local-kv.db" if [ -f "$NETWORK_DB" ]; then - log "備份並刪除損壞的 local-kv.db..." + if [ "$DOCKER_NETWORK_METADATA_REPAIR_ALLOWED" != "1" ]; then + log "❌ Docker network metadata repair 未授權；禁止自動備份/刪除 local-kv.db" + exit 1 + fi + log "Docker network metadata repair 已授權；先備份再移除 local-kv.db..." cp "$NETWORK_DB" "${NETWORK_DB}.bak.$(date +%Y%m%d%H%M%S)" rm -f "$NETWORK_DB" fi @@ -131,16 +143,21 @@ for i in $(seq 1 30); do done pg_isready -h localhost -p 5432 && log "✅ PostgreSQL accepting connections" || { log "❌ PostgreSQL 無法接受連線"; exit 1; } -# kine 表維護（若有 stale 連線或 WAL 剛重置） -log "維護 k3s_datastore kine 表..." -sudo -u postgres psql -d k3s_datastore -c " - SELECT pg_terminate_backend(pid) - FROM pg_stat_activity - WHERE datname='k3s_datastore' AND pid!=pg_backend_pid() AND state='active' - AND query_start < now() - interval '5 minutes'; -" 2>/dev/null || true - -sudo -u postgres psql -d k3s_datastore -c "VACUUM ANALYZE kine;" 2>/dev/null && log "✅ kine VACUUM ANALYZE 完成" || log "⚠️ kine VACUUM 跳過（可能無法連線）" +# kine 維護不屬於開機恢復的必要寫入，預設停用。 +if [ "$KINE_MAINTENANCE_ALLOWED" = "1" ]; then + log "kine maintenance 已授權，執行 bounded stale-session cleanup / VACUUM..." + sudo -u postgres psql -d k3s_datastore -c " + SELECT pg_terminate_backend(pid) + FROM pg_stat_activity + WHERE datname='k3s_datastore' AND pid!=pg_backend_pid() AND state='active' + AND query_start < now() - interval '5 minutes'; + " 2>/dev/null || true + sudo -u postgres psql -d k3s_datastore -c "VACUUM ANALYZE kine;" 2>/dev/null \ + && log "✅ kine VACUUM ANALYZE 完成" \ + || log "⚠️ kine VACUUM 未完成" +else + log "✅ kine maintenance 預設停用，開機恢復不寫入資料庫" +fi # ────────────────────────────────────────────── # STEP 4: Redis @@ -171,7 +188,9 @@ log "[7/7] 啟動 Docker Compose 服務..." # SignOz SIGNOZ_DIR="/home/ollama/signoz/deploy/docker" -if [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then +if [ "$SIGNOZ_188_START_ALLOWED" != "1" ]; then + log "✅ 188 legacy SignOz startup 預設停用；正式 SignOz 由 110 runtime 提供" +elif [ -f "$SIGNOZ_DIR/docker-compose.yaml" ]; then log "啟動 SignOz..." cd "$SIGNOZ_DIR" docker compose up -d 2>&1 | tail -3 @@ -194,7 +213,7 @@ fi # ClawBot（依賴 aiops-network） # 確保 aiops-network 存在（重開機後 external network 需手動建立） -if ! docker network ls | grep -q aiops-network; then +if ! docker network inspect aiops-network >/dev/null 2>&1; then log "建立 aiops-network..." docker network create aiops-network && log "✅ aiops-network 建立" || log "⚠️ aiops-network 建立失敗" fi diff --git a/scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh b/scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh new file mode 100755 index 000000000..35305edd6 --- /dev/null +++ b/scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MODE="check" + +EDGE_HOST="${EDGE_HOST:-192.168.0.188}" +EDGE_USER="${EDGE_USER:-ollama}" +EDGE_TARGET="${EDGE_USER}@${EDGE_HOST}" +REMOTE_HELPER="/usr/local/sbin/awoooi-edge-maintenance" +REMOTE_STAGE_ROOT="${REMOTE_STAGE_ROOT:-/home/ollama/.cache/awoooi-edge-bootstrap}" +BOOTSTRAP_IMAGE="${BOOTSTRAP_IMAGE:-alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc}" + +LOCAL_HELPER="${ROOT_DIR}/ops/reboot-recovery/awoooi-edge-maintenance-helper.sh" +LOCAL_SUDOERS="${ROOT_DIR}/ops/reboot-recovery/awoooi-edge-maintenance.sudoers" +LOCAL_CONF="${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf" +LOCAL_MAINT="${ROOT_DIR}/ops/maintenance/maintenance.html" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}" + -o StrictHostKeyChecking=accept-new +) + +usage() { + cat <<'USAGE' +Usage: + bootstrap-edge-maintenance-executor.sh --check + bootstrap-edge-maintenance-executor.sh --apply + +--apply is a one-time bounded bootstrap over the edge host's existing Docker +root-equivalent channel. It uses only a pinned local image with no network, +installs a fixed helper and exact-command sudoers allowlist, and never applies +the Nginx payload itself. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "UNKNOWN_ARG=$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +for file in "$LOCAL_HELPER" "$LOCAL_SUDOERS" "$LOCAL_CONF" "$LOCAL_MAINT"; do + if [ ! -f "$file" ]; then + echo "BLOCKER bootstrap_source_missing path=$file" + exit 66 + fi +done + +remote_check() { + echo "AWOOOI_EDGE_EXECUTOR_BOOTSTRAP=1" + echo "MODE=check" + if ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "sudo -n '$REMOTE_HELPER' check"; then + echo "EDGE_EXECUTOR_BOOTSTRAP_READY=1" + else + echo "EDGE_EXECUTOR_BOOTSTRAP_READY=0" + echo "BLOCKER privileged_edge_apply_channel_unavailable" + return 75 + fi +} + +if [ "$MODE" = "check" ]; then + remote_check + exit $? +fi + +RUN_ID="edge-bootstrap-$(date +%Y%m%dT%H%M%S%z)-$$" +REMOTE_STAGE="${REMOTE_STAGE_ROOT}/${RUN_ID}" + +echo "AWOOOI_EDGE_EXECUTOR_BOOTSTRAP=1" +echo "MODE=apply" +echo "RUN_ID=$RUN_ID" +echo "BOOTSTRAP_IMAGE=$BOOTSTRAP_IMAGE" +echo "LOCAL_HELPER_SHA256=$(sha256sum "$LOCAL_HELPER" | awk '{print $1}')" +echo "LOCAL_SUDOERS_SHA256=$(sha256sum "$LOCAL_SUDOERS" | awk '{print $1}')" +echo "LOCAL_CONF_SHA256=$(sha256sum "$LOCAL_CONF" | awk '{print $1}')" +echo "LOCAL_MAINT_SHA256=$(sha256sum "$LOCAL_MAINT" | awk '{print $1}')" + +ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "install -d -m 0700 '$REMOTE_STAGE'" +scp -q "${SSH_OPTS[@]}" \ + "$LOCAL_HELPER" "$LOCAL_SUDOERS" "$LOCAL_CONF" "$LOCAL_MAINT" \ + "${EDGE_TARGET}:${REMOTE_STAGE}/" + +ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "docker image inspect --format '{{.Id}}' '$BOOTSTRAP_IMAGE'" + +ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ + "docker run -i --rm --pull=never --network=none --userns=host -e RUN_ID='$RUN_ID' -v /:/host -v '$REMOTE_STAGE':/payload:ro '$BOOTSTRAP_IMAGE' /bin/sh -s" <<'CONTAINER' +set -eu + +HELPER=/host/usr/local/sbin/awoooi-edge-maintenance +SUDOERS=/host/etc/sudoers.d/awoooi-edge-maintenance +CANONICAL_DIR=/host/usr/local/share/awoooi-edge-maintenance +CANONICAL_CONF=$CANONICAL_DIR/awoooi.wooo.work.conf +CANONICAL_MAINT=$CANONICAL_DIR/maintenance.html +BACKUP_ROOT=/host/var/lib/awoooi-edge-maintenance/bootstrap-backups +BACKUP_DIR=$BACKUP_ROOT/$RUN_ID + +mkdir -p /host/usr/local/sbin /host/etc/sudoers.d "$CANONICAL_DIR" "$BACKUP_DIR" +chmod 0700 "$BACKUP_ROOT" "$BACKUP_DIR" + +backup_file() { + src=$1 + name=$2 + if [ -e "$src" ]; then + cp -a "$src" "$BACKUP_DIR/$name" + else + : >"$BACKUP_DIR/$name.missing" + fi +} + +restore_file() { + dst=$1 + name=$2 + if [ -e "$BACKUP_DIR/$name" ]; then + cp -a "$BACKUP_DIR/$name" "$dst" + elif [ -e "$BACKUP_DIR/$name.missing" ]; then + rm -f "$dst" + fi +} + +backup_file "$HELPER" helper +backup_file "$SUDOERS" sudoers +backup_file "$CANONICAL_CONF" canonical-conf +backup_file "$CANONICAL_MAINT" canonical-maint + +committed=0 +rollback_on_error() { + rc=$? + if [ "$rc" -ne 0 ] && [ "$committed" -eq 0 ]; then + restore_file "$HELPER" helper + restore_file "$SUDOERS" sudoers + restore_file "$CANONICAL_CONF" canonical-conf + restore_file "$CANONICAL_MAINT" canonical-maint + echo "BOOTSTRAP_ROLLBACK_DONE=1" + fi + exit "$rc" +} +trap rollback_on_error EXIT + +cp /payload/awoooi-edge-maintenance-helper.sh "$HELPER" +cp /payload/awoooi-edge-maintenance.sudoers "$SUDOERS" +cp /payload/awoooi.wooo.work.conf "$CANONICAL_CONF" +cp /payload/maintenance.html "$CANONICAL_MAINT" +chown 0:0 "$HELPER" "$SUDOERS" "$CANONICAL_CONF" "$CANONICAL_MAINT" +chmod 0755 "$HELPER" +chmod 0440 "$SUDOERS" +chmod 0644 "$CANONICAL_CONF" "$CANONICAL_MAINT" + +chroot /host /bin/bash -n /usr/local/sbin/awoooi-edge-maintenance +chroot /host /usr/sbin/visudo -cf /etc/sudoers.d/awoooi-edge-maintenance + +committed=1 +trap - EXIT +echo "BOOTSTRAP_BACKUP_DIR=${BACKUP_DIR#/host}" +echo "BOOTSTRAP_INSTALL_DONE=1" +CONTAINER + +ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "rm -rf '$REMOTE_STAGE'" +remote_check diff --git a/scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh b/scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh new file mode 100755 index 000000000..525173f50 --- /dev/null +++ b/scripts/reboot-recovery/bootstrap-host188-hygiene-executor.sh @@ -0,0 +1,171 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MODE="check" +TARGET="${TARGET:-ollama@192.168.0.188}" +REMOTE_HELPER="/usr/local/sbin/awoooi-host188-hygiene" +REMOTE_STAGE_ROOT="${REMOTE_STAGE_ROOT:-/home/ollama/.cache/awoooi-host188-hygiene-bootstrap}" +BOOTSTRAP_IMAGE="${BOOTSTRAP_IMAGE:-alpine@sha256:d9e853e87e55526f6b2917df91a2115c36dd7c696a35be12163d44e6e2a4b6bc}" + +LOCAL_HELPER="${ROOT_DIR}/ops/reboot-recovery/awoooi-host188-hygiene-helper.sh" +LOCAL_SUDOERS="${ROOT_DIR}/ops/reboot-recovery/awoooi-host188-hygiene.sudoers" +LOCAL_STARTUP="${ROOT_DIR}/scripts/reboot-recovery/awoooi-startup.sh" +LOCAL_UNIT="${ROOT_DIR}/scripts/reboot-recovery/awoooi-startup.service" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}" + -o StrictHostKeyChecking=accept-new +) + +usage() { + cat <<'USAGE' +Usage: + bootstrap-host188-hygiene-executor.sh --check + bootstrap-host188-hygiene-executor.sh --apply + +Installs only the root-owned fixed-action host hygiene helper, canonical startup +payload, and exact sudoers commands. It does not reconcile or restart services. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "UNKNOWN_ARG=$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +for file in "$LOCAL_HELPER" "$LOCAL_SUDOERS" "$LOCAL_STARTUP" "$LOCAL_UNIT"; do + [ -f "$file" ] || { + echo "BLOCKER host188_hygiene_bootstrap_source_missing path=$file" + exit 66 + } +done + +check() { + echo "AWOOOI_HOST188_HYGIENE_BOOTSTRAP=1" + echo "MODE=check" + if ssh "${SSH_OPTS[@]}" "$TARGET" "sudo -n '$REMOTE_HELPER' check"; then + echo "HOST188_HYGIENE_BOOTSTRAP_READY=1" + else + echo "HOST188_HYGIENE_BOOTSTRAP_READY=0" + echo "BLOCKER host188_hygiene_privileged_executor_unavailable" + return 75 + fi +} + +if [ "$MODE" = "check" ]; then + check + exit $? +fi + +RUN_ID="host188-hygiene-bootstrap-$(date +%Y%m%dT%H%M%S%z)-$$" +REMOTE_STAGE="${REMOTE_STAGE_ROOT}/${RUN_ID}" + +echo "AWOOOI_HOST188_HYGIENE_BOOTSTRAP=1" +echo "MODE=apply" +echo "RUN_ID=$RUN_ID" +echo "BOOTSTRAP_IMAGE=$BOOTSTRAP_IMAGE" +echo "LOCAL_HELPER_SHA256=$(sha256sum "$LOCAL_HELPER" | awk '{print $1}')" +echo "LOCAL_STARTUP_SHA256=$(sha256sum "$LOCAL_STARTUP" | awk '{print $1}')" +echo "LOCAL_UNIT_SHA256=$(sha256sum "$LOCAL_UNIT" | awk '{print $1}')" + +ssh "${SSH_OPTS[@]}" "$TARGET" "install -d -m 0700 '$REMOTE_STAGE'" +scp -q "${SSH_OPTS[@]}" \ + "$LOCAL_HELPER" "$LOCAL_SUDOERS" "$LOCAL_STARTUP" "$LOCAL_UNIT" \ + "${TARGET}:${REMOTE_STAGE}/" +ssh "${SSH_OPTS[@]}" "$TARGET" \ + "docker image inspect --format '{{.Id}}' '$BOOTSTRAP_IMAGE'" + +ssh "${SSH_OPTS[@]}" "$TARGET" \ + "docker run -i --rm --pull=never --network=none --userns=host -e RUN_ID='$RUN_ID' -v /:/host -v '$REMOTE_STAGE':/payload:ro '$BOOTSTRAP_IMAGE' /bin/sh -s" <<'CONTAINER' +set -eu + +HELPER=/host/usr/local/sbin/awoooi-host188-hygiene +SUDOERS=/host/etc/sudoers.d/awoooi-host188-hygiene +CANONICAL_DIR=/host/usr/local/share/awoooi-host188-hygiene +CANONICAL_STARTUP=$CANONICAL_DIR/awoooi-startup.sh +CANONICAL_UNIT=$CANONICAL_DIR/awoooi-startup.service +BACKUP_ROOT=/host/var/lib/awoooi-host188-hygiene/bootstrap-backups +BACKUP_DIR=$BACKUP_ROOT/$RUN_ID + +mkdir -p /host/usr/local/sbin /host/etc/sudoers.d "$CANONICAL_DIR" "$BACKUP_DIR" +chmod 0700 "$BACKUP_ROOT" "$BACKUP_DIR" + +backup_file() { + src=$1 + name=$2 + if [ -e "$src" ]; then + cp -a "$src" "$BACKUP_DIR/$name" + else + : >"$BACKUP_DIR/$name.missing" + fi +} + +restore_file() { + dst=$1 + name=$2 + if [ -e "$BACKUP_DIR/$name" ]; then + cp -a "$BACKUP_DIR/$name" "$dst" + elif [ -e "$BACKUP_DIR/$name.missing" ]; then + rm -f "$dst" + fi +} + +backup_file "$HELPER" helper +backup_file "$SUDOERS" sudoers +backup_file "$CANONICAL_STARTUP" canonical-startup +backup_file "$CANONICAL_UNIT" canonical-unit + +committed=0 +rollback_on_error() { + rc=$? + if [ "$rc" -ne 0 ] && [ "$committed" -eq 0 ]; then + restore_file "$HELPER" helper + restore_file "$SUDOERS" sudoers + restore_file "$CANONICAL_STARTUP" canonical-startup + restore_file "$CANONICAL_UNIT" canonical-unit + echo "BOOTSTRAP_ROLLBACK_DONE=1" + fi + exit "$rc" +} +trap rollback_on_error EXIT + +cp /payload/awoooi-host188-hygiene-helper.sh "$HELPER" +cp /payload/awoooi-host188-hygiene.sudoers "$SUDOERS" +cp /payload/awoooi-startup.sh "$CANONICAL_STARTUP" +cp /payload/awoooi-startup.service "$CANONICAL_UNIT" +chown 0:0 "$HELPER" "$SUDOERS" "$CANONICAL_STARTUP" "$CANONICAL_UNIT" +chmod 0755 "$HELPER" "$CANONICAL_STARTUP" +chmod 0440 "$SUDOERS" +chmod 0644 "$CANONICAL_UNIT" + +chroot /host /bin/bash -n /usr/local/sbin/awoooi-host188-hygiene +chroot /host /bin/bash -n /usr/local/share/awoooi-host188-hygiene/awoooi-startup.sh +chroot /host /usr/sbin/visudo -cf /etc/sudoers.d/awoooi-host188-hygiene +chroot /host /usr/bin/systemd-analyze verify /usr/local/share/awoooi-host188-hygiene/awoooi-startup.service + +committed=1 +trap - EXIT +echo "BOOTSTRAP_BACKUP_DIR=${BACKUP_DIR#/host}" +echo "BOOTSTRAP_INSTALL_DONE=1" +CONTAINER + +ssh "${SSH_OPTS[@]}" "$TARGET" "rm -rf '$REMOTE_STAGE'" +check diff --git a/scripts/reboot-recovery/collect-windows99-vmware-verify.sh b/scripts/reboot-recovery/collect-windows99-vmware-verify.sh index ef5c90912..31cc324f5 100755 --- a/scripts/reboot-recovery/collect-windows99-vmware-verify.sh +++ b/scripts/reboot-recovery/collect-windows99-vmware-verify.sh @@ -7,17 +7,17 @@ TARGET_HOST="${WINDOWS99_HOST:-192.168.0.99}" VIA_HOST="${WINDOWS99_VIA_HOST:-}" CONNECT_TIMEOUT="${WINDOWS99_CONNECT_TIMEOUT:-3}" SSH_TIMEOUT="${WINDOWS99_SSH_TIMEOUT:-3}" -REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}" +REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}" SSH_PORT="${WINDOWS99_SSH_PORT:-22}" MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-}" MAX_AUTH_USERS_EXPLICIT=0 if [[ -n "${WINDOWS99_MAX_AUTH_USERS:-}" ]]; then MAX_AUTH_USERS_EXPLICIT=1 fi -KNOWN_HOSTS_FILE="${WINDOWS99_KNOWN_HOSTS_FILE:-/tmp/awoooi-windows99-known_hosts}" +KNOWN_HOSTS_FILE="${WINDOWS99_KNOWN_HOSTS_FILE:-${HOME:-/home/wooo}/.ssh/known_hosts}" LOCAL_VERIFY_SCRIPT="${WINDOWS99_LOCAL_VERIFY_SCRIPT:-${SCRIPT_DIR}/windows99-vmware-autostart.ps1}" REMOTE_VERIFY_COMMAND="${WINDOWS99_REMOTE_VERIFY_COMMAND:-powershell -NoProfile -ExecutionPolicy Bypass -Command \"& ([scriptblock]::Create([Console]::In.ReadToEnd())) -Mode Verify\"}" -SSH_USERS=(ogt wooo ooo administrator Administrator) +SSH_USERS=(Administrator ogt wooo ooo administrator) SSH_USERS_EXPLICIT=0 if [[ -n "${WINDOWS99_SSH_USERS:-}" ]]; then @@ -37,7 +37,7 @@ if ! is_positive_int "${SSH_TIMEOUT}"; then SSH_TIMEOUT=3 fi if ! is_positive_int "${REMOTE_VERIFY_TIMEOUT}"; then - REMOTE_VERIFY_TIMEOUT=180 + REMOTE_VERIFY_TIMEOUT=45 fi if ! is_positive_int "${MAX_AUTH_USERS}"; then MAX_AUTH_USERS="${#SSH_USERS[@]}" @@ -99,7 +99,7 @@ if ! is_positive_int "${SSH_TIMEOUT}"; then SSH_TIMEOUT=3 fi if ! is_positive_int "${REMOTE_VERIFY_TIMEOUT}"; then - REMOTE_VERIFY_TIMEOUT=180 + REMOTE_VERIFY_TIMEOUT=45 fi if ! is_positive_int "${MAX_AUTH_USERS}"; then MAX_AUTH_USERS="${#SSH_USERS[@]}" @@ -187,9 +187,11 @@ SSH_COMMON_OPTS=( -o NumberOfPasswordPrompts=0 -o ConnectTimeout="${SSH_TIMEOUT}" -o ConnectionAttempts=1 + -o ServerAliveInterval=5 + -o ServerAliveCountMax=2 -o GSSAPIAuthentication=no -o LogLevel=ERROR - -o StrictHostKeyChecking=no + -o StrictHostKeyChecking=accept-new -o UserKnownHostsFile="${KNOWN_HOSTS_FILE}" ) SSH_TARGET_OPTS=( @@ -223,14 +225,14 @@ run_ssh_timed() { timeout_seconds="${SSH_TIMEOUT}" timeout_wrapper_seconds="$((SSH_TIMEOUT + 1))" fi - local target_ssh=(ssh "${SSH_TARGET_OPTS[@]}" -o ConnectTimeout="${timeout_seconds}" "${user}@${TARGET_HOST}" "$@") + local target_ssh=(ssh "${SSH_TARGET_OPTS[@]}" "${user}@${TARGET_HOST}" "$@") if [[ -n "${VIA_HOST}" ]]; then local inner_command inner_command="$(quote_words "${target_ssh[@]}")" run_with_timeout_wrapper \ "${timeout_seconds}" \ "${timeout_wrapper_seconds}" \ - ssh "${SSH_COMMON_OPTS[@]}" -o ConnectTimeout="${timeout_seconds}" \ + ssh "${SSH_COMMON_OPTS[@]}" \ "${VIA_HOST}" "bash -lc $(printf '%q' "${inner_command}")" else run_with_timeout_wrapper \ diff --git a/scripts/reboot-recovery/enforce-110-runner-failclosed.sh b/scripts/reboot-recovery/enforce-110-runner-failclosed.sh index 42ce24b99..158f60aa3 100755 --- a/scripts/reboot-recovery/enforce-110-runner-failclosed.sh +++ b/scripts/reboot-recovery/enforce-110-runner-failclosed.sh @@ -14,9 +14,10 @@ COMPAT_ENFORCER="/usr/local/bin/awoooi-enforce-runner-failclosed-110.sh" usage() { cat <<'USAGE' -Usage: enforce-110-runner-failclosed.sh [--check|--apply] +Usage: enforce-110-runner-failclosed.sh [--check|--ensure|--apply] --check Read-only status check. Exit non-zero if runner/CD lane is open. +--ensure Check first and apply only when fail-closed state has drifted. --apply Stop/mask runner/CD lane entrypoints and seal restore sources. USAGE } @@ -26,6 +27,9 @@ while [ "$#" -gt 0 ]; do --check) MODE="check" ;; + --ensure) + MODE="ensure" + ;; --apply) MODE="apply" ;; @@ -663,7 +667,7 @@ repair_enforcer_cron_authority() { cat >"$tmp" <<'EOF' SHELL=/bin/bash PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin -* * * * * root /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --apply >>/var/log/awoooi-runner-failclosed-authority-cron.log 2>&1 +* * * * * root /usr/bin/flock -n /run/lock/awoooi-runner-failclosed-authority.lock /usr/bin/timeout 180 /usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh --ensure >>/var/log/awoooi-runner-failclosed-authority-cron.log 2>&1 EOF as_root chattr -i /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true as_root install -o root -g root -m 0644 "$tmp" /etc/cron.d/awoooi-runner-failclosed-authority >/dev/null 2>&1 || true @@ -959,6 +963,14 @@ print_readback() { done < <(list_action_runner_units) } +failclosed_state_ok() { + [ "$(count_active_job_containers)" = "0" ] \ + && lane_process_count_ok \ + && [ "$(count_runner_processes)" = "0" ] \ + && [ "$(root_restore_sources_left)" = "0" ] \ + && [ "$(runner_units_bad_count)" = "0" ] +} + apply_failclosed() { APPLY_PERFORMED=1 repair_enforcer_entrypoints @@ -996,17 +1008,15 @@ fi if [ "$MODE" = "apply" ]; then apply_failclosed +elif [ "$MODE" = "ensure" ] && ! failclosed_state_ok; then + apply_failclosed fi write_metrics "/var/lib/node_exporter/textfile_collector" write_metrics "/home/wooo/node_exporter_textfiles" print_readback -if [ "$(count_active_job_containers)" = "0" ] \ - && lane_process_count_ok \ - && [ "$(count_runner_processes)" = "0" ] \ - && [ "$(root_restore_sources_left)" = "0" ] \ - && [ "$(runner_units_bad_count)" = "0" ]; then +if failclosed_state_ok; then exit 0 fi diff --git a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh new file mode 100755 index 000000000..f0b27a539 --- /dev/null +++ b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh @@ -0,0 +1,152 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +MODE="check" + +CONTROL_HOST="${CONTROL_HOST:-wooo@192.168.0.110}" +CONTROL_PUBLIC_KEY_PATH="${CONTROL_PUBLIC_KEY_PATH:-/home/wooo/.ssh/id_ed25519.pub}" +TARGET_HOST="${TARGET_HOST:-ollama-111-gpu}" +TARGET_ADDRESS="${TARGET_ADDRESS:-192.168.0.111}" +TARGET_USER="${TARGET_USER:-ooo}" +REMOTE_SCRIPT="${REMOTE_SCRIPT:-/Users/ooo/.local/bin/awoooi-host-boot-readback}" +LOCAL_SCRIPT="${ROOT_DIR}/scripts/reboot-recovery/macos-host-boot-readback.sh" + +SSH_OPTS=( + -o BatchMode=yes + -o ConnectTimeout="${SSH_CONNECT_TIMEOUT_SECONDS:-8}" + -o ConnectionAttempts=1 + -o NumberOfPasswordPrompts=0 +) + +usage() { + cat <<'USAGE' +Usage: + install-macos111-host-boot-readback.sh --check + install-macos111-host-boot-readback.sh --apply + +Installs a no-secret Darwin boot readback and authorizes host 110's existing +public key with a forced command. The key cannot open a shell or forwarding. +USAGE +} + +while [ "$#" -gt 0 ]; do + case "$1" in + --check) + MODE="check" + ;; + --apply) + MODE="apply" + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "UNKNOWN_ARG=$1" >&2 + usage >&2 + exit 64 + ;; + esac + shift +done + +if [ ! -f "$LOCAL_SCRIPT" ]; then + echo "BLOCKER macos_boot_readback_source_missing" + exit 66 +fi + +control_path_check() { + ssh "${SSH_OPTS[@]}" "$CONTROL_HOST" \ + "ssh -i /home/wooo/.ssh/id_ed25519 -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/home/wooo/.ssh/known_hosts -o ConnectTimeout=6 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 '${TARGET_USER}@${TARGET_ADDRESS}' ignored-by-forced-command" +} + +check() { + local direct_output control_output + echo "AWOOOI_MACOS111_BOOT_READBACK=1" + echo "MODE=check" + if ! direct_output="$(ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "'$REMOTE_SCRIPT'")"; then + echo "MACOS111_READBACK_SCRIPT_READY=0" + echo "BLOCKER macos111_readback_script_unavailable" + return 75 + fi + echo "$direct_output" + echo "MACOS111_READBACK_SCRIPT_READY=1" + + if ! control_output="$(control_path_check)"; then + echo "MACOS111_CONTROL_PATH_READY=0" + echo "BLOCKER macos111_restricted_key_unavailable" + return 75 + fi + echo "$control_output" + echo "MACOS111_CONTROL_PATH_READY=1" +} + +if [ "$MODE" = "check" ]; then + check + exit $? +fi + +RUN_ID="macos111-boot-readback-$(date +%Y%m%dT%H%M%S%z)-$$" +tmp_dir="$(mktemp -d "${TMPDIR:-/tmp}/macos111-readback.XXXXXX")" +trap 'rm -rf "$tmp_dir"' EXIT + +scp -q "${SSH_OPTS[@]}" \ + "${CONTROL_HOST}:${CONTROL_PUBLIC_KEY_PATH}" "$tmp_dir/control.pub" +ssh-keygen -lf "$tmp_dir/control.pub" >/dev/null + +remote_stage="/Users/ooo/.cache/${RUN_ID}" +ssh "${SSH_OPTS[@]}" "$TARGET_HOST" "install -d -m 0700 '$remote_stage'" +scp -q "${SSH_OPTS[@]}" "$LOCAL_SCRIPT" "$tmp_dir/control.pub" \ + "${TARGET_HOST}:${remote_stage}/" + +echo "AWOOOI_MACOS111_BOOT_READBACK=1" +echo "MODE=apply" +echo "RUN_ID=$RUN_ID" +echo "LOCAL_SCRIPT_SHA256=$(shasum -a 256 "$LOCAL_SCRIPT" | awk '{print $1}')" +echo "CONTROL_KEY_FINGERPRINT=$(ssh-keygen -lf "$tmp_dir/control.pub" | awk '{print $2}')" + +ssh "${SSH_OPTS[@]}" "$TARGET_HOST" \ + "RUN_ID='$RUN_ID' REMOTE_STAGE='$remote_stage' REMOTE_SCRIPT='$REMOTE_SCRIPT' bash -s" <<'REMOTE' +set -euo pipefail + +auth_dir="$HOME/.ssh" +auth_file="$auth_dir/authorized_keys" +backup_dir="$auth_dir/awoooi-backups/$RUN_ID" +install -d -m 0700 "$auth_dir" "$backup_dir" "$(dirname "$REMOTE_SCRIPT")" +if [ -f "$auth_file" ]; then + cp -p "$auth_file" "$backup_dir/authorized_keys" +else + : >"$backup_dir/authorized_keys.missing" + : >"$auth_file" +fi + +install -m 0755 "$REMOTE_STAGE/macos-host-boot-readback.sh" "$REMOTE_SCRIPT" +read -r key_type key_blob _ <"$REMOTE_STAGE/control.pub" +case "$key_type" in + ssh-ed25519|ssh-rsa|ecdsa-sha2-*) ;; + *) + echo "BLOCKER unsupported_control_public_key_type" + exit 65 + ;; +esac +[[ "$key_blob" =~ ^[A-Za-z0-9+/=]+$ ]] || { + echo "BLOCKER invalid_control_public_key_blob" + exit 65 +} + +auth_tmp="${auth_file}.tmp.$$" +awk -v blob="$key_blob" 'index($0, blob) == 0' "$auth_file" >"$auth_tmp" +printf 'restrict,command="%s" %s %s awoooi-host-probe-110\n' \ + "$REMOTE_SCRIPT" "$key_type" "$key_blob" >>"$auth_tmp" +chmod 0600 "$auth_tmp" +mv "$auth_tmp" "$auth_file" +chmod 0600 "$auth_file" +rm -rf "$REMOTE_STAGE" + +echo "AUTHORIZED_KEYS_BACKUP=$backup_dir" +echo "READBACK_SCRIPT_SHA256=$(shasum -a 256 "$REMOTE_SCRIPT" | awk '{print $1}')" +echo "RESTRICTED_AUTHORIZED_KEY_INSTALLED=1" +REMOTE + +check diff --git a/scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh b/scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh index 88130148f..b3bc21421 100644 --- a/scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh +++ b/scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh @@ -221,6 +221,7 @@ create_payload() { ops/nginx \ ops/maintenance \ ops/monitoring \ + ops/reboot-recovery \ docs/runbooks/PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md } @@ -231,7 +232,7 @@ case "$MODE" in cat </dev/null 2>&1; then + startup_enabled="enabled" +else + startup_enabled="unknown" +fi + +if curl --max-time 2 -fsS http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then + startup_active="active" +else + startup_active="inactive" +fi + +printf 'boot_id=darwin_%s uptime_seconds=%s systemd_state=darwin_ssh startup_enabled=%s startup_active=%s\n' \ + "$boot_epoch" "$uptime_seconds" "$startup_enabled" "$startup_active" diff --git a/scripts/reboot-recovery/post-reboot-readiness-summary.sh b/scripts/reboot-recovery/post-reboot-readiness-summary.sh index 6e0823555..06cc745bc 100755 --- a/scripts/reboot-recovery/post-reboot-readiness-summary.sh +++ b/scripts/reboot-recovery/post-reboot-readiness-summary.sh @@ -11,6 +11,7 @@ SHOW_LOGS=0 RUN_188_HYGIENE=1 RUN_WAZUH_GATES=1 ARTIFACT_DIR="${ARTIFACT_DIR:-/tmp/awoooi-post-reboot-readiness-$(date '+%Y%m%d-%H%M%S')}" +POST_START_QUICK_CHECK_TIMEOUT_SECONDS="${POST_START_QUICK_CHECK_TIMEOUT_SECONDS:-90}" usage() { cat <<'USAGE' @@ -63,6 +64,18 @@ done mkdir -p "$ARTIFACT_DIR" +run_with_timeout() { + local timeout_seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout --foreground --kill-after=3s "${timeout_seconds}s" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout --foreground --kill-after=3s "${timeout_seconds}s" "$@" + else + "$@" + fi +} + extract_key() { local key="$1" local file="$2" @@ -91,7 +104,15 @@ if [[ "${POST_REBOOT_SKIP_RUNNER_GUARD:-0}" == "1" ]]; then fi post_start_rc=0 -bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" >"$post_start_log" 2>&1 || post_start_rc=$? +post_start_timed_out=0 +run_with_timeout "$POST_START_QUICK_CHECK_TIMEOUT_SECONDS" \ + bash "$ROOT_DIR/scripts/reboot-recovery/post-start-quick-check.sh" "${post_start_args[@]}" \ + >"$post_start_log" 2>&1 || post_start_rc=$? +if [[ "$post_start_rc" == "124" || "$post_start_rc" == "137" ]]; then + post_start_timed_out=1 + printf 'POST_START_QUICK_CHECK_TIMEOUT=1 timeout_seconds=%s\n' \ + "$POST_START_QUICK_CHECK_TIMEOUT_SECONDS" >>"$post_start_log" +fi post_summary="$(grep -E '^POST_START_QUICK_CHECK PASS=[0-9]+ WARN=[0-9]+ BLOCKED=[0-9]+' "$post_start_log" | tail -n 1 || true)" post_warning_summary="$(grep -E '^POST_START_QUICK_CHECK_WARNINGS SERVICE=[0-9]+ BOUNDARY=[0-9]+ EVIDENCE=[0-9]+' "$post_start_log" | tail -n 1 || true)" @@ -215,6 +236,7 @@ elif [[ "$product_data_green" != "1" ]]; then next_required_gates+=("product_data_freshness_recovery") fi [[ "$backup_core_green" != "1" ]] && next_required_gates+=("backup_core_readback_recovery") +[[ "$post_start_timed_out" == "1" ]] && next_required_gates+=("post_start_quick_check_timeout_recovery") [[ "$dr_escrow_blocked" == "1" ]] && next_required_gates+=("credential_escrow_evidence") [[ "$host_188_hygiene_blocked" == "1" ]] && next_required_gates+=("host_188_hygiene_maintenance_window") [[ "$wazuh_registry_accepted" == "0" ]] && next_required_gates+=("wazuh_manager_registry_export") @@ -232,6 +254,8 @@ AWOOOI_POST_REBOOT_READINESS_SUMMARY=1 TIMESTAMP=$(date '+%Y-%m-%d %H:%M:%S %Z') ARTIFACT_DIR=$ARTIFACT_DIR POST_START_RC=$post_start_rc +POST_START_TIMEOUT_SECONDS=$POST_START_QUICK_CHECK_TIMEOUT_SECONDS +POST_START_TIMED_OUT=$post_start_timed_out POST_START_RESULT=${post_result:-unknown} POST_START_PASS=${post_pass:-unknown} POST_START_WARN=${post_warn:-unknown} diff --git a/scripts/reboot-recovery/post-start-quick-check.sh b/scripts/reboot-recovery/post-start-quick-check.sh index 83dc3a836..3302c8daf 100755 --- a/scripts/reboot-recovery/post-start-quick-check.sh +++ b/scripts/reboot-recovery/post-start-quick-check.sh @@ -8,6 +8,7 @@ set -uo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" SSH_CONNECT_TIMEOUT="${SSH_CONNECT_TIMEOUT:-6}" SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-25}" +SSH_TOTAL_TIMEOUT_SECONDS="${SSH_TOTAL_TIMEOUT_SECONDS:-$((SSH_CONNECT_TIMEOUT + SSH_COMMAND_TIMEOUT_SECONDS + 5))}" ROUTE_RETRY_ATTEMPTS="${ROUTE_RETRY_ATTEMPTS:-3}" ROUTE_RETRY_DELAY_SECONDS="${ROUTE_RETRY_DELAY_SECONDS:-2}" STOCK_FRESHNESS_RETRY_ATTEMPTS="${STOCK_FRESHNESS_RETRY_ATTEMPTS:-6}" @@ -208,16 +209,28 @@ is_local_target() { grep -Fxq "$host" <<<"$ips" } +run_with_timeout() { + local timeout_seconds="$1" + shift + if command -v timeout >/dev/null 2>&1; then + timeout --foreground --kill-after=3s "${timeout_seconds}s" "$@" + elif command -v gtimeout >/dev/null 2>&1; then + gtimeout --foreground --kill-after=3s "${timeout_seconds}s" "$@" + else + "$@" + fi +} + ssh_read() { local user_host="$1" local command="$2" local quoted_command="" if is_local_target "$user_host"; then - bash -lc "$command" + run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" bash -lc "$command" return $? fi printf -v quoted_command '%q' "$command" - ssh \ + local ssh_command=(ssh \ -o BatchMode=yes \ -o ConnectTimeout="$SSH_CONNECT_TIMEOUT" \ -o ConnectionAttempts=1 \ @@ -226,7 +239,8 @@ ssh_read() { -o PreferredAuthentications=publickey \ -o NumberOfPasswordPrompts=0 \ "$user_host" \ - "if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi" + "if command -v timeout >/dev/null 2>&1; then timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}; else bash -lc ${quoted_command}; fi") + run_with_timeout "$SSH_TOTAL_TIMEOUT_SECONDS" "${ssh_command[@]}" } service_route_recovered() { diff --git a/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh b/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh index c2324425e..bb8c6ab69 100755 --- a/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh +++ b/scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh @@ -14,6 +14,7 @@ REMOTE_MAINT_FILE="${REMOTE_MAINT_FILE:-${REMOTE_MAINT_DIR}/maintenance.html}" REMOTE_BACKUP_DIR="${REMOTE_BACKUP_DIR:-/home/ollama/awoooi-edge-maintenance-backups}" LOCAL_CONF="${LOCAL_CONF:-${ROOT_DIR}/ops/nginx/awoooi.wooo.work.conf}" LOCAL_MAINT_FILE="${LOCAL_MAINT_FILE:-${ROOT_DIR}/ops/maintenance/maintenance.html}" +REMOTE_HELPER="${REMOTE_HELPER:-/usr/local/sbin/awoooi-edge-maintenance}" SSH_OPTS=( -o BatchMode=yes @@ -27,9 +28,10 @@ Usage: public-maintenance-edge-fallback-apply.sh --check public-maintenance-edge-fallback-apply.sh --apply public-maintenance-edge-fallback-apply.sh --rollback + public-maintenance-edge-fallback-apply.sh --rollback-latest -No secret collection is allowed. --apply requires an existing passwordless sudo -channel on the edge host; otherwise it exits fail-closed before writing. +No secret collection is allowed. --apply uses only the root-owned fixed-action +edge helper; otherwise it exits fail-closed before writing. USAGE } @@ -46,6 +48,10 @@ while [ "$#" -gt 0 ]; do shift ROLLBACK_CONF="${1:-}" ;; + --rollback-latest) + MODE="rollback" + ROLLBACK_CONF="latest" + ;; -h|--help) usage exit 0 @@ -69,8 +75,11 @@ if [ ! -f "$LOCAL_MAINT_FILE" ]; then fi remote_check() { + local expected_conf_sha expected_maint_sha + expected_conf_sha="$(sha256sum "$LOCAL_CONF" | awk '{print $1}')" + expected_maint_sha="$(sha256sum "$LOCAL_MAINT_FILE" | awk '{print $1}')" ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ - "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' bash -s" <<'REMOTE' + "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_HELPER='$REMOTE_HELPER' EXPECTED_CONF_SHA='$expected_conf_sha' EXPECTED_MAINT_SHA='$expected_maint_sha' bash -s" <<'REMOTE' set -eu echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" echo "MODE=check" @@ -104,8 +113,25 @@ else echo "MAINTENANCE_FILE_PRESENT=0" echo "MAINTENANCE_FILE_HAS_MARKER=0" fi -SUDO_N_READY=$(if sudo -n true >/dev/null 2>&1; then echo 1; else echo 0; fi) -echo "SUDO_N_READY=${SUDO_N_READY}" +PRIVILEGED_EXECUTOR_READY=0 +EDGE_EXECUTOR_PAYLOAD_PARITY=0 +CANONICAL_CONF_SHA256="" +CANONICAL_MAINT_SHA256="" +helper_status="$(sudo -n "$REMOTE_HELPER" check 2>/dev/null || true)" +if printf '%s\n' "$helper_status" | grep -q '^EDGE_EXECUTOR_READY=1$'; then + PRIVILEGED_EXECUTOR_READY=1 + CANONICAL_CONF_SHA256="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_CONF_SHA256" {print $2; exit}')" + CANONICAL_MAINT_SHA256="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_MAINT_SHA256" {print $2; exit}')" + if [ "$CANONICAL_CONF_SHA256" = "$EXPECTED_CONF_SHA" ] \ + && [ "$CANONICAL_MAINT_SHA256" = "$EXPECTED_MAINT_SHA" ]; then + EDGE_EXECUTOR_PAYLOAD_PARITY=1 + fi +fi +echo "PRIVILEGED_EXECUTOR_READY=${PRIVILEGED_EXECUTOR_READY}" +echo "SUDO_N_READY=${PRIVILEGED_EXECUTOR_READY}" +echo "EDGE_EXECUTOR_PAYLOAD_PARITY=${EDGE_EXECUTOR_PAYLOAD_PARITY}" +echo "CANONICAL_CONF_SHA256=${CANONICAL_CONF_SHA256}" +echo "CANONICAL_MAINT_SHA256=${CANONICAL_MAINT_SHA256}" REMOTE_CONF_READABLE=$(if [ -r "$REMOTE_CONF" ]; then echo 1; else echo 0; fi) LIVE_CONF_HAS_INTERCEPT=$(if [ -r "$REMOTE_CONF" ] && grep -q 'proxy_intercept_errors on;' "$REMOTE_CONF"; then echo 1; else echo 0; fi) LIVE_CONF_HAS_ERROR_PAGE_503=$(if [ -r "$REMOTE_CONF" ] && grep -q 'error_page 502 503 504 =503 /__awoooi-maintenance.html;' "$REMOTE_CONF"; then echo 1; else echo 0; fi) @@ -128,53 +154,37 @@ if [ "$REMOTE_CONF_READABLE" = 1 ] \ else echo "EDGE_FALLBACK_READY=0" echo "BLOCKER edge_public_maintenance_live_config_drift" - if [ "$SUDO_N_READY" = 0 ]; then + if [ "$PRIVILEGED_EXECUTOR_READY" = 0 ]; then echo "BLOCKER privileged_edge_apply_channel_unavailable" + elif [ "$EDGE_EXECUTOR_PAYLOAD_PARITY" = 0 ]; then + echo "BLOCKER edge_executor_payload_drift" fi fi REMOTE } -remote_apply() { - local tmp_conf="/tmp/awoooi.wooo.work.conf.$$" - local tmp_maint="/tmp/awoooi-maintenance.$$" - scp -q "${SSH_OPTS[@]}" "$LOCAL_CONF" "${EDGE_TARGET}:${tmp_conf}" - scp -q "${SSH_OPTS[@]}" "$LOCAL_MAINT_FILE" "${EDGE_TARGET}:${tmp_maint}" - ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ - "REMOTE_CONF='$REMOTE_CONF' REMOTE_MAINT_DIR='$REMOTE_MAINT_DIR' REMOTE_MAINT_FILE='$REMOTE_MAINT_FILE' REMOTE_BACKUP_DIR='$REMOTE_BACKUP_DIR' TMP_CONF='$tmp_conf' TMP_MAINT='$tmp_maint' bash -s" <<'REMOTE' -set -eu -echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" -echo "MODE=apply" -if ! sudo -n true >/dev/null 2>&1; then - echo "SUDO_N_READY=0" - echo "BLOCKER privileged_edge_apply_channel_unavailable" - rm -f "$TMP_CONF" "$TMP_MAINT" - exit 75 -fi -echo "SUDO_N_READY=1" -mkdir -p "$REMOTE_BACKUP_DIR" -backup="${REMOTE_BACKUP_DIR}/awoooi.wooo.work.conf.before-maintenance.$(date +%Y%m%d%H%M%S)" -if [ -f "$REMOTE_CONF" ]; then - sudo cp "$REMOTE_CONF" "$backup" -else - : >"${backup}.missing" -fi -sudo mkdir -p "$REMOTE_MAINT_DIR" -sudo install -m 0644 "$TMP_CONF" "$REMOTE_CONF" -sudo install -m 0644 "$TMP_MAINT" "$REMOTE_MAINT_FILE" -rm -f "$TMP_CONF" "$TMP_MAINT" -if ! sudo nginx -t; then - echo "BLOCKER nginx_test_failed_restoring_backup" - if [ -f "$backup" ]; then - sudo cp "$backup" "$REMOTE_CONF" - sudo nginx -t || true +require_executor_payload_parity() { + local expected_conf_sha expected_maint_sha helper_status remote_conf_sha remote_maint_sha + expected_conf_sha="$(sha256sum "$LOCAL_CONF" | awk '{print $1}')" + expected_maint_sha="$(sha256sum "$LOCAL_MAINT_FILE" | awk '{print $1}')" + if ! helper_status="$(ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' check")"; then + echo "BLOCKER privileged_edge_apply_channel_unavailable" + return 75 fi - exit 76 -fi -sudo systemctl reload nginx -echo "BACKUP_CONF=$backup" -echo "NGINX_RELOAD_DONE=1" -REMOTE + remote_conf_sha="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_CONF_SHA256" {print $2; exit}')" + remote_maint_sha="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "CANONICAL_MAINT_SHA256" {print $2; exit}')" + if [ "$remote_conf_sha" != "$expected_conf_sha" ] \ + || [ "$remote_maint_sha" != "$expected_maint_sha" ]; then + echo "BLOCKER edge_executor_payload_drift" + return 78 + fi +} + +remote_apply() { + require_executor_payload_parity + echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" + echo "MODE=apply" + ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' apply" remote_check } @@ -183,25 +193,19 @@ remote_rollback() { echo "BLOCKER rollback_conf_required" exit 64 fi - ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" \ - "REMOTE_CONF='$REMOTE_CONF' ROLLBACK_CONF='$ROLLBACK_CONF' bash -s" <<'REMOTE' -set -eu -echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" -echo "MODE=rollback" -if ! sudo -n true >/dev/null 2>&1; then - echo "SUDO_N_READY=0" - echo "BLOCKER privileged_edge_apply_channel_unavailable" - exit 75 -fi -if [ ! -f "$ROLLBACK_CONF" ]; then - echo "BLOCKER rollback_conf_not_found path=$ROLLBACK_CONF" - exit 66 -fi -sudo cp "$ROLLBACK_CONF" "$REMOTE_CONF" -sudo nginx -t -sudo systemctl reload nginx -echo "ROLLBACK_DONE=1" -REMOTE + require_executor_payload_parity + local helper_status last_backup + helper_status="$(ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' check")" + last_backup="$(printf '%s\n' "$helper_status" | awk -F= '$1 == "LAST_BACKUP_DIR" {print $2; exit}')" + if [ "$ROLLBACK_CONF" != "latest" ] \ + && [ "$ROLLBACK_CONF" != "$last_backup" ] \ + && [ "$ROLLBACK_CONF" != "$last_backup/live.conf" ]; then + echo "BLOCKER rollback_conf_not_latest" + exit 66 + fi + echo "AWOOOI_PUBLIC_MAINTENANCE_EDGE_FALLBACK=1" + echo "MODE=rollback" + ssh "${SSH_OPTS[@]}" "$EDGE_TARGET" "sudo -n '$REMOTE_HELPER' rollback-latest" remote_check } diff --git a/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh b/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh index 08ad26936..9b65a9dae 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh @@ -17,10 +17,11 @@ SSH_OPTS=( SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-15}" NODE_EXPORTER_PORT="${NODE_EXPORTER_PORT:-9100}" NODE_EXPORTER_TIMEOUT_SECONDS="${NODE_EXPORTER_TIMEOUT_SECONDS:-4}" +WINDOWS99_SSH_TARGET="${WINDOWS99_SSH_TARGET:-Administrator@192.168.0.99}" HOST_SPECS=( "99=192.168.0.99:vmware-host-autostart" "110=wooo@192.168.0.110:awoooi-startup-110.service" - "111=192.168.0.111:vm-host-boot" + "111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy" "112=192.168.0.112:vm-host-boot" "120=wooo@192.168.0.120:k3s.service" "121=wooo@192.168.0.121:k3s.service" @@ -164,6 +165,45 @@ probe_reachable_only() { return 1 } +probe_windows_ssh() { + local alias="$1" + local target="$2" + local unit="$3" + local powershell_payload encoded_payload remote_command output + local boot_id uptime_seconds systemd_state enabled active + + command -v iconv >/dev/null 2>&1 || return 1 + command -v base64 >/dev/null 2>&1 || return 1 + + powershell_payload='$ErrorActionPreference = "Stop" +$boot = (Get-CimInstance Win32_OperatingSystem).LastBootUpTime +$bootEpoch = ([DateTimeOffset]$boot).ToUnixTimeSeconds() +$uptimeSeconds = [math]::Floor(((Get-Date) - $boot).TotalSeconds) +Write-Output ("boot_id=windows_{0} uptime_seconds={1} systemd_state=windows_ssh startup_enabled=unknown startup_active=unknown" -f $bootEpoch, $uptimeSeconds)' + encoded_payload="$( + printf '%s' "$powershell_payload" \ + | iconv -f UTF-8 -t UTF-16LE \ + | base64 \ + | tr -d '\r\n' + )" + [[ -n "$encoded_payload" ]] || return 1 + remote_command="powershell.exe -NoProfile -NonInteractive -EncodedCommand $encoded_payload" + output="$( + run_with_timeout "$SSH_COMMAND_TIMEOUT_SECONDS" \ + ssh "${SSH_OPTS[@]}" "$WINDOWS99_SSH_TARGET" "$remote_command" 2>/dev/null + )" || return 1 + + boot_id="$(sed -n 's/.*boot_id=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + uptime_seconds="$(sed -n 's/.*uptime_seconds=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + systemd_state="$(sed -n 's/.*systemd_state=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + enabled="$(sed -n 's/.*startup_enabled=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + active="$(sed -n 's/.*startup_active=\([^ ]*\).*/\1/p' <<<"$output" | tail -n 1)" + [[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]] || return 1 + + emit_boot_row "$alias" "$target" "$unit" 1 "$boot_id" "$uptime_seconds" \ + "${systemd_state:-windows_ssh}" "${enabled:-unknown}" "${active:-unknown}" +} + probe_host() { local alias="$1" local target="$2" @@ -177,6 +217,10 @@ probe_host() { return 0 fi + if [[ "$alias" == "99" ]] && probe_windows_ssh "$alias" "$target" "$unit"; then + return 0 + fi + if [[ "$target" != *@* ]]; then if probe_node_exporter "$alias" "$target" "$unit"; then return 0 diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 8caafc5ec..f3c87c0ef 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -21,7 +21,9 @@ POST_REBOOT_READINESS_TIMEOUT_SECONDS="${POST_REBOOT_READINESS_TIMEOUT_SECONDS:- PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS="${PUBLIC_MAINTENANCE_READBACK_TIMEOUT_SECONDS:-8}" PUBLIC_MAINTENANCE_URLS="${PUBLIC_MAINTENANCE_URLS:-https://awoooi.wooo.work/api/v1/health https://awoooi.wooo.work/ https://stock.wooo.work/api/v1/system/freshness https://mo.wooo.work/health https://bitan.wooo.work/ https://www.tsenyang.com/}" WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS="${WINDOWS99_VMWARE_FALLBACK_MAX_AGE_SECONDS:-900}" -WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-111 188 120 121 112}" +# Host 111 is a physical MacBook/Ollama node. It remains in whole-host probes, +# while Windows 99 owns VMware guests 110/188/120/121/112. +WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}" POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS="${POST_REBOOT_READINESS_FALLBACK_MAX_AGE_SECONDS:-900}" AI_LOG_TRIAGE_AUTO_RECOVERY_MODE="${AI_LOG_TRIAGE_AUTO_RECOVERY_MODE:-check}" AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS="${AI_LOG_TRIAGE_AUTO_RECOVERY_TIMEOUT_SECONDS:-420}" @@ -365,8 +367,9 @@ if [ -f "$ROOT_DIR/scripts/reboot-recovery/windows99-management-channel-probe.py || rm -f "$windows99_management_file" fi if [ -x "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" ]; then - WINDOWS99_MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-5}" \ - WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}" \ + WINDOWS99_SSH_USERS="${WINDOWS99_SSH_USERS:-Administrator}" \ + WINDOWS99_MAX_AUTH_USERS="${WINDOWS99_MAX_AUTH_USERS:-1}" \ + WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}" \ bash "$ROOT_DIR/scripts/reboot-recovery/collect-windows99-vmware-verify.sh" --collect \ >"$windows99_vmware_file" 2>"$artifact_dir/windows99-vmware-verify.err" \ || true diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index ef4a2cd8c..20f1f936f 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -16,7 +16,9 @@ from typing import Any ROOT = Path(__file__).resolve().parents[2] SCHEMA_VERSION = "awoooi_reboot_auto_recovery_slo_scorecard_v1" REQUIRED_HOSTS = {"99", "110", "111", "112", "120", "121", "188"} -WINDOWS99_REQUIRED_VM_ALIASES = {"111", "112", "120", "121", "188"} +# Host 111 is a physical MacBook/Ollama node and remains a required host probe. +# Windows 99 owns these five VMware guests. +WINDOWS99_REQUIRED_VM_ALIASES = {"110", "112", "120", "121", "188"} def parse_args() -> argparse.Namespace: @@ -810,7 +812,7 @@ def source_controls() -> dict[str, bool]: "slo_systemd_timer_source_present": file_contains( source_file("scripts/reboot-recovery/awoooi-reboot-auto-recovery-slo.timer"), "OnBootSec=", - "OnUnitActiveSec=", + "OnUnitInactiveSec=", ), "slo_installer_source_present": file_contains( source_file("scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh"), diff --git a/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py b/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py index 5722bbad6..ee6737121 100644 --- a/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py +++ b/scripts/reboot-recovery/tests/test_188_host_hygiene_checklist.py @@ -8,6 +8,8 @@ SCRIPT = ROOT / "scripts" / "reboot-recovery" / "188-host-hygiene-maintenance-ch STARTUP_188 = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.sh" STARTUP_188_SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-startup.service" DEPLOY_188 = ROOT / "scripts" / "reboot-recovery" / "deploy-to-188.sh" +HYGIENE_HELPER = ROOT / "ops" / "reboot-recovery" / "awoooi-host188-hygiene-helper.sh" +HYGIENE_BOOTSTRAP = ROOT / "scripts" / "reboot-recovery" / "bootstrap-host188-hygiene-executor.sh" def test_188_and_110_default_to_reachable_runtime_identities() -> None: @@ -46,6 +48,20 @@ def test_188_startup_keeps_clawbot_rebuild_bounded_and_opt_in() -> None: assert "post-reboot SLO" in text +def test_188_startup_defaults_data_and_metadata_repairs_to_fail_closed() -> None: + text = STARTUP_188.read_text(encoding="utf-8") + + assert 'CONTAINERD_METADATA_REPAIR_ALLOWED="${CONTAINERD_METADATA_REPAIR_ALLOWED:-0}"' in text + assert 'DOCKER_NETWORK_METADATA_REPAIR_ALLOWED="${DOCKER_NETWORK_METADATA_REPAIR_ALLOWED:-0}"' in text + assert 'KINE_MAINTENANCE_ALLOWED="${KINE_MAINTENANCE_ALLOWED:-0}"' in text + assert 'SIGNOZ_188_START_ALLOWED="${SIGNOZ_188_START_ALLOWED:-0}"' in text + assert 'if [ "$CONTAINERD_METADATA_REPAIR_ALLOWED" != "1" ]; then' in text + assert 'if [ "$DOCKER_NETWORK_METADATA_REPAIR_ALLOWED" != "1" ]; then' in text + assert 'if [ "$KINE_MAINTENANCE_ALLOWED" = "1" ]; then' in text + assert 'if [ "$SIGNOZ_188_START_ALLOWED" != "1" ]; then' in text + assert "docker network inspect aiops-network" in text + + def test_188_startup_unit_timeout_matches_reboot_slo() -> None: text = STARTUP_188_SERVICE.read_text(encoding="utf-8") @@ -60,3 +76,24 @@ def test_188_deploy_helper_does_not_embed_sudo_password() -> None: assert "PASSWORD" not in text assert "sudo -n true" in text assert "BLOCKED sudo_password_required_passwordless_sudo_or_tty_required" in text + + +def test_188_hygiene_executor_is_fixed_action_and_does_not_restart_products() -> None: + helper = HYGIENE_HELPER.read_text(encoding="utf-8") + bootstrap = HYGIENE_BOOTSTRAP.read_text(encoding="utf-8") + + assert "check|reconcile|rollback-latest" not in helper + assert "reconcile)" in helper + assert "rollback-latest)" in helper + assert "POSTGRES_START_CONF" in helper + assert "manual" in helper + assert "systemctl daemon-reload" in helper + assert "systemctl reset-failed postgresql@14-main.service awoooi-startup.service" in helper + assert "PRODUCT_SERVICE_RESTARTS=0" in helper + assert "systemctl restart" not in helper + assert "systemctl start" not in helper + assert "--network=none" in bootstrap + assert "--userns=host" in bootstrap + assert "--pull=never" in bootstrap + assert "alpine@sha256:" in bootstrap + assert "visudo -cf" in bootstrap diff --git a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py index 95cb2f235..8e866105f 100644 --- a/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py +++ b/scripts/reboot-recovery/tests/test_cold_start_monitor_bounded_probes.py @@ -266,6 +266,26 @@ def test_runner_failclosed_enforcer_preserves_controlled_drain_staging_or_guarde assert "CONTROLLED_DRAIN_PRESERVE_ALLOWED=" in text +def test_runner_failclosed_authority_is_singleton_and_idempotent() -> None: + text = FAILCLOSED_ENFORCER.read_text(encoding="utf-8") + + assert "--ensure Check first and apply only when fail-closed state has drifted." in text + assert "failclosed_state_ok()" in text + assert 'elif [ "$MODE" = "ensure" ] && ! failclosed_state_ok; then' in text + assert ( + "* * * * * root /usr/bin/flock -n " + "/run/lock/awoooi-runner-failclosed-authority.lock " + "/usr/bin/timeout 180 " + "/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh " + "--ensure" + ) in text + assert ( + "* * * * * root " + "/usr/local/lib/awoooi/enforce-110-runner-failclosed.authority.sh " + "--apply" + ) not in text + + def test_controlled_cd_lane_unit_source_has_required_accounting_guardrails() -> None: text = CONTROLLED_CD_LANE_DRAIN_UNIT.read_text(encoding="utf-8") diff --git a/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py b/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py index c2f148b0b..aa1536df6 100644 --- a/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py +++ b/scripts/reboot-recovery/tests/test_post_reboot_readiness_summary_contract.py @@ -24,3 +24,8 @@ def test_unknown_backup_escrow_evidence_stays_blocked() -> None: '[[ "$dr_escrow_blocked" == "1" ]] ' '&& next_required_gates+=("credential_escrow_evidence")' ) in text + assert 'POST_START_QUICK_CHECK_TIMEOUT_SECONDS="${POST_START_QUICK_CHECK_TIMEOUT_SECONDS:-90}"' in text + assert 'run_with_timeout "$POST_START_QUICK_CHECK_TIMEOUT_SECONDS"' in text + assert "POST_START_QUICK_CHECK_TIMEOUT=1" in text + assert "POST_START_TIMED_OUT=$post_start_timed_out" in text + assert 'next_required_gates+=("post_start_quick_check_timeout_recovery")' in text diff --git a/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py b/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py index a260c516b..71763c7bf 100644 --- a/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py +++ b/scripts/reboot-recovery/tests/test_post_start_quick_check_contract.py @@ -15,12 +15,15 @@ def test_post_start_self_checks_can_run_locally_on_110() -> None: assert 'local_ip_list | grep' not in text assert 'bash -lc "$command"' in text assert 'SSH_COMMAND_TIMEOUT_SECONDS="${SSH_COMMAND_TIMEOUT_SECONDS:-25}"' in text + assert 'SSH_TOTAL_TIMEOUT_SECONDS="${SSH_TOTAL_TIMEOUT_SECONDS:-$((SSH_CONNECT_TIMEOUT + SSH_COMMAND_TIMEOUT_SECONDS + 5))}"' in text assert "printf -v quoted_command '%q' \"$command\"" in text assert '-o ConnectionAttempts=1' in text assert '-o ServerAliveInterval=5' in text assert '-o ServerAliveCountMax=1' in text assert '-o NumberOfPasswordPrompts=0' in text assert "timeout ${SSH_COMMAND_TIMEOUT_SECONDS}s bash -lc ${quoted_command}" in text + assert 'run_with_timeout "$SSH_TOTAL_TIMEOUT_SECONDS" "${ssh_command[@]}"' in text + assert 'timeout --foreground --kill-after=3s "${timeout_seconds}s"' in text def test_runner_pressure_is_capacity_evidence_after_routes_recover() -> None: diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py index 2e067f63a..96b6a7a84 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_installer.py @@ -8,6 +8,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[3] INSTALLER = ROOT / "scripts" / "reboot-recovery" / "install-reboot-auto-recovery-slo-110.sh" SERVICE = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.service" +TIMER = ROOT / "scripts" / "reboot-recovery" / "awoooi-reboot-auto-recovery-slo.timer" EXPORTER = ROOT / "scripts" / "reboot-recovery" / "reboot-auto-recovery-slo-exporter.sh" @@ -26,6 +27,7 @@ def test_installer_dry_run_exposes_apply_rollback_and_verify_contract() -> None: assert "would_include_source_gates=" in result.stdout assert "awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in result.stdout assert "ops/nginx" in result.stdout + assert "ops/reboot-recovery" in result.stdout assert "would_enable_timer=awoooi-reboot-auto-recovery-slo.timer" in result.stdout @@ -60,6 +62,7 @@ def test_installer_is_limited_to_verifier_timer_not_product_restarts() -> None: assert 'payload="${payload_dir}/payload.tar.gz"' in text assert "apps/api/migrations/awooop_conversation_event_hot_path_indexes_2026-07-01.sql" in text assert "ops/nginx" in text + assert "ops/reboot-recovery" in text assert "ops/maintenance" in text assert "ops/monitoring" in text assert "PUBLIC-MAINTENANCE-FALLBACK-RUNBOOK.md" in text @@ -83,6 +86,14 @@ def test_service_uses_deployed_source_root_and_bounded_oneshot() -> None: assert "TimeoutStartSec=600" in text +def test_timer_waits_after_completion_instead_of_running_back_to_back() -> None: + text = TIMER.read_text(encoding="utf-8") + + assert "OnBootSec=2min" in text + assert "OnUnitInactiveSec=5min" in text + assert "OnUnitActiveSec" not in text + + def test_exporter_uses_user_writable_lock_after_creating_log_dir() -> None: text = EXPORTER.read_text(encoding="utf-8") @@ -108,9 +119,10 @@ def test_exporter_projects_each_scorecard_blocker_to_textfile_metric() -> None: assert "windows99-vmware-verify.txt" in text assert "collect-windows99-vmware-verify.sh\" --collect" in text - assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-5}\"" in text - assert "WINDOWS99_REMOTE_VERIFY_TIMEOUT=\"${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}\"" in text - assert "WINDOWS99_REQUIRED_VM_ALIASES=\"${WINDOWS99_REQUIRED_VM_ALIASES:-111 188 120 121 112}\"" in text + assert "WINDOWS99_SSH_USERS=\"${WINDOWS99_SSH_USERS:-Administrator}\"" in text + assert "WINDOWS99_MAX_AUTH_USERS=\"${WINDOWS99_MAX_AUTH_USERS:-1}\"" in text + assert "WINDOWS99_REMOTE_VERIFY_TIMEOUT=\"${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}\"" in text + assert "WINDOWS99_REQUIRED_VM_ALIASES=\"${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}\"" in text assert "windows99_vmware_verify_alias_set_matches_current" in text assert "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1" in text assert "scorecard_args+=(--windows99-vmware-file" in text diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index b2f5d93d1..002d7cf93 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -77,8 +77,26 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> for host in ["99", "110", "111", "112", "120", "121", "188"]: assert host in host_probe assert 'run_with_timeout "${TCP_CONNECT_TIMEOUT_SECONDS:-2}"' in host_probe + assert 'WINDOWS99_SSH_TARGET="${WINDOWS99_SSH_TARGET:-Administrator@192.168.0.99}"' in host_probe + assert "probe_windows_ssh()" in host_probe + assert "Get-CimInstance Win32_OperatingSystem" in host_probe + assert "systemd_state=windows_ssh" in host_probe + assert '[[ "$boot_id" == windows_* && "$uptime_seconds" =~ ^[0-9]+$ ]]' in host_probe + assert '"111=ooo@192.168.0.111:com.momo.ollama111-allow-proxy"' in host_probe + + macos_readback = read("scripts/reboot-recovery/macos-host-boot-readback.sh") + macos_installer = read("scripts/reboot-recovery/install-macos111-host-boot-readback.sh") + assert "sysctl -n kern.boottime" in macos_readback + assert "s/^\\{ sec = ([0-9]+),.*/\\1/" in macos_readback + assert "boot_id=darwin_%s" in macos_readback + assert "127.0.0.1:11434/api/tags" in macos_readback + assert "restrict,command=" in macos_installer + assert "IdentitiesOnly=yes" in macos_installer + assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer + assert "authorized_keys" in macos_installer assert "AWOOOI-Start-VMware-VMs" in windows99 assert "NoAutoRebootWithLoggedOnUsers" in windows99 + assert 'New-ItemProperty -Path $auPath -Name "AUPowerManagement" -Value 0' in windows99 assert "Host110Vmx" in windows99 assert "Host111Vmx" in windows99 assert "Host188Vmx" in windows99 @@ -88,10 +106,24 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "EnableRecursiveDiscovery" in windows99 assert "D:\\Documents\\Virtual Machines" in windows99 assert "D:\\Downloads" in windows99 - assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' in windows99 + assert '[string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112")' in windows99 + assert '[string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112")' not in windows99 + assert '$SuppliedVmx["111"] -or $VmOrder -contains "111"' in windows99 + + agent99_update_policy = read("agent99-windows-update-policy.ps1") + assert 'New-ItemProperty -Path $auPolicy -Name "AUPowerManagement" -Value 0' in agent99_update_policy assert "HOST111_VMWARE_TARGET=explicit_or_required" in windows99 assert "VMWARE_AUTOSTART_VERIFY_READY" in windows99 assert "WINDOWS_UPDATE_POLICY" in windows99 + + exporter = read("scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh") + assert 'WINDOWS99_REQUIRED_VM_ALIASES="${WINDOWS99_REQUIRED_VM_ALIASES:-110 188 120 121 112}"' in exporter + + scorecard = read("scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py") + assert 'WINDOWS99_REQUIRED_VM_ALIASES = {"110", "112", "120", "121", "188"}' in scorecard + + legacy_installer = read("scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1") + assert '[string[]]$VmNameAllowList = @("110", "188", "120", "121", "112")' in legacy_installer assert "VM_POWER alias=$alias" in windows99 assert "Get-VmwareVmxProcessCommandLines" in windows99 assert "Get-VmxRunningSource" in windows99 @@ -186,7 +218,7 @@ def test_reboot_p0_contract_has_ai_log_triage_controlled_apply() -> None: assert "HARBOR_REGISTRY_NOT_READY" in runner assert "registry_public_ready" in runner assert "registry_110_readback_missing" in runner - assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-180}"' in runner + assert 'WINDOWS99_REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"' in runner assert "WINDOWS99_REMOTE_VERIFY_WRAPPER_TIMEOUT_SECONDS" in runner assert "https://registry\\.wooo\\.work/v2/" in runner assert "STOCK_UPSTREAM_NOT_READY" in runner @@ -258,6 +290,7 @@ def test_reboot_p0_contract_applies_l0_fallback_to_awoooi_nginx_sources() -> Non assert "192.168.0.125:32334" in ops_conf assert "192.168.0.125:32335" in ops_conf for marker in [ + "location /api/sentry-tunnel {", "location /api/ {", "location ~ ^/api/v1/(dashboard/stream|agent/thinking) {", "location /api/v1/ws {", @@ -306,17 +339,32 @@ def test_public_maintenance_probe_does_not_count_fallback_5xx_as_raw( def test_public_maintenance_edge_apply_script_is_guarded_and_reversible() -> None: script = read("scripts/reboot-recovery/public-maintenance-edge-fallback-apply.sh") + helper = read("ops/reboot-recovery/awoooi-edge-maintenance-helper.sh") + bootstrap = read("scripts/reboot-recovery/bootstrap-edge-maintenance-executor.sh") assert "MODE=\"check\"" in script assert "MODE=\"apply\"" in script assert "MODE=\"rollback\"" in script - assert "sudo -n true" in script + assert "sudo -n '$REMOTE_HELPER' check" in script + assert "sudo -n '$REMOTE_HELPER' apply" in script + assert "sudo -n '$REMOTE_HELPER' rollback-latest" in script + assert "edge_executor_payload_drift" in script assert "BLOCKER privileged_edge_apply_channel_unavailable" in script - assert "nginx -t" in script - assert "systemctl reload nginx" in script - assert "BACKUP_CONF=" in script assert "EDGE_FALLBACK_READY=1" in script assert "EDGE_FALLBACK_READY=0" in script + assert "nginx -t" in helper + assert "systemctl reload nginx" in helper + assert "BACKUP_CONF=" in helper + assert "rollback-latest" in helper + assert "canonical_payload_not_root_owned" in helper + assert "canonical_payload_is_mutable" in helper + assert "grep -Ev '^[[:space:]]*#'" in helper + assert "docker run -i" in bootstrap + assert "--network=none" in bootstrap + assert "--userns=host" in bootstrap + assert "--pull=never" in bootstrap + assert "alpine@sha256:" in bootstrap + assert "visudo -cf" in bootstrap assert "raw_secret" not in script.lower() assert "docker restart" not in script assert "reboot " not in script diff --git a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py index 8e59ba504..2d3773858 100644 --- a/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py +++ b/scripts/reboot-recovery/tests/test_windows99_console_verify_artifact_validator.py @@ -18,8 +18,8 @@ VALID_RED_VERIFY = """\ AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1 MODE=Verify VMRUN_PRESENT=1 -HOST111_VMWARE_TARGET=explicit_or_required -VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1 +HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99 +VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1 VMX alias=188 path=D:\\Documents\\Virtual Machines\\188\\188.vmx present=1 VMX alias=120 path=D:\\Documents\\Virtual Machines\\120\\120.vmx present=1 VMX alias=121 path=D:\\Documents\\Virtual Machines\\121\\121.vmx present=1 @@ -28,7 +28,7 @@ VMWARE_SERVICE name=VMAuthdService present=1 status=Running startup_type=Auto ok VMWARE_SERVICE name=VMnetDHCP present=1 status=Running startup_type=Auto ok=1 VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs present=1 enabled=1 state=Ready trigger_count=1 action_count=1 ok=1 WINDOWS_UPDATE_POLICY name=NoAutoRebootWithLoggedOnUsers value=1 expected=1 ok=1 -VM_POWER alias=111 vmx_present=1 running=0 +VM_POWER alias=110 vmx_present=1 running=0 VM_POWER alias=188 vmx_present=1 running=0 VM_POWER alias=120 vmx_present=1 running=0 VM_POWER alias=121 vmx_present=1 running=0 @@ -94,7 +94,8 @@ def test_valid_console_verify_red_artifact_is_accepted_and_normalized( assert values["status"] == "valid_console_verify_artifact_vmware_not_ready" assert values["missing_required_field_count"] == "0" assert values["missing_vmx_aliases"] == "" - assert values["powered_off_aliases"] == "111,188,120,121,112" + assert values["required_aliases"] == "110,188,120,121,112" + assert values["powered_off_aliases"] == "110,188,120,121,112" assert values["secret_value_read"] == "false" assert normalized.read_text(encoding="utf-8").startswith( "AWOOOI_WINDOWS99_VMWARE_AUTOSTART=1\n" @@ -152,11 +153,17 @@ def test_explicit_host111_artifact_can_be_validated_with_required_alias_override raw = tmp_path / "console-111.txt" raw.write_text( VALID_RED_VERIFY.replace( - "HOST111_VMWARE_TARGET=explicit_or_required", + "HOST111_VMWARE_TARGET=not_required_by_default_missing_no_vmx_found_on_99", "HOST111_VMWARE_TARGET=explicit_or_required", ) - .replace("VMX alias=111 path=D:\\Documents\\Virtual Machines\\111\\111.vmx present=1", "VMX alias=111 path= present=0") - .replace("VM_POWER alias=111 vmx_present=1 running=0", "VM_POWER alias=111 vmx_present=0 running=0"), + .replace( + "VMX alias=110 path=D:\\Documents\\Virtual Machines\\110\\110.vmx present=1", + "VMX alias=111 path= present=0", + ) + .replace( + "VM_POWER alias=110 vmx_present=1 running=0", + "VM_POWER alias=111 vmx_present=0 running=0", + ), encoding="utf-8", ) diff --git a/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py b/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py index 60b25e995..9302cd169 100644 --- a/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py +++ b/scripts/reboot-recovery/tests/test_windows99_vmware_verify_collector.py @@ -49,6 +49,12 @@ def test_collector_contract_forbids_secret_and_runtime_actions() -> None: assert "KbdInteractiveAuthentication=no" in text assert "NumberOfPasswordPrompts=0" in text assert "ConnectionAttempts=1" in text + assert "ServerAliveInterval=5" in text + assert "ServerAliveCountMax=2" in text + assert "StrictHostKeyChecking=accept-new" in text + assert "StrictHostKeyChecking=no" not in text + assert 'REMOTE_VERIFY_TIMEOUT="${WINDOWS99_REMOTE_VERIFY_TIMEOUT:-45}"' in text + assert "SSH_USERS=(Administrator ogt wooo ooo administrator)" in text assert "GSSAPIAuthentication=no" in text assert "--via-host" in text assert "via_host=" in text diff --git a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py index 2c379dede..227dff0dd 100755 --- a/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py +++ b/scripts/reboot-recovery/validate-windows99-console-verify-artifact.py @@ -9,7 +9,7 @@ from pathlib import Path SCHEMA_VERSION = "windows99_console_verify_artifact_validator_v1" -DEFAULT_REQUIRED_ALIASES = ("111", "188", "120", "121", "112") +DEFAULT_REQUIRED_ALIASES = ("110", "188", "120", "121", "112") SENSITIVE_RE = re.compile( r"(?i)(password|passwd|secret|token|authorization|cookie|private[_ -]?key)\s*[=:]" ) diff --git a/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 index aef865ec0..af0a4d12d 100644 --- a/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows-99/configure-vmware-autostart.ps1 @@ -1,5 +1,5 @@ param( - [string[]]$VmNameAllowList = @("111", "188", "120", "121", "112"), + [string[]]$VmNameAllowList = @("110", "188", "120", "121", "112"), [string[]]$SearchRoots = @("C:\VMs", "D:\VMs", "$env:USERPROFILE\Documents\Virtual Machines"), [string]$TaskName = "AWOOOI VMware VM AutoStart", [switch]$WhatIfOnly diff --git a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 index d842551d8..b132b497d 100644 --- a/scripts/reboot-recovery/windows99-vmware-autostart.ps1 +++ b/scripts/reboot-recovery/windows99-vmware-autostart.ps1 @@ -8,7 +8,8 @@ param( [string]$Host120Vmx = "", [string]$Host121Vmx = "", [string]$Host112Vmx = "", - [string[]]$RequiredVmAliases = @("111", "188", "120", "121", "112"), + # Host 111 is a physical MacBook/Ollama node; keep it in host probes, not VMware guest inventory. + [string[]]$RequiredVmAliases = @("110", "188", "120", "121", "112"), [switch]$EnableRecursiveDiscovery, [string[]]$DiscoveryRoot = @( "D:\Documents\Virtual Machines",