feat(recovery): add reboot auto recovery slo guard

docs/LOGBOOK.md

@@ -49409,3 +49409,26 @@ production browser smoke:
 
 **未做**：
 - 沒有使用 GitHub / gh / GitHub API；沒有讀 token / secret / `.env` / raw sessions / SQLite / auth；沒有寫 Gitea repo / refs / branch / secret；沒有 host 或 runtime 操作。
+
+## 2026-06-29 — 14:25 P0-006A reboot auto-recovery 10-minute SLO control plane
+
+**完成內容**：
+- 釐清正確 P0-006 目標：不是手動 cold-start readback，而是「所有 P0 主機重啟被自動判斷、自動觸發恢復/驗證，並在 10 分鐘內自證所有服務恢復」。
+- 清理本輪 Codex 產生的 `/tmp` 大型暫存與 generated cache，將本機 Data volume 可用空間從約 `137Mi` 拉回約 `3.3Gi`；這是 reboot SLO 的必要前置，因為先前連 `git fetch` 都因 `No space left on device` 失敗。
+- 新增 `scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh`：只讀探測 110 / 120 / 121 / 188 的 `boot_id`、uptime、systemd state 與 startup unit 狀態。
+- 新增 `scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh`、`awoooi-reboot-auto-recovery-slo.service`、`awoooi-reboot-auto-recovery-slo.timer`：提供 boot-triggered SLO verifier 與 Prometheus textfile metrics source。
+- 新增 `scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py` 與 snapshot `docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json`；目前 fail-closed：`can_claim_all_services_recovered_within_target=false`。
+
+**目前真 blocker**：
+- `all_host_reboot_detection_missing`
+- `host_boot_probe_missing_hosts`
+- `wazuh_dashboard_degraded`
+
+**本地驗證結果**：
+- `bash -n scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh`：通過。
+- `python3.11 -m py_compile scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`：通過。
+- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py -q`：`3 passed`。
+- 使用最新 `/tmp/awoooi-post-reboot-readiness-20260629-122825/summary.txt` 產生 scorecard：容量已非 blocker，但缺 all-host boot probe 且 Wazuh dashboard degraded，所以不得宣稱 10 分鐘全服務恢復。
+
+**未做**：
+- 沒有重啟任何主機；沒有 restart Docker / Nginx / K3s / DB / firewall；沒有讀 secret / token / `.env` / raw sessions / SQLite / auth；沒有使用 GitHub。

docs/operations/awoooi-priority-work-order-readback.snapshot.json

@@ -1,7 +1,7 @@
 {
   "schema_version": "awoooi_priority_work_order_readback_v1",
-  "generated_at": "2026-06-29T14:11:34+08:00",
-  "status": "p0_005_waiting_refs_p0_003_gitea_only_scorecard_ready",
+  "generated_at": "2026-06-29T14:27:32+08:00",
+  "status": "p0_006a_reboot_auto_recovery_slo_control_plane_added_blocked_until_live_probe",
   "source_refs": {
     "global_scorecard": "~/.codex/product-runtime-governance-completion-scorecard.snapshot.json",
     "workstation_dashboard": "~/.codex/codex-workstation-sync-dashboard.snapshot.json",
@@ -11,10 +11,11 @@
     "public_gitea_queue_readback": "ops/runner/read-public-gitea-actions-queue.py --json",
     "credential_escrow_scorecard": "/tmp/awoooi-credential-escrow-intake-scorecard-20260629-1200-priority.json",
     "dr_escrow_evidence_checklist_generator": "scripts/reboot-recovery/dr-escrow-evidence-checklist.py",
-    "gitea_private_inventory_p0_scorecard": "docs/operations/awoooi-gitea-private-inventory-p0-scorecard.snapshot.json"
+    "gitea_private_inventory_p0_scorecard": "docs/operations/awoooi-gitea-private-inventory-p0-scorecard.snapshot.json",
+    "reboot_auto_recovery_slo_scorecard": "docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json"
   },
   "current_head": {
-    "gitea_main_sha": "86e674cf9e31118c6c52067a27b9c4e8e69cf8a0",
+    "gitea_main_sha": "748ee37ca958df1e1e25453363de3d9f3f02a6c1",
     "latest_successful_deploy_marker": "9362588ce chore(cd): deploy a423301 [skip ci]",
     "latest_successful_deployed_source_sha": "a4233017ad5fd03977233f3db6a4bb45d71507ed",
     "latest_source_readiness_commit_sha": "0c8d4e88c39157b92322fa41a92e6b15c317ac49",
@@ -179,18 +180,41 @@
     },
     {
       "workplan_id": "P0-006",
-      "title": "清理 source-to-runtime drift 與 stale routes",
-      "status": "pending_after_p0_005_and_p0_003",
-      "reason": "P0-004 source readiness and P0-002 product manifest are green; keep drift cleanup after credential escrow and authenticated Gitea inventory readback.",
+      "title": "主機重啟自動偵測、自動觸發與 10 分鐘恢復 SLO",
+      "status": "blocked_reboot_auto_recovery_slo_not_ready",
+      "reason": "The required target is automatic all-host reboot detection plus boot-triggered recovery verification. Current source now has the control-plane verifier, but live all-host boot probe has not been collected and Wazuh dashboard is still degraded.",
+      "evidence": {
+        "target_minutes": 10,
+        "can_claim_all_services_recovered_within_target": false,
+        "source_controls_added": true,
+        "host_boot_probe_source_present": true,
+        "slo_systemd_timer_source_present": true,
+        "slo_exporter_source_present": true,
+        "post_start_blocked": 0,
+        "service_green": true,
+        "product_data_green": true,
+        "backup_core_green": true,
+        "wazuh_dashboard_degraded": true,
+        "all_host_reboot_detection_missing": true,
+        "host_boot_probe_missing_hosts": true,
+        "local_disk_free_gib_after_cleanup": 3.271
+      },
       "professional_fix": {
-        "owner": "source-runtime drift lane",
-        "action": "Compare product.awoooi.yaml, committed K8s/docker/workflow sources, production readback, and public routes; generate one drift list with owner and fix command per row.",
+        "owner": "reboot auto-recovery lane",
+        "action": "Deploy the boot-triggered SLO timer/exporter, collect all-host boot probes, and rerun the scorecard until it can prove all services recovered inside 10 minutes.",
         "exit_criteria": [
-          "runtime_without_source_count=0",
-          "source_without_runtime_count=0",
-          "stale_route_count=0 or explicitly retired"
+          "can_claim_all_services_recovered_within_target=true",
+          "observed_hosts=110,120,121,188",
+          "max_observed_uptime_seconds<=600",
+          "POST_START_BLOCKED=0",
+          "SERVICE_GREEN=1",
+          "PRODUCT_DATA_GREEN=1",
+          "BACKUP_CORE_GREEN=1",
+          "WAZUH_DASHBOARD_DEGRADED=0",
+          "local_disk_free_gib>=2"
         ]
-      }
+      },
+      "safe_next_step": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard"
     }
   ],
   "noise_integrated_risk_register": [
@@ -264,6 +288,6 @@
   "next_execution_order": [
     "P0-005: fill the single DR escrow evidence checklist with five non-secret refs and rerun one preflight.",
     "P0-003: convert private/internal inventory to Gitea-only readback and remove retired GitHub from active P0 blocker math.",
-    "P0-006: run source-to-runtime drift cleanup using product manifest, committed runtime sources, production readback, and public route evidence."
+    "P0-006: deploy boot-triggered reboot auto-recovery SLO verifier, collect all-host boot probe, and prove or block the 10-minute recovery claim."
   ]
 }

docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json

@@ -0,0 +1,61 @@
+{
+  "active_blockers": [
+    "all_host_reboot_detection_missing",
+    "host_boot_probe_missing_hosts",
+    "wazuh_dashboard_degraded"
+  ],
+  "can_claim_all_services_recovered_within_target": false,
+  "capacity": {
+    "checked": true,
+    "free_gib": 2.707,
+    "min_free_gib": 2.0
+  },
+  "generated_at": "2026-06-29T14:27:32+08:00",
+  "host_boot_detection": {
+    "host_rows": [],
+    "max_observed_uptime_seconds": 0,
+    "missing_hosts": [
+      "110",
+      "120",
+      "121",
+      "188"
+    ],
+    "observed_hosts": [],
+    "required_hosts": [
+      "110",
+      "120",
+      "121",
+      "188"
+    ],
+    "stale_hosts": [],
+    "unknown_uptime_hosts": [],
+    "unreachable_hosts": []
+  },
+  "post_reboot_readiness": {
+    "backup_core_green": true,
+    "host_188_service_green": true,
+    "next_required_gates": "credential_escrow_evidence",
+    "overall_declaration": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED",
+    "post_start_blocked": 0,
+    "post_start_result": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED",
+    "product_data_green": true,
+    "service_green": true,
+    "summary_present": true,
+    "wazuh_dashboard_degraded": true
+  },
+  "safe_next_step": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard_until_status_slo_ready",
+  "schema_version": "awoooi_reboot_auto_recovery_slo_scorecard_v1",
+  "source_controls": {
+    "cold_start_textfile_exporter_source_present": true,
+    "host_110_startup_unit_source_present": true,
+    "host_188_startup_unit_source_present": true,
+    "host_boot_probe_source_present": true,
+    "post_reboot_summary_source_present": true,
+    "slo_exporter_source_present": true,
+    "slo_systemd_service_source_present": true,
+    "slo_systemd_timer_source_present": true
+  },
+  "status": "blocked_reboot_auto_recovery_slo_not_ready",
+  "target_minutes": 10,
+  "target_seconds": 600
+}