fix(ops): close post-reboot recovery guardrails

docs/operations/post-reboot-runtime-recovery-readback-2026-07-01.snapshot.json

@@ -0,0 +1,151 @@
+{
+  "schema_version": "awoooi_post_reboot_runtime_recovery_readback_v1",
+  "generated_at": "2026-07-02T00:17:36+08:00",
+  "scope": "99_110_111_112_120_121_188",
+  "purpose": "Record live post-reboot recovery evidence after 2026-07-01 all-host reboot and prevent source drift.",
+  "boundaries": {
+    "read_secret_values": false,
+    "read_env_files": false,
+    "read_raw_sessions": false,
+    "read_sqlite": false,
+    "used_github": false,
+    "rebooted_hosts": false,
+    "restarted_docker_nginx_k3s_db_firewall": false,
+    "used_docker_system_prune": false,
+    "touched_containers": false,
+    "touched_volumes": false,
+    "touched_databases": false,
+    "touched_backups": false
+  },
+  "live_actions": [
+    {
+      "host": "188",
+      "target": "momo-db",
+      "action": "docker update --cpus=2 --memory=4g --memory-swap=6g momo-db",
+      "blast_radius": "container resource guardrail only; no container restart",
+      "readback": {
+        "running": true,
+        "nanocpus": 2000000000,
+        "memory_bytes": 4294967296,
+        "memswap_bytes": 6442450944,
+        "restart_policy": "unless-stopped",
+        "public_health": "https://mo.wooo.work/health returned status=healthy database=postgresql version=V10.725"
+      }
+    },
+    {
+      "host": "110",
+      "target": "sentry-self-hosted-relay-1",
+      "action": "docker update --memory=3g --memory-swap=4g sentry-self-hosted-relay-1",
+      "blast_radius": "container resource guardrail only; no container restart",
+      "readback": {
+        "running": true,
+        "health": "healthy",
+        "memory_bytes": 3221225472,
+        "memswap_bytes": 4294967296,
+        "public_health": "https://sentry.wooo.work/_health/ returned ok"
+      }
+    },
+    {
+      "host": "110",
+      "target": "root filesystem Docker pressure",
+      "action": "bounded dangling image and BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
+      "blast_radius": "unreferenced dangling images and builder cache only",
+      "readback": {
+        "filesystem_used_percent_after_prometheus": 79.68293081715491,
+        "direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 983G Used 743G Avail 200G Use% 79% /",
+        "docker_images_after": "92.91GB total, 29.81GB reclaimable",
+        "docker_build_cache_after": "43.57GB total, 12.59GB reclaimable"
+      }
+    },
+    {
+      "host": "188",
+      "target": "root filesystem Docker pressure",
+      "action": "bounded BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
+      "blast_radius": "builder cache only in final zero-age pass",
+      "readback": {
+        "filesystem_used_percent_after_prometheus": 79.22498388062007,
+        "direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 982G Used 737G Avail 204G Use% 79% /",
+        "docker_images_after": "42.93GB total, 14.96GB reclaimable",
+        "docker_build_cache_after": "33.31GB total, 42.82MB reclaimable"
+      }
+    },
+    {
+      "host": "110",
+      "target": "reboot auto recovery SLO textfile",
+      "action": "bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh",
+      "blast_radius": "script/timer/textfile install and verifier run only",
+      "readback": {
+        "scope": "99_110_111_112_120_121_188",
+        "awoooi_reboot_event_detected": 1,
+        "awoooi_reboot_auto_recovery_slo_ready": 0,
+        "awoooi_reboot_auto_recovery_slo_blocker_count": 6,
+        "awoooi_reboot_event_target_seconds_remaining": 0
+      }
+    }
+  ],
+  "prometheus_readback": {
+    "alerts_after_recovery": [
+      {
+        "alertname": "HostRebootEventDetected",
+        "severity": "warning",
+        "host": "110"
+      },
+      {
+        "alertname": "RebootAutoRecoverySLOMissed",
+        "severity": "critical",
+        "scope": "99_110_111_112_120_121_188"
+      }
+    ],
+    "cleared_alert_classes": [
+      "DockerContainerMissingResourceLimit",
+      "DockerContainerMemoryLimitPressure",
+      "HostOutOfDiskSpace",
+      "HostDiskUsageHigh",
+      "RebootEventDetectorMissing"
+    ],
+    "filesystem_used_percent": {
+      "110": 79.68293081715491,
+      "188": 79.22498388062007
+    }
+  },
+  "cold_start_scorecard": {
+    "command": "SSH_CONNECT_TIMEOUT=8 bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color",
+    "pass": 96,
+    "warn": 0,
+    "blocked": 0,
+    "result": "GREEN",
+    "runtime_truth": "Full stack is ready for controlled runner/CD release.",
+    "declaration_limit": "This proves current runtime recovery, not that the 10-minute reboot SLO was met."
+  },
+  "source_updates": [
+    "scripts/ops/docker-disk-pressure-retention-cleanup.py",
+    "scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
+    "ops/reboot-recovery/full-stack-cold-start-baseline.yml",
+    "docs/runbooks/FULL-STACK-COLD-START-SOP.md",
+    "docs/LOGBOOK.md"
+  ],
+  "local_verification": [
+    {
+      "command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py -q",
+      "result": "6 passed"
+    },
+    {
+      "command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py scripts/ops/tests/test_host_pressure_alert_contract.py -q",
+      "result": "9 passed"
+    },
+    {
+      "command": "python3.11 -m py_compile scripts/ops/docker-disk-pressure-retention-cleanup.py scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
+      "result": "passed"
+    }
+  ],
+  "remaining_truth": {
+    "runtime_full_stack_green": true,
+    "ten_minute_slo_met_for_this_reboot": false,
+    "why": "Reboot event was detected after the target window had elapsed, so Prometheus correctly keeps RebootAutoRecoverySLOMissed as incident evidence.",
+    "next_mainline": [
+      "Persist resource guardrails in host compose/source where owned by the matching repo.",
+      "Replace stale reboot SLO API snapshot with live textfile/Prometheus backed readback.",
+      "Convert the six reboot SLO blockers into automatic preflight/remediation steps without silencing the SLO miss."
+    ]
+  }
+}