{
  "schema_version": "awoooi_post_reboot_runtime_recovery_readback_v1",
  "generated_at": "2026-07-02T00:17:36+08:00",
  "scope": "99_110_111_112_120_121_188",
  "purpose": "Record live post-reboot recovery evidence after 2026-07-01 all-host reboot and prevent source drift.",
  "boundaries": {
    "read_secret_values": false,
    "read_env_files": false,
    "read_raw_sessions": false,
    "read_sqlite": false,
    "used_github": false,
    "rebooted_hosts": false,
    "restarted_docker_nginx_k3s_db_firewall": false,
    "used_docker_system_prune": false,
    "touched_containers": false,
    "touched_volumes": false,
    "touched_databases": false,
    "touched_backups": false
  },
  "live_actions": [
    {
      "host": "188",
      "target": "momo-db",
      "action": "docker update --cpus=2 --memory=4g --memory-swap=6g momo-db",
      "blast_radius": "container resource guardrail only; no container restart",
      "readback": {
        "running": true,
        "nanocpus": 2000000000,
        "memory_bytes": 4294967296,
        "memswap_bytes": 6442450944,
        "restart_policy": "unless-stopped",
        "public_health": "https://mo.wooo.work/health returned status=healthy database=postgresql version=V10.725"
      }
    },
    {
      "host": "110",
      "target": "sentry-self-hosted-relay-1",
      "action": "docker update --memory=3g --memory-swap=4g sentry-self-hosted-relay-1",
      "blast_radius": "container resource guardrail only; no container restart",
      "readback": {
        "running": true,
        "health": "healthy",
        "memory_bytes": 3221225472,
        "memswap_bytes": 4294967296,
        "public_health": "https://sentry.wooo.work/_health/ returned ok"
      }
    },
    {
      "host": "110",
      "target": "root filesystem Docker pressure",
      "action": "bounded dangling image and BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
      "blast_radius": "unreferenced dangling images and builder cache only",
      "readback": {
        "filesystem_used_percent_after_prometheus": 79.68293081715491,
        "direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 983G Used 743G Avail 200G Use% 79% /",
        "docker_images_after": "92.91GB total, 29.81GB reclaimable",
        "docker_build_cache_after": "43.57GB total, 12.59GB reclaimable"
      }
    },
    {
      "host": "188",
      "target": "root filesystem Docker pressure",
      "action": "bounded BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
      "blast_radius": "builder cache only in final zero-age pass",
      "readback": {
        "filesystem_used_percent_after_prometheus": 79.22498388062007,
        "direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 982G Used 737G Avail 204G Use% 79% /",
        "docker_images_after": "42.93GB total, 14.96GB reclaimable",
        "docker_build_cache_after": "33.31GB total, 42.82MB reclaimable"
      }
    },
    {
      "host": "110",
      "target": "reboot auto recovery SLO textfile",
      "action": "bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh",
      "blast_radius": "script/timer/textfile install and verifier run only",
      "readback": {
        "scope": "99_110_111_112_120_121_188",
        "awoooi_reboot_event_detected": 1,
        "awoooi_reboot_auto_recovery_slo_ready": 0,
        "awoooi_reboot_auto_recovery_slo_blocker_count": 6,
        "awoooi_reboot_event_target_seconds_remaining": 0
      }
    },
    {
      "host": "110",
      "target": "stockplatform-v2-postgres-1",
      "action": "docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped stockplatform-v2-postgres-1",
      "blast_radius": "container resource guardrail only; no container restart and no database write",
      "readback": {
        "running": true,
        "health": "healthy",
        "nanocpus": 1500000000,
        "memory_bytes": 4294967296,
        "memswap_bytes": 6442450944,
        "restart_policy": "unless-stopped",
        "direct_docker_stats_after": "cpu=149.38% mem=506.2MiB / 4GiB",
        "prometheus_after_convergence": {
          "stockplatform_v2_postgres_cpu_cores": 1.6508,
          "gitea_cpu_cores": 1.8097,
          "host_110_load5_per_core": 0.673333
        },
        "public_route_health": {
          "root": 200,
          "healthz": 200,
          "api_healthz": 200,
          "freshness_status": "ok",
          "latest_trading_date": "2026-07-01"
        }
      }
    },
    {
      "host": "110",
      "target": "gitea",
      "action": "docker update --cpus=1.5 gitea",
      "blast_radius": "container CPU quota update only; no container restart",
      "readback": {
        "running": true,
        "nanocpus": 1500000000,
        "memory_bytes": 3221225472,
        "restart_policy": "always",
        "public_version": "1.25.5",
        "prometheus_after_convergence": {
          "gitea_cpu_cores": 0.9882,
          "stockplatform_v2_postgres_cpu_cores": 0.7296,
          "host_110_load5_per_core": 0.725833
        }
      }
    }
  ],
  "prometheus_readback": {
    "alerts_after_recovery": [
      {
        "alertname": "HostRebootEventDetected",
        "severity": "warning",
        "host": "110"
      },
      {
        "alertname": "RebootAutoRecoverySLOMissed",
        "severity": "critical",
        "scope": "99_110_111_112_120_121_188"
      }
    ],
    "cleared_alert_classes": [
      "DockerContainerMissingResourceLimit",
      "DockerContainerMemoryLimitPressure",
      "HostOutOfDiskSpace",
      "HostDiskUsageHigh",
      "RebootEventDetectorMissing",
      "Host110SustainedModeratePressure",
      "HostLoadAverageSustainedHigh",
      "DockerContainerCpuSustainedHigh",
      "StockWoooWorkDown"
    ],
    "filesystem_used_percent": {
      "110": 79.68293081715491,
      "188": 79.22498388062007
    }
  },
  "cold_start_scorecard": {
    "command": "SSH_CONNECT_TIMEOUT=8 bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color",
    "pass": 96,
    "warn": 0,
    "blocked": 0,
    "result": "GREEN",
    "runtime_truth": "Full stack is ready for controlled runner/CD release.",
    "declaration_limit": "This proves current runtime recovery, not that the 10-minute reboot SLO was met."
  },
  "source_updates": [
    "scripts/ops/docker-disk-pressure-retention-cleanup.py",
    "scripts/ops/host-runaway-process-exporter.py",
    "scripts/ops/host-sustained-load-evidence.py",
    "scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
    "scripts/ops/tests/test_host_runaway_process_exporter.py",
    "ops/reboot-recovery/full-stack-cold-start-baseline.yml",
    "docs/runbooks/FULL-STACK-COLD-START-SOP.md",
    "docs/LOGBOOK.md"
  ],
  "local_verification": [
    {
      "command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py -q",
      "result": "6 passed"
    },
    {
      "command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py scripts/ops/tests/test_host_pressure_alert_contract.py -q",
      "result": "9 passed"
    },
    {
      "command": "python3.11 -m py_compile scripts/ops/docker-disk-pressure-retention-cleanup.py scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
      "result": "passed"
    }
  ],
  "remaining_truth": {
    "runtime_full_stack_green": true,
    "ten_minute_slo_met_for_this_reboot": false,
    "why": "Reboot event was detected after the target window had elapsed, so Prometheus correctly keeps RebootAutoRecoverySLOMissed as incident evidence.",
    "next_mainline": [
      "Persist resource guardrails in host compose/source where owned by the matching repo.",
      "Replace stale reboot SLO API snapshot with live textfile/Prometheus backed readback.",
      "Convert the six reboot SLO blockers into automatic preflight/remediation steps without silencing the SLO miss."
    ]
  }
}
