204 lines
7.8 KiB
JSON
204 lines
7.8 KiB
JSON
{
|
|
"schema_version": "awoooi_post_reboot_runtime_recovery_readback_v1",
|
|
"generated_at": "2026-07-02T00:17:36+08:00",
|
|
"scope": "99_110_111_112_120_121_188",
|
|
"purpose": "Record live post-reboot recovery evidence after 2026-07-01 all-host reboot and prevent source drift.",
|
|
"boundaries": {
|
|
"read_secret_values": false,
|
|
"read_env_files": false,
|
|
"read_raw_sessions": false,
|
|
"read_sqlite": false,
|
|
"used_github": false,
|
|
"rebooted_hosts": false,
|
|
"restarted_docker_nginx_k3s_db_firewall": false,
|
|
"used_docker_system_prune": false,
|
|
"touched_containers": false,
|
|
"touched_volumes": false,
|
|
"touched_databases": false,
|
|
"touched_backups": false
|
|
},
|
|
"live_actions": [
|
|
{
|
|
"host": "188",
|
|
"target": "momo-db",
|
|
"action": "docker update --cpus=2 --memory=4g --memory-swap=6g momo-db",
|
|
"blast_radius": "container resource guardrail only; no container restart",
|
|
"readback": {
|
|
"running": true,
|
|
"nanocpus": 2000000000,
|
|
"memory_bytes": 4294967296,
|
|
"memswap_bytes": 6442450944,
|
|
"restart_policy": "unless-stopped",
|
|
"public_health": "https://mo.wooo.work/health returned status=healthy database=postgresql version=V10.725"
|
|
}
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target": "sentry-self-hosted-relay-1",
|
|
"action": "docker update --memory=3g --memory-swap=4g sentry-self-hosted-relay-1",
|
|
"blast_radius": "container resource guardrail only; no container restart",
|
|
"readback": {
|
|
"running": true,
|
|
"health": "healthy",
|
|
"memory_bytes": 3221225472,
|
|
"memswap_bytes": 4294967296,
|
|
"public_health": "https://sentry.wooo.work/_health/ returned ok"
|
|
}
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target": "root filesystem Docker pressure",
|
|
"action": "bounded dangling image and BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
|
|
"blast_radius": "unreferenced dangling images and builder cache only",
|
|
"readback": {
|
|
"filesystem_used_percent_after_prometheus": 79.68293081715491,
|
|
"direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 983G Used 743G Avail 200G Use% 79% /",
|
|
"docker_images_after": "92.91GB total, 29.81GB reclaimable",
|
|
"docker_build_cache_after": "43.57GB total, 12.59GB reclaimable"
|
|
}
|
|
},
|
|
{
|
|
"host": "188",
|
|
"target": "root filesystem Docker pressure",
|
|
"action": "bounded BuildKit cache cleanup with scripts/ops/docker-disk-pressure-retention-cleanup.py",
|
|
"blast_radius": "builder cache only in final zero-age pass",
|
|
"readback": {
|
|
"filesystem_used_percent_after_prometheus": 79.22498388062007,
|
|
"direct_df_after": "/dev/mapper/ubuntu--vg-ubuntu--lv ext4 982G Used 737G Avail 204G Use% 79% /",
|
|
"docker_images_after": "42.93GB total, 14.96GB reclaimable",
|
|
"docker_build_cache_after": "33.31GB total, 42.82MB reclaimable"
|
|
}
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target": "reboot auto recovery SLO textfile",
|
|
"action": "bash scripts/reboot-recovery/install-reboot-auto-recovery-slo-110.sh",
|
|
"blast_radius": "script/timer/textfile install and verifier run only",
|
|
"readback": {
|
|
"scope": "99_110_111_112_120_121_188",
|
|
"awoooi_reboot_event_detected": 1,
|
|
"awoooi_reboot_auto_recovery_slo_ready": 0,
|
|
"awoooi_reboot_auto_recovery_slo_blocker_count": 6,
|
|
"awoooi_reboot_event_target_seconds_remaining": 0
|
|
}
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target": "stockplatform-v2-postgres-1",
|
|
"action": "docker update --cpus=1.5 --memory=4g --memory-swap=6g --restart unless-stopped stockplatform-v2-postgres-1",
|
|
"blast_radius": "container resource guardrail only; no container restart and no database write",
|
|
"readback": {
|
|
"running": true,
|
|
"health": "healthy",
|
|
"nanocpus": 1500000000,
|
|
"memory_bytes": 4294967296,
|
|
"memswap_bytes": 6442450944,
|
|
"restart_policy": "unless-stopped",
|
|
"direct_docker_stats_after": "cpu=149.38% mem=506.2MiB / 4GiB",
|
|
"prometheus_after_convergence": {
|
|
"stockplatform_v2_postgres_cpu_cores": 1.6508,
|
|
"gitea_cpu_cores": 1.8097,
|
|
"host_110_load5_per_core": 0.673333
|
|
},
|
|
"public_route_health": {
|
|
"root": 200,
|
|
"healthz": 200,
|
|
"api_healthz": 200,
|
|
"freshness_status": "ok",
|
|
"latest_trading_date": "2026-07-01"
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"host": "110",
|
|
"target": "gitea",
|
|
"action": "docker update --cpus=1.5 gitea",
|
|
"blast_radius": "container CPU quota update only; no container restart",
|
|
"readback": {
|
|
"running": true,
|
|
"nanocpus": 1500000000,
|
|
"memory_bytes": 3221225472,
|
|
"restart_policy": "always",
|
|
"public_version": "1.25.5",
|
|
"prometheus_after_convergence": {
|
|
"gitea_cpu_cores": 0.9882,
|
|
"stockplatform_v2_postgres_cpu_cores": 0.7296,
|
|
"host_110_load5_per_core": 0.725833
|
|
}
|
|
}
|
|
}
|
|
],
|
|
"prometheus_readback": {
|
|
"alerts_after_recovery": [
|
|
{
|
|
"alertname": "HostRebootEventDetected",
|
|
"severity": "warning",
|
|
"host": "110"
|
|
},
|
|
{
|
|
"alertname": "RebootAutoRecoverySLOMissed",
|
|
"severity": "critical",
|
|
"scope": "99_110_111_112_120_121_188"
|
|
}
|
|
],
|
|
"cleared_alert_classes": [
|
|
"DockerContainerMissingResourceLimit",
|
|
"DockerContainerMemoryLimitPressure",
|
|
"HostOutOfDiskSpace",
|
|
"HostDiskUsageHigh",
|
|
"RebootEventDetectorMissing",
|
|
"Host110SustainedModeratePressure",
|
|
"HostLoadAverageSustainedHigh",
|
|
"DockerContainerCpuSustainedHigh",
|
|
"StockWoooWorkDown"
|
|
],
|
|
"filesystem_used_percent": {
|
|
"110": 79.68293081715491,
|
|
"188": 79.22498388062007
|
|
}
|
|
},
|
|
"cold_start_scorecard": {
|
|
"command": "SSH_CONNECT_TIMEOUT=8 bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color",
|
|
"pass": 96,
|
|
"warn": 0,
|
|
"blocked": 0,
|
|
"result": "GREEN",
|
|
"runtime_truth": "Full stack is ready for controlled runner/CD release.",
|
|
"declaration_limit": "This proves current runtime recovery, not that the 10-minute reboot SLO was met."
|
|
},
|
|
"source_updates": [
|
|
"scripts/ops/docker-disk-pressure-retention-cleanup.py",
|
|
"scripts/ops/host-runaway-process-exporter.py",
|
|
"scripts/ops/host-sustained-load-evidence.py",
|
|
"scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
|
|
"scripts/ops/tests/test_host_runaway_process_exporter.py",
|
|
"ops/reboot-recovery/full-stack-cold-start-baseline.yml",
|
|
"docs/runbooks/FULL-STACK-COLD-START-SOP.md",
|
|
"docs/LOGBOOK.md"
|
|
],
|
|
"local_verification": [
|
|
{
|
|
"command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py -q",
|
|
"result": "6 passed"
|
|
},
|
|
{
|
|
"command": "python3.11 -m pytest scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py scripts/ops/tests/test_host_pressure_alert_contract.py -q",
|
|
"result": "9 passed"
|
|
},
|
|
{
|
|
"command": "python3.11 -m py_compile scripts/ops/docker-disk-pressure-retention-cleanup.py scripts/ops/tests/test_docker_disk_pressure_retention_cleanup.py",
|
|
"result": "passed"
|
|
}
|
|
],
|
|
"remaining_truth": {
|
|
"runtime_full_stack_green": true,
|
|
"ten_minute_slo_met_for_this_reboot": false,
|
|
"why": "Reboot event was detected after the target window had elapsed, so Prometheus correctly keeps RebootAutoRecoverySLOMissed as incident evidence.",
|
|
"next_mainline": [
|
|
"Persist resource guardrails in host compose/source where owned by the matching repo.",
|
|
"Replace stale reboot SLO API snapshot with live textfile/Prometheus backed readback.",
|
|
"Convert the six reboot SLO blockers into automatic preflight/remediation steps without silencing the SLO miss."
|
|
]
|
|
}
|
|
}
|