338 lines
9.9 KiB
YAML
338 lines
9.9 KiB
YAML
# AWOOOI full-stack cold-start dependency baseline.
|
|
# This is the machine-readable companion to docs/runbooks/FULL-STACK-COLD-START-SOP.md.
|
|
#
|
|
# Intent:
|
|
# - document the reboot startup order and service dependency graph
|
|
# - define release gates for operators and AI automation
|
|
# - keep stateful services out of generic auto-restart loops
|
|
|
|
version: "2026-05-06"
|
|
incident_reference: "2026-05-05 full-stack reboot recovery"
|
|
scope:
|
|
managed_hosts:
|
|
"110":
|
|
address: "192.168.0.110"
|
|
ssh_user: "wooo"
|
|
roles:
|
|
- registry
|
|
- git
|
|
- observability
|
|
- sentry
|
|
- runners
|
|
"120":
|
|
address: "192.168.0.120"
|
|
ssh_user: "wooo"
|
|
roles:
|
|
- k3s_server
|
|
- keepalived_vip
|
|
- awoooi_nodeport
|
|
"121":
|
|
address: "192.168.0.121"
|
|
ssh_user: "wooo"
|
|
roles:
|
|
- k3s_node
|
|
- keepalived_peer
|
|
- dr_drill
|
|
"188":
|
|
address: "192.168.0.188"
|
|
ssh_user: "ollama"
|
|
roles:
|
|
- postgres_datastore
|
|
- redis
|
|
- momo
|
|
- signoz
|
|
- ai_proxy
|
|
intentionally_skipped:
|
|
"112":
|
|
role: "kali"
|
|
reason: "scanner host is not required for production cold-start release"
|
|
|
|
global_policy:
|
|
startup_rule: "Recover the dependency chain before releasing high-load work."
|
|
runner_cd_rule: "Release runners and CD only after data, registry, K3s, workload, routes, schedules, and alert E2E gates are green."
|
|
ai_auto_repair_rule: "Observe-only until all green gates pass and host load stays below baseline."
|
|
destructive_state_rule: "No DROP, data directory deletion, volume recreation, pg_resetwal, fsck, or backup restore without explicit human approval."
|
|
no_generic_restart_rule: "Never run generic docker restart against all containers during cold start."
|
|
|
|
phases:
|
|
- id: "P0-NETWORK"
|
|
order: 0
|
|
start_after: []
|
|
owns:
|
|
- "LAN reachability"
|
|
- "SSH reachability"
|
|
- "ARP evidence"
|
|
gates:
|
|
- "ping 192.168.0.110/120/121/188 succeeds"
|
|
- "TCP 22 open on 192.168.0.110/120/121/188"
|
|
- "reboot evidence captured before repair"
|
|
blocks:
|
|
- "all other phases"
|
|
|
|
- id: "P0-188-DATA"
|
|
order: 1
|
|
start_after:
|
|
- "P0-NETWORK"
|
|
host: "188"
|
|
service_order:
|
|
- "containerd"
|
|
- "docker"
|
|
- "postgresql@14-main"
|
|
- "k3s_datastore.kine maintenance"
|
|
- "redis-server"
|
|
- "ollama or current AI proxy dependencies"
|
|
- "nginx"
|
|
- "Docker networks"
|
|
- "MinIO / OpenClaw / SignOz"
|
|
- "momo / litellm / batch services"
|
|
gates:
|
|
- "PostgreSQL port 5432 open"
|
|
- "pg_isready reports accepting connections"
|
|
- "Redis replies PONG"
|
|
- "momo health endpoint returns 200"
|
|
- "SignOz HTTP route is reachable"
|
|
blocks:
|
|
- "120/121 K3s"
|
|
- "AWOOOI API database access"
|
|
- "Alertmanager webhook"
|
|
- "momo public site"
|
|
|
|
- id: "P0-110-REGISTRY-OBSERVABILITY"
|
|
order: 2
|
|
start_after:
|
|
- "P0-NETWORK"
|
|
- "P0-188-DATA"
|
|
host: "110"
|
|
service_order:
|
|
- "docker"
|
|
- "orphan Exited(128/137) cleanup if needed"
|
|
- "Harbor log"
|
|
- "Harbor registry stack"
|
|
- "Gitea"
|
|
- "Prometheus / Alertmanager / Grafana / exporters"
|
|
- "Langfuse"
|
|
- "SignOz or local observability companions"
|
|
- "Sentry DB layer"
|
|
- "Sentry web / worker / consumer layer"
|
|
- "Gitea host runner and actions runners"
|
|
gates:
|
|
- "Harbor /v2/ returns 200 or 401"
|
|
- "Gitea returns 200 or 302"
|
|
- "Prometheus /-/ready returns 200"
|
|
- "Alertmanager /-/healthy returns 200"
|
|
- "Sentry HTTP returns 200, 302, or 400"
|
|
- "runner CPUQuota=200%, MemoryMax=2G, WatchdogUSec=0"
|
|
blocks:
|
|
- "K3s image pulls"
|
|
- "runtime CD"
|
|
- "alert rules deploy"
|
|
- "code-review runners"
|
|
|
|
- id: "P1-K3S"
|
|
order: 3
|
|
start_after:
|
|
- "P0-188-DATA"
|
|
- "P0-110-REGISTRY-OBSERVABILITY"
|
|
hosts:
|
|
- "120"
|
|
- "121"
|
|
service_order:
|
|
- "120 k3s.service"
|
|
- "121 k3s-agent.service or live role"
|
|
- "CNI / kube-proxy"
|
|
- "nodes Ready"
|
|
- "core pods"
|
|
- "awoooi-prod pods"
|
|
- "keepalived VIP 192.168.0.125"
|
|
- "NodePorts 32334 and 32335"
|
|
gates:
|
|
- "120 can reach 188:5432"
|
|
- "K3s nodes show Ready"
|
|
- "VIP 192.168.0.125 is present"
|
|
- "awoooi-prod pods are Running or Completed"
|
|
blocks:
|
|
- "AWOOOI workload health"
|
|
- "public AWOOOI route"
|
|
- "Alertmanager webhook"
|
|
|
|
- id: "P2-WORKLOAD-ALERTCHAIN"
|
|
order: 4
|
|
start_after:
|
|
- "P1-K3S"
|
|
owners:
|
|
- "AWOOOI API"
|
|
- "AWOOOI Web"
|
|
- "Alertmanager webhook"
|
|
- "Telegram delivery"
|
|
gates:
|
|
- "http://192.168.0.125:32334/api/v1/health returns 2xx/3xx"
|
|
- "http://192.168.0.125:32335/ returns 2xx/3xx"
|
|
- "Alertmanager webhook POST returns 2xx"
|
|
- "K8s Telegram secrets are present and non-placeholder"
|
|
blocks:
|
|
- "AI auto-remediation"
|
|
- "full alert confidence"
|
|
|
|
- id: "P2-PUBLIC-ROUTES"
|
|
order: 5
|
|
start_after:
|
|
- "P2-WORKLOAD-ALERTCHAIN"
|
|
gates:
|
|
- "https://awoooi.wooo.work/api/v1/health returns 2xx/3xx"
|
|
- "https://awoooi.wooo.work/ returns 2xx/3xx"
|
|
- "https://mo.wooo.work/ returns 2xx/3xx"
|
|
- "https://mo.wooo.work/health returns 2xx/3xx"
|
|
blocks:
|
|
- "external release complete"
|
|
|
|
- id: "P2-SCHEDULES"
|
|
order: 6
|
|
start_after:
|
|
- "P2-PUBLIC-ROUTES"
|
|
gates:
|
|
- "110/120/121/188 cron services active"
|
|
- "188 backup-from-110 success age below 25h"
|
|
- "188 docker restart/stats textfiles fresh"
|
|
- "188 momo-scheduler container healthy and registration evidence present within 6h"
|
|
- "110 docker/systemd textfiles fresh"
|
|
- "120 awoooi-prod CronJobs present and unsuspended"
|
|
- "120 awoooi-prod has no failed Jobs"
|
|
- "121 DR drill cron present"
|
|
blocks:
|
|
- "done criteria"
|
|
- "AI auto-remediation release"
|
|
|
|
- id: "P3-HIGH-LOAD-RELEASE"
|
|
order: 7
|
|
start_after:
|
|
- "P2-SCHEDULES"
|
|
release_last:
|
|
- "momo-scheduler / Chrome crawlers"
|
|
- "Sentry Snuba consumers"
|
|
- "SignOz ClickHouse merge-heavy work"
|
|
- "Gitea actions runners"
|
|
- "runtime CD jobs"
|
|
gates:
|
|
- "all prior gates green"
|
|
- "host load per CPU below 1.0 for 15 minutes before releasing batch/runner work"
|
|
- "ClickHouse/Kafka/Snuba backlog decreasing for two consecutive checks if backlog exists"
|
|
|
|
baselines:
|
|
endpoints:
|
|
awoooi_vip_api_health: "http://192.168.0.125:32334/api/v1/health"
|
|
awoooi_vip_web: "http://192.168.0.125:32335/"
|
|
awoooi_public_api_health: "https://awoooi.wooo.work/api/v1/health"
|
|
awoooi_public_web: "https://awoooi.wooo.work/"
|
|
momo_public_web: "https://mo.wooo.work/"
|
|
momo_public_health: "https://mo.wooo.work/health"
|
|
harbor_registry: "http://127.0.0.1:5000/v2/"
|
|
gitea: "http://127.0.0.1:3001/"
|
|
prometheus_ready: "http://127.0.0.1:9090/-/ready"
|
|
alertmanager_healthy: "http://127.0.0.1:9093/-/healthy"
|
|
sentry: "http://127.0.0.1:9000/"
|
|
expected_codes:
|
|
harbor_registry:
|
|
- 200
|
|
- 401
|
|
gitea:
|
|
- 200
|
|
- 302
|
|
prometheus_ready:
|
|
- 200
|
|
alertmanager_healthy:
|
|
- 200
|
|
sentry:
|
|
- 200
|
|
- 302
|
|
- 400
|
|
workload_and_public:
|
|
- "2xx"
|
|
- "3xx"
|
|
runner_guardrails:
|
|
CPUQuotaPerSecUSec: "2s"
|
|
MemoryMax: "2147483648"
|
|
WatchdogUSec: "0"
|
|
freshness_seconds:
|
|
docker_textfiles: 300
|
|
systemd_textfiles: 300
|
|
backup_success: 90000
|
|
|
|
stateful_services:
|
|
hard_block_auto_repair:
|
|
- "188 PostgreSQL data directory"
|
|
- "188 k3s_datastore"
|
|
- "188 momo database"
|
|
- "110 Harbor DB"
|
|
- "110 Sentry DB"
|
|
- "Sentry ClickHouse data"
|
|
- "SignOz ClickHouse data"
|
|
- "Kafka topic/log directories"
|
|
human_in_loop_required:
|
|
- "pg_resetwal"
|
|
- "ClickHouse clean-clone recovery"
|
|
- "Kafka checkpoint file quarantine"
|
|
- "backup restore"
|
|
- "filesystem repair"
|
|
|
|
ai_automation_gate:
|
|
observe_only_until:
|
|
- "P0-NETWORK green"
|
|
- "P0-188-DATA green"
|
|
- "P0-110-REGISTRY-OBSERVABILITY green"
|
|
- "P1-K3S green"
|
|
- "P2-WORKLOAD-ALERTCHAIN green"
|
|
- "P2-PUBLIC-ROUTES green"
|
|
- "P2-SCHEDULES green"
|
|
- "no active restart storm"
|
|
- "host load per CPU below 1.0 for 15 minutes"
|
|
allowed_before_green:
|
|
- "diagnose"
|
|
- "collect evidence"
|
|
- "notify"
|
|
blocked_before_green:
|
|
- "stateful restart"
|
|
- "destructive repair"
|
|
- "runner/CD release"
|
|
- "generic container restart"
|
|
|
|
persistent_monitoring:
|
|
host: "110"
|
|
install_command: "bash scripts/reboot-recovery/install-cold-start-monitor-110.sh"
|
|
schedule: "*/10 * * * *"
|
|
mode: "read_only"
|
|
send_alert_test: false
|
|
scripts:
|
|
check: "/home/wooo/scripts/full-stack-cold-start-check.sh"
|
|
exporter: "/home/wooo/scripts/cold-start-textfile-exporter.sh"
|
|
outputs:
|
|
textfile: "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom"
|
|
last_log: "/home/wooo/reboot-recovery/cold-start-last.log"
|
|
metrics:
|
|
- "awoooi_cold_start_monitor_up"
|
|
- "awoooi_cold_start_pass_gates"
|
|
- "awoooi_cold_start_warn_gates"
|
|
- "awoooi_cold_start_blocked_gates"
|
|
- "awoooi_cold_start_last_run_timestamp"
|
|
- "awoooi_cold_start_last_green_timestamp"
|
|
- "awoooi_cold_start_last_result"
|
|
prometheus_alerts:
|
|
- "ColdStartMonitorMissing"
|
|
- "ColdStartMonitorStale"
|
|
- "ColdStartRecoveryBlocked"
|
|
- "ColdStartRecoveryDegraded"
|
|
- "ColdStartLastGreenTooOld"
|
|
ai_contract:
|
|
monitor_missing: "diagnose cron/textfile path only"
|
|
stale: "collect cron log and last check log"
|
|
degraded: "collect evidence, do not release high-load work"
|
|
blocked: "follow first BLOCKED gate in phase order"
|
|
forbidden: "generic restart, stateful restart, destructive repair"
|
|
|
|
final_confirmation:
|
|
command: "bash scripts/reboot-recovery/full-stack-cold-start-check.sh --watch --interval 60 --max-attempts 30 --send-alert-test"
|
|
green_result:
|
|
PASS: "greater than 0"
|
|
WARN: 0
|
|
BLOCKED: 0
|
|
summary: "Result: GREEN"
|