# AWOOOI full-stack cold-start dependency baseline.
# This is the machine-readable companion to docs/runbooks/FULL-STACK-COLD-START-SOP.md.
#
# Intent:
# - document the reboot startup order and service dependency graph
# - define release gates for operators and AI automation
# - keep stateful services out of generic auto-restart loops

version: "2026-05-06"
incident_reference: "2026-05-05 full-stack reboot recovery"
scope:
  managed_hosts:
    "110":
      address: "192.168.0.110"
      ssh_user: "wooo"
      roles:
        - registry
        - git
        - observability
        - sentry
        - runners
    "120":
      address: "192.168.0.120"
      ssh_user: "wooo"
      roles:
        - k3s_server
        - keepalived_vip
        - awoooi_nodeport
    "121":
      address: "192.168.0.121"
      ssh_user: "wooo"
      roles:
        - k3s_node
        - keepalived_peer
        - dr_drill
    "188":
      address: "192.168.0.188"
      ssh_user: "ollama"
      roles:
        - postgres_datastore
        - redis
        - momo
        - signoz
        - ai_proxy
  intentionally_skipped:
    "112":
      role: "kali"
      reason: "scanner host is not required for production cold-start release"

global_policy:
  startup_rule: "Recover the dependency chain before releasing high-load work."
  runner_cd_rule: "Release runners and CD only after data, registry, K3s, workload, routes, schedules, and alert E2E gates are green."
  ai_auto_repair_rule: "Observe-only until all green gates pass and host load stays below baseline."
  destructive_state_rule: "No DROP, data directory deletion, volume recreation, pg_resetwal, fsck, or backup restore without explicit human approval."
  no_generic_restart_rule: "Never run generic docker restart against all containers during cold start."

phases:
  - id: "P0-NETWORK"
    order: 0
    start_after: []
    owns:
      - "LAN reachability"
      - "SSH reachability"
      - "ARP evidence"
    gates:
      - "ping 192.168.0.110/120/121/188 succeeds"
      - "TCP 22 open on 192.168.0.110/120/121/188"
      - "reboot evidence captured before repair"
    blocks:
      - "all other phases"

  - id: "P0-188-DATA"
    order: 1
    start_after:
      - "P0-NETWORK"
    host: "188"
    service_order:
      - "containerd"
      - "docker"
      - "postgresql@14-main"
      - "k3s_datastore.kine maintenance"
      - "redis-server"
      - "ollama or current AI proxy dependencies"
      - "nginx"
      - "Docker networks"
      - "MinIO / OpenClaw / SignOz"
      - "momo / litellm / batch services"
    gates:
      - "PostgreSQL port 5432 open"
      - "pg_isready reports accepting connections"
      - "Redis replies PONG"
      - "momo health endpoint returns 200"
      - "SignOz HTTP route is reachable"
    blocks:
      - "120/121 K3s"
      - "AWOOOI API database access"
      - "Alertmanager webhook"
      - "momo public site"

  - id: "P0-110-REGISTRY-OBSERVABILITY"
    order: 2
    start_after:
      - "P0-NETWORK"
      - "P0-188-DATA"
    host: "110"
    service_order:
      - "docker"
      - "orphan Exited(128/137) cleanup if needed"
      - "Harbor log"
      - "Harbor registry stack"
      - "Gitea"
      - "Prometheus / Alertmanager / Grafana / exporters"
      - "Langfuse"
      - "SignOz or local observability companions"
      - "Sentry DB layer"
      - "Sentry web / worker / consumer layer"
      - "Gitea host runner and actions runners"
    gates:
      - "Harbor /v2/ returns 200 or 401"
      - "Gitea returns 200 or 302"
      - "Prometheus /-/ready returns 200"
      - "Alertmanager /-/healthy returns 200"
      - "Sentry HTTP returns 200, 302, or 400"
      - "runner CPUQuota=200%, MemoryMax=2G, WatchdogUSec=0"
    blocks:
      - "K3s image pulls"
      - "runtime CD"
      - "alert rules deploy"
      - "code-review runners"

  - id: "P1-K3S"
    order: 3
    start_after:
      - "P0-188-DATA"
      - "P0-110-REGISTRY-OBSERVABILITY"
    hosts:
      - "120"
      - "121"
    service_order:
      - "120 k3s.service"
      - "121 k3s-agent.service or live role"
      - "CNI / kube-proxy"
      - "nodes Ready"
      - "core pods"
      - "awoooi-prod pods"
      - "keepalived VIP 192.168.0.125"
      - "NodePorts 32334 and 32335"
    gates:
      - "120 can reach 188:5432"
      - "K3s nodes show Ready"
      - "VIP 192.168.0.125 is present"
      - "awoooi-prod pods are Running or Completed"
    blocks:
      - "AWOOOI workload health"
      - "public AWOOOI route"
      - "Alertmanager webhook"

  - id: "P2-WORKLOAD-ALERTCHAIN"
    order: 4
    start_after:
      - "P1-K3S"
    owners:
      - "AWOOOI API"
      - "AWOOOI Web"
      - "Alertmanager webhook"
      - "Telegram delivery"
    gates:
      - "http://192.168.0.125:32334/api/v1/health returns 2xx/3xx"
      - "http://192.168.0.125:32335/ returns 2xx/3xx"
      - "Alertmanager webhook POST returns 2xx"
      - "K8s Telegram secrets are present and non-placeholder"
    blocks:
      - "AI auto-remediation"
      - "full alert confidence"

  - id: "P2-PUBLIC-ROUTES"
    order: 5
    start_after:
      - "P2-WORKLOAD-ALERTCHAIN"
    gates:
      - "https://awoooi.wooo.work/api/v1/health returns 2xx/3xx"
      - "https://awoooi.wooo.work/ returns 2xx/3xx"
      - "https://mo.wooo.work/ returns 2xx/3xx"
      - "https://mo.wooo.work/health returns 2xx/3xx"
    blocks:
      - "external release complete"

  - id: "P2-SCHEDULES"
    order: 6
    start_after:
      - "P2-PUBLIC-ROUTES"
    gates:
      - "110/120/121/188 cron services active"
      - "188 backup-from-110 success age below 25h"
      - "188 docker restart/stats textfiles fresh"
      - "188 momo-scheduler container healthy and registration evidence present within 6h"
      - "110 docker/systemd textfiles fresh"
      - "120 awoooi-prod CronJobs present and unsuspended"
      - "120 awoooi-prod has no failed Jobs"
      - "121 DR drill cron present"
    blocks:
      - "done criteria"
      - "AI auto-remediation release"

  - id: "P3-HIGH-LOAD-RELEASE"
    order: 7
    start_after:
      - "P2-SCHEDULES"
    release_last:
      - "momo-scheduler / Chrome crawlers"
      - "Sentry Snuba consumers"
      - "SignOz ClickHouse merge-heavy work"
      - "Gitea actions runners"
      - "runtime CD jobs"
    gates:
      - "all prior gates green"
      - "host load per CPU below 1.0 for 15 minutes before releasing batch/runner work"
      - "ClickHouse/Kafka/Snuba backlog decreasing for two consecutive checks if backlog exists"

baselines:
  endpoints:
    awoooi_vip_api_health: "http://192.168.0.125:32334/api/v1/health"
    awoooi_vip_web: "http://192.168.0.125:32335/"
    awoooi_public_api_health: "https://awoooi.wooo.work/api/v1/health"
    awoooi_public_web: "https://awoooi.wooo.work/"
    momo_public_web: "https://mo.wooo.work/"
    momo_public_health: "https://mo.wooo.work/health"
    harbor_registry: "http://127.0.0.1:5000/v2/"
    gitea: "http://127.0.0.1:3001/"
    prometheus_ready: "http://127.0.0.1:9090/-/ready"
    alertmanager_healthy: "http://127.0.0.1:9093/-/healthy"
    sentry: "http://127.0.0.1:9000/"
  expected_codes:
    harbor_registry:
      - 200
      - 401
    gitea:
      - 200
      - 302
    prometheus_ready:
      - 200
    alertmanager_healthy:
      - 200
    sentry:
      - 200
      - 302
      - 400
    workload_and_public:
      - "2xx"
      - "3xx"
  runner_guardrails:
    CPUQuotaPerSecUSec: "2s"
    MemoryMax: "2147483648"
    WatchdogUSec: "0"
  freshness_seconds:
    docker_textfiles: 300
    systemd_textfiles: 300
    backup_success: 90000

stateful_services:
  hard_block_auto_repair:
    - "188 PostgreSQL data directory"
    - "188 k3s_datastore"
    - "188 momo database"
    - "110 Harbor DB"
    - "110 Sentry DB"
    - "Sentry ClickHouse data"
    - "SignOz ClickHouse data"
    - "Kafka topic/log directories"
  human_in_loop_required:
    - "pg_resetwal"
    - "ClickHouse clean-clone recovery"
    - "Kafka checkpoint file quarantine"
    - "backup restore"
    - "filesystem repair"

ai_automation_gate:
  observe_only_until:
    - "P0-NETWORK green"
    - "P0-188-DATA green"
    - "P0-110-REGISTRY-OBSERVABILITY green"
    - "P1-K3S green"
    - "P2-WORKLOAD-ALERTCHAIN green"
    - "P2-PUBLIC-ROUTES green"
    - "P2-SCHEDULES green"
    - "no active restart storm"
    - "host load per CPU below 1.0 for 15 minutes"
  allowed_before_green:
    - "diagnose"
    - "collect evidence"
    - "notify"
  blocked_before_green:
    - "stateful restart"
    - "destructive repair"
    - "runner/CD release"
    - "generic container restart"

persistent_monitoring:
  host: "110"
  install_command: "bash scripts/reboot-recovery/install-cold-start-monitor-110.sh"
  schedule: "*/10 * * * *"
  mode: "read_only"
  send_alert_test: false
  scripts:
    check: "/home/wooo/scripts/full-stack-cold-start-check.sh"
    exporter: "/home/wooo/scripts/cold-start-textfile-exporter.sh"
  outputs:
    textfile: "/home/wooo/node_exporter_textfiles/cold_start_recovery.prom"
    last_log: "/home/wooo/reboot-recovery/cold-start-last.log"
  metrics:
    - "awoooi_cold_start_monitor_up"
    - "awoooi_cold_start_pass_gates"
    - "awoooi_cold_start_warn_gates"
    - "awoooi_cold_start_blocked_gates"
    - "awoooi_cold_start_last_run_timestamp"
    - "awoooi_cold_start_last_green_timestamp"
    - "awoooi_cold_start_last_result"
  prometheus_alerts:
    - "ColdStartMonitorMissing"
    - "ColdStartMonitorStale"
    - "ColdStartRecoveryBlocked"
    - "ColdStartRecoveryDegraded"
    - "ColdStartLastGreenTooOld"
  ai_contract:
    monitor_missing: "diagnose cron/textfile path only"
    stale: "collect cron log and last check log"
    degraded: "collect evidence, do not release high-load work"
    blocked: "follow first BLOCKED gate in phase order"
    forbidden: "generic restart, stateful restart, destructive repair"

final_confirmation:
  command: "bash scripts/reboot-recovery/full-stack-cold-start-check.sh --watch --interval 60 --max-attempts 30 --send-alert-test"
  green_result:
    PASS: "greater than 0"
    WARN: 0
    BLOCKED: 0
    summary: "Result: GREEN"
