version: 2026-05-06.v1 scope: included_hosts: "110": "DevOps, registry, observability, Sentry, runners" "120": "K3s control plane and VIP" "121": "K3s peer node and DR drill cron" "188": "Data, AI, web, momo, SignOz, public nginx gateway" excluded_hosts: "112": "Kali security host; recorded but not part of cold-start release gate" principles: - recover_dependency_chain_before_workloads - keep_ai_auto_repair_observe_only_until_green - never_generic_restart_stateful_services - preserve_corrupt_parts_in_quarantine_not_delete - release_runners_and_crawlers_last phases: - id: P0-NETWORK order: 0 gates: - ping_110_120_121_188 - ssh_port_110_120_121_188 - arp_evidence_or_monitor_mode_fallback - id: P0-188-DATA order: 10 required_before: - P1-K3S - P2-WORKLOAD-ALERTCHAIN gates: - containerd_docker_postgresql_redis_ollama_nginx_active - postgresql_5432_accepting_connections - redis_pong - momo_db_not_restarting - signoz_http_reachable - momo_health_200 - id: P0-110-REGISTRY-OBSERVABILITY order: 20 required_before: - P1-K3S - P3-RUNNER-CD gates: - docker_active - harbor_v2_200_or_401 - gitea_200_or_302 - prometheus_ready - alertmanager_healthy - sentry_http_reachable - docker_containers_all_up - runner_watchdog_disabled - sentry_clickhouse_not_restarting - cadvisor_image_v0_47_0 - cadvisor_cpu_cap_0_3 - id: P1-K3S order: 30 gates: - 120_can_reach_188_postgres - mon_and_mon1_ready - no_non_running_non_succeeded_pods - awoooi_dev_api_nodeport_200 - vip_192_168_0_125_present - id: P2-WORKLOAD-ALERTCHAIN order: 40 gates: - awoooi_api_vip_health_2xx_or_3xx - awoooi_web_vip_2xx_or_3xx - alertmanager_webhook_e2e_2xx_when_release_gate - id: P2-PUBLIC-ROUTES order: 50 public_https_routes: - https://awoooi.wooo.work/api/v1/health - https://awoooi.wooo.work/ - https://awoooi.wooo.work/zh-TW/iwooos - https://vibework.wooo.work/ - https://awooogo.wooo.work/ - https://2026fifa.wooo.work/ - https://agent.wooo.work/ - https://mo.wooo.work/ - https://mo.wooo.work/health - https://stock.wooo.work/ - https://stock.wooo.work/healthz - https://stock.wooo.work/api/healthz - https://bitan.wooo.work/ - https://tsenyang.com/ - https://www.tsenyang.com/ - https://vtuber.wooo.work/ - https://gitea.wooo.work/ - https://harbor.wooo.work/ - https://registry.wooo.work/ - https://sentry.wooo.work/ - https://signoz.wooo.work/ - https://langfuse.wooo.work/ - https://aiops.wooo.work/ - id: P2-SCHEDULES order: 60 gates: - cron_active_188_110_120_121 - docker_restart_textfile_fresh_188 - docker_stats_textfile_fresh_188_110 - systemd_units_textfile_fresh_110 - backup_health_textfile_fresh_188_110 - backup_from_110_success_under_25h - expected_backup_jobs_fresh_188_110 - host_service_config_backup_success_under_48h - sentry_dedicated_backup_success_under_48h - backup_integrity_check_success_under_8d - backup_restore_drill_success_under_31d - velero_schedule_present_and_latest_completed_under_25h - velero_restore_test_cron_present - momo_scheduler_registered_jobs - momo_import_config_daily_sales_intake - momo_source_absence_evidence_when_freshness_blocked - stockplatform_system_freshness_ok - stockplatform_latest_trading_day_sources_current - k8s_cronjobs_unsuspended - k8s_failed_jobs_zero - dr_drill_cron_present_121 - id: P3-HIGH-LOAD-WORK order: 70 release_after: - P0-NETWORK - P0-188-DATA - P0-110-REGISTRY-OBSERVABILITY - P1-K3S - P2-WORKLOAD-ALERTCHAIN - P2-PUBLIC-ROUTES - P2-SCHEDULES release_conditions: - host_load_per_core_below_1_0_for_15m - no_restart_storm - clickhouse_merge_or_kafka_lag_not_increasing_two_checks examples: - sentry_snuba_consumers - momo_scheduler_chrome_crawlers - gitea_actions_jobs - id: P3-RUNNER-CD order: 80 release_conditions: - all_previous_gates_green - runner_cpuquota_200_percent - runner_memorymax_2g - watchdogusec_0 - active_awoooi_cd_or_gitea_actions_task_containers_cpu_capped_during_cold_start automation_policy: before_green: ai_auto_repair: observe_only alertmanager_smoke_test: manual_or_release_gate_only stateful_service_actions: human_approval_required generic_restart: forbidden after_green: ai_auto_repair: limited_execution_for_stateless_exporters_only stateful_service_actions: human_in_the_loop runner_cd: controlled_release plan_b: purpose: degraded_operation_and_recovery_path not_authorization: - docker_daemon_restart - nginx_reload - firewall_or_iptables_change - kubectl_patch_live - secret_read - destructive_recovery - production_write red_lines: - do_not_call_route_200_full_stack_green - do_not_silence_correct_red_lights - do_not_run_unapproved_live_writes - do_not_release_high_risk_automation_before_dependency_chain_green triggers: - id: backup_or_offsite_running condition: "02:00 backup, 03:00 offsite sync, or full verifier is still running" immediate_action: abort_or_delay_reboot_read_only_wait max_declaration: B0_ABORTED_BEFORE_REBOOT - id: p0_host_unreachable_after_15m condition: any_p0_host_ping_or_ssh_unreachable_after_reboot_15m immediate_action: stop_next_phase_and_enter_host_plan_b max_declaration: B1_HOST_RECOVERY_ONLY - id: data_188_unhealthy condition: postgresql_redis_momo_signoz_or_ai_provider_route_unhealthy immediate_action: freeze_k3s_deploy_runner_ai_auto_remediation max_declaration: B1_HOST_RECOVERY_ONLY - id: registry_observability_110_unhealthy condition: harbor_gitea_prometheus_or_alertmanager_unhealthy immediate_action: freeze_cd_deploy_image_pull_and_alert_outbound max_declaration: B2_CORE_SERVICE_READY - id: single_k3s_control_plane_degraded condition: one_of_120_or_121_unhealthy_while_peer_can_carry_control_plane immediate_action: keep_single_node_k3s_service_mode_and_preserve_ha_red_light max_declaration: B2_CORE_SERVICE_READY - id: route_green_only condition: public_route_green_but_db_backup_alert_or_scheduler_not_green immediate_action: record_route_green_only_and_continue_cross_surface_checks max_declaration: B2_CORE_SERVICE_READY - id: cold_start_warn_no_blocked condition: cold_start_warn_greater_than_zero_and_blocked_zero immediate_action: declare_available_degraded_with_explicit_warns max_declaration: B3_SERVICE_AVAILABLE_DEGRADED - id: credential_escrow_missing condition: credential_escrow_missing_count_greater_than_zero immediate_action: keep_dr_red_light_and_forbid_dr_complete_claim max_declaration: B4_FULL_STACK_GREEN host_paths: "110": degraded_path: preserve_k3s_and_188_data_freeze_cd_runner_harbor_push_alert_outbound return_to_plan_a: - host_ready - harbor_gitea_prometheus_alertmanager_healthy - backup_status_no_110_core_blocker - cold_start_110_checks_green "120": degraded_path: let_121_carry_k3s_preserve_120_degraded_red_light_forbid_k3s_aa_claim return_to_plan_a: - ping_ok - ssh_ok - root_filesystem_rw - k3s_active - node_mon_ready - backup_configs_backup_all_offsite_cold_start_chain_green "121": degraded_path: let_120_carry_k3s_preserve_121_degraded_red_light_forbid_workload_balanced_claim return_to_plan_a: - ping_ok - ssh_ok - k3s_active - node_mon1_ready - api_web_placement_max_skew_less_or_equal_1 "188": degraded_path: preserve_data_layer_freeze_batch_crawler_ai_flows return_to_plan_a: - host_ready - postgresql_healthy - redis_healthy - momo_parity_green - signoz_healthy - ai_provider_route_healthy - backup_status_no_188_core_blocker k3s: degraded_path: keep_healthy_pods_read_nodes_pods_events_vip_nodeport_no_blind_restart return_to_plan_a: - mon_ready - mon1_ready - api_web_worker_rollout_healthy - public_api_web_alert_webhook_scorecard_green public_gateway: degraded_path: preserve_internal_api_vip_data_no_unapproved_nginx_dns_tls_certbot_firewall_change return_to_plan_a: - nginx_config_owner_evidence - route_smoke - tls_acme_check - rollback_owner - post_check_plan service_levels: B0_ABORTED_BEFORE_REBOOT: preflight_no_go_no_runtime_write B1_HOST_RECOVERY_ONLY: host_layer_recovered_service_not_fully_verified B2_CORE_SERVICE_READY: core_service_available_cross_surface_checks_incomplete B3_SERVICE_AVAILABLE_DEGRADED: cold_start_blocked_zero_warn_greater_than_zero B4_FULL_STACK_GREEN: cold_start_warn_zero_blocked_zero_backup_offsite_db_alert_scheduler_green B5_DR_COMPLETE: b4_plus_credential_escrow_missing_zero_restore_and_escrow_evidence_complete timeline: T+0: freeze_cd_runner_ai_auto_remediation_heavy_batch_preserve_evidence T+5: decide_host_powered_booted_ready_enter_host_plan_b_if_needed T+15: stop_if_188_data_or_110_registry_observability_unhealthy T+30: route_green_only_guard_b2_ceiling T+60: run_cold_start_scorecard_and_record_warn_blocked T+120: open_incident_or_followup_no_unapproved_runtime_write closeout_states: RETURNED_TO_PLAN_A: blocker_cleared_and_full_plan_a_chain_verified SERVICE_AVAILABLE_DEGRADED: service_available_but_warn_dr_escrow_or_governance_gate_open OPEN_INCIDENT_REQUIRED: p0_host_data_k3s_gateway_backup_or_alert_hard_blocked resource_guardrails: "110": cadvisor: image: gcr.io/cadvisor/cadvisor:v0.47.0 cpus: 0.3 mem_limit: 512m sentry_snuba_cold_start_consumers: cpus: 0.5 persist_in: /opt/sentry/docker-compose.override.yml sentry_self_hosted_memory_limits: taskscheduler_mem_limit: 1g relay_mem_limit: 3g relay_memswap_limit: 4g persist_in: /opt/sentry/docker-compose.override.yml note: "taskscheduler 不得回退到 512m/1g；relay 不得回退到 2g 或以下造成長期 >85% memory-limit pressure；110 主機仍以 ClickHouse/Kafka/Snuba CPU caps 防止冷啟動過載。" actions_runner_systemd: cpu_quota: 200% memory_max: 2G watchdog: disabled gitea: cpus: 1.5 memory: 3G restart_policy: always note: "2026-07-02 live recovery 將 gitea 從 2 CPU cap 收斂到 1.5 CPU；不得回退到無限額或 3 CPU，rollback 只允許暫回 2 CPU 並需 Prometheus readback。" stockplatform_v2_postgres: cpus: 1.5 memory: 4G memswap: 6G restart_policy: unless-stopped note: "2026-07-02 live recovery 將 stockplatform-v2-postgres-1 補上 Docker resource guardrail；不得回退到 nanocpus=0 / memory=0 / restart=no 造成 110 host pressure。" "188": ollama_systemd: cpu_quota: 300% memory_high: 20G memory_max: 24G max_loaded_models: 1 num_parallel: 1 note: "188 本機 Ollama 是 cold-start 依賴與 Open-WebUI local endpoint；不得維持 disabled/inactive，也不得保留 700%/45G 無節制 guardrail。" litellm: mode: optional_retired note: "188 currently has no litellm container, unit, port 4000, or /opt/litellm tree. Do not hard-start a provider proxy without provider route/cost approval; P3 gate treats absent litellm as warning evidence." momo_scheduler: cpus: 2.0 memory: 2G momo_db: cpus: 2.0 memory: 4G memswap: 6G restart_policy: unless-stopped note: "2026-07-01 live recovery 將 momo-db 補上 Docker resource guardrail；不得回退到 nanocpus=0 / memory=0 / memswap=0。" signoz_clickhouse: memory: 24G note: do_not_lower_during_merge_backlog authoritative_checks: read_only_monitor: command: bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color expected_for_cron: PASS>0 WARN=0 BLOCKED=0 release_gate: command: SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test expected: PASS=64 WARN=0 BLOCKED=0 textfile_metric: path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1 backup_baseline: path: ops/reboot-recovery/full-stack-backup-baseline.yml required_metrics: - awoooi_backup_health_monitor_up - awoooi_backup_job_fresh - awoooi_backup_integrity_fresh - awoooi_velero_restore_test_cron_present - awoooi_velero_restore_test_last_success_fresh