353 lines
13 KiB
YAML
353 lines
13 KiB
YAML
version: 2026-05-06.v1
|
||
scope:
|
||
included_hosts:
|
||
"110": "DevOps, registry, observability, Sentry, runners"
|
||
"120": "K3s control plane and VIP"
|
||
"121": "K3s peer node and DR drill cron"
|
||
"188": "Data, AI, web, momo, SignOz, public nginx gateway"
|
||
excluded_hosts:
|
||
"112": "Kali security host; recorded but not part of cold-start release gate"
|
||
|
||
principles:
|
||
- recover_dependency_chain_before_workloads
|
||
- keep_ai_auto_repair_observe_only_until_green
|
||
- never_generic_restart_stateful_services
|
||
- preserve_corrupt_parts_in_quarantine_not_delete
|
||
- release_runners_and_crawlers_last
|
||
|
||
phases:
|
||
- id: P0-NETWORK
|
||
order: 0
|
||
gates:
|
||
- ping_110_120_121_188
|
||
- ssh_port_110_120_121_188
|
||
- arp_evidence_or_monitor_mode_fallback
|
||
|
||
- id: P0-188-DATA
|
||
order: 10
|
||
required_before:
|
||
- P1-K3S
|
||
- P2-WORKLOAD-ALERTCHAIN
|
||
gates:
|
||
- containerd_docker_postgresql_redis_ollama_nginx_active
|
||
- postgresql_5432_accepting_connections
|
||
- redis_pong
|
||
- momo_db_not_restarting
|
||
- signoz_http_reachable
|
||
- momo_health_200
|
||
|
||
- id: P0-110-REGISTRY-OBSERVABILITY
|
||
order: 20
|
||
required_before:
|
||
- P1-K3S
|
||
- P3-RUNNER-CD
|
||
gates:
|
||
- docker_active
|
||
- harbor_v2_200_or_401
|
||
- gitea_200_or_302
|
||
- prometheus_ready
|
||
- alertmanager_healthy
|
||
- sentry_http_reachable
|
||
- docker_containers_all_up
|
||
- runner_watchdog_disabled
|
||
- sentry_clickhouse_not_restarting
|
||
- cadvisor_image_v0_47_0
|
||
- cadvisor_cpu_cap_0_3
|
||
|
||
- id: P1-K3S
|
||
order: 30
|
||
gates:
|
||
- 120_can_reach_188_postgres
|
||
- mon_and_mon1_ready
|
||
- no_non_running_non_succeeded_pods
|
||
- awoooi_dev_api_nodeport_200
|
||
- vip_192_168_0_125_present
|
||
|
||
- id: P2-WORKLOAD-ALERTCHAIN
|
||
order: 40
|
||
gates:
|
||
- awoooi_api_vip_health_2xx_or_3xx
|
||
- awoooi_web_vip_2xx_or_3xx
|
||
- alertmanager_webhook_e2e_2xx_when_release_gate
|
||
|
||
- id: P2-PUBLIC-ROUTES
|
||
order: 50
|
||
public_https_routes:
|
||
- https://awoooi.wooo.work/api/v1/health
|
||
- https://awoooi.wooo.work/
|
||
- https://awoooi.wooo.work/zh-TW/iwooos
|
||
- https://vibework.wooo.work/
|
||
- https://awooogo.wooo.work/
|
||
- https://2026fifa.wooo.work/
|
||
- https://agent.wooo.work/
|
||
- https://mo.wooo.work/
|
||
- https://mo.wooo.work/health
|
||
- https://stock.wooo.work/
|
||
- https://stock.wooo.work/healthz
|
||
- https://stock.wooo.work/api/healthz
|
||
- https://bitan.wooo.work/
|
||
- https://tsenyang.com/
|
||
- https://www.tsenyang.com/
|
||
- https://vtuber.wooo.work/
|
||
- https://gitea.wooo.work/
|
||
- https://harbor.wooo.work/
|
||
- https://registry.wooo.work/
|
||
- https://sentry.wooo.work/
|
||
- https://signoz.wooo.work/
|
||
- https://langfuse.wooo.work/
|
||
- https://aiops.wooo.work/
|
||
|
||
- id: P2-SCHEDULES
|
||
order: 60
|
||
gates:
|
||
- cron_active_188_110_120_121
|
||
- docker_restart_textfile_fresh_188
|
||
- docker_stats_textfile_fresh_188_110
|
||
- systemd_units_textfile_fresh_110
|
||
- backup_health_textfile_fresh_188_110
|
||
- backup_from_110_success_under_25h
|
||
- expected_backup_jobs_fresh_188_110
|
||
- host_service_config_backup_success_under_48h
|
||
- sentry_dedicated_backup_success_under_48h
|
||
- backup_integrity_check_success_under_8d
|
||
- backup_restore_drill_success_under_31d
|
||
- velero_schedule_present_and_latest_completed_under_25h
|
||
- velero_restore_test_cron_present
|
||
- momo_scheduler_registered_jobs
|
||
- momo_import_config_daily_sales_intake
|
||
- momo_source_absence_evidence_when_freshness_blocked
|
||
- stockplatform_system_freshness_ok
|
||
- stockplatform_latest_trading_day_sources_current
|
||
- k8s_cronjobs_unsuspended
|
||
- k8s_failed_jobs_zero
|
||
- dr_drill_cron_present_121
|
||
|
||
- id: P3-HIGH-LOAD-WORK
|
||
order: 70
|
||
release_after:
|
||
- P0-NETWORK
|
||
- P0-188-DATA
|
||
- P0-110-REGISTRY-OBSERVABILITY
|
||
- P1-K3S
|
||
- P2-WORKLOAD-ALERTCHAIN
|
||
- P2-PUBLIC-ROUTES
|
||
- P2-SCHEDULES
|
||
release_conditions:
|
||
- host_load_per_core_below_1_0_for_15m
|
||
- no_restart_storm
|
||
- clickhouse_merge_or_kafka_lag_not_increasing_two_checks
|
||
examples:
|
||
- sentry_snuba_consumers
|
||
- momo_scheduler_chrome_crawlers
|
||
- gitea_actions_jobs
|
||
|
||
- id: P3-RUNNER-CD
|
||
order: 80
|
||
release_conditions:
|
||
- all_previous_gates_green
|
||
- runner_cpuquota_200_percent
|
||
- runner_memorymax_2g
|
||
- watchdogusec_0
|
||
- active_awoooi_cd_or_gitea_actions_task_containers_cpu_capped_during_cold_start
|
||
|
||
automation_policy:
|
||
before_green:
|
||
ai_auto_repair: observe_only
|
||
alertmanager_smoke_test: manual_or_release_gate_only
|
||
stateful_service_actions: human_approval_required
|
||
generic_restart: forbidden
|
||
after_green:
|
||
ai_auto_repair: limited_execution_for_stateless_exporters_only
|
||
stateful_service_actions: human_in_the_loop
|
||
runner_cd: controlled_release
|
||
|
||
plan_b:
|
||
purpose: degraded_operation_and_recovery_path
|
||
not_authorization:
|
||
- docker_daemon_restart
|
||
- nginx_reload
|
||
- firewall_or_iptables_change
|
||
- kubectl_patch_live
|
||
- secret_read
|
||
- destructive_recovery
|
||
- production_write
|
||
red_lines:
|
||
- do_not_call_route_200_full_stack_green
|
||
- do_not_silence_correct_red_lights
|
||
- do_not_run_unapproved_live_writes
|
||
- do_not_release_high_risk_automation_before_dependency_chain_green
|
||
triggers:
|
||
- id: backup_or_offsite_running
|
||
condition: "02:00 backup, 03:00 offsite sync, or full verifier is still running"
|
||
immediate_action: abort_or_delay_reboot_read_only_wait
|
||
max_declaration: B0_ABORTED_BEFORE_REBOOT
|
||
- id: p0_host_unreachable_after_15m
|
||
condition: any_p0_host_ping_or_ssh_unreachable_after_reboot_15m
|
||
immediate_action: stop_next_phase_and_enter_host_plan_b
|
||
max_declaration: B1_HOST_RECOVERY_ONLY
|
||
- id: data_188_unhealthy
|
||
condition: postgresql_redis_momo_signoz_or_ai_provider_route_unhealthy
|
||
immediate_action: freeze_k3s_deploy_runner_ai_auto_remediation
|
||
max_declaration: B1_HOST_RECOVERY_ONLY
|
||
- id: registry_observability_110_unhealthy
|
||
condition: harbor_gitea_prometheus_or_alertmanager_unhealthy
|
||
immediate_action: freeze_cd_deploy_image_pull_and_alert_outbound
|
||
max_declaration: B2_CORE_SERVICE_READY
|
||
- id: single_k3s_control_plane_degraded
|
||
condition: one_of_120_or_121_unhealthy_while_peer_can_carry_control_plane
|
||
immediate_action: keep_single_node_k3s_service_mode_and_preserve_ha_red_light
|
||
max_declaration: B2_CORE_SERVICE_READY
|
||
- id: route_green_only
|
||
condition: public_route_green_but_db_backup_alert_or_scheduler_not_green
|
||
immediate_action: record_route_green_only_and_continue_cross_surface_checks
|
||
max_declaration: B2_CORE_SERVICE_READY
|
||
- id: cold_start_warn_no_blocked
|
||
condition: cold_start_warn_greater_than_zero_and_blocked_zero
|
||
immediate_action: declare_available_degraded_with_explicit_warns
|
||
max_declaration: B3_SERVICE_AVAILABLE_DEGRADED
|
||
- id: credential_escrow_missing
|
||
condition: credential_escrow_missing_count_greater_than_zero
|
||
immediate_action: keep_dr_red_light_and_forbid_dr_complete_claim
|
||
max_declaration: B4_FULL_STACK_GREEN
|
||
host_paths:
|
||
"110":
|
||
degraded_path: preserve_k3s_and_188_data_freeze_cd_runner_harbor_push_alert_outbound
|
||
return_to_plan_a:
|
||
- host_ready
|
||
- harbor_gitea_prometheus_alertmanager_healthy
|
||
- backup_status_no_110_core_blocker
|
||
- cold_start_110_checks_green
|
||
"120":
|
||
degraded_path: let_121_carry_k3s_preserve_120_degraded_red_light_forbid_k3s_aa_claim
|
||
return_to_plan_a:
|
||
- ping_ok
|
||
- ssh_ok
|
||
- root_filesystem_rw
|
||
- k3s_active
|
||
- node_mon_ready
|
||
- backup_configs_backup_all_offsite_cold_start_chain_green
|
||
"121":
|
||
degraded_path: let_120_carry_k3s_preserve_121_degraded_red_light_forbid_workload_balanced_claim
|
||
return_to_plan_a:
|
||
- ping_ok
|
||
- ssh_ok
|
||
- k3s_active
|
||
- node_mon1_ready
|
||
- api_web_placement_max_skew_less_or_equal_1
|
||
"188":
|
||
degraded_path: preserve_data_layer_freeze_batch_crawler_ai_flows
|
||
return_to_plan_a:
|
||
- host_ready
|
||
- postgresql_healthy
|
||
- redis_healthy
|
||
- momo_parity_green
|
||
- signoz_healthy
|
||
- ai_provider_route_healthy
|
||
- backup_status_no_188_core_blocker
|
||
k3s:
|
||
degraded_path: keep_healthy_pods_read_nodes_pods_events_vip_nodeport_no_blind_restart
|
||
return_to_plan_a:
|
||
- mon_ready
|
||
- mon1_ready
|
||
- api_web_worker_rollout_healthy
|
||
- public_api_web_alert_webhook_scorecard_green
|
||
public_gateway:
|
||
degraded_path: preserve_internal_api_vip_data_no_unapproved_nginx_dns_tls_certbot_firewall_change
|
||
return_to_plan_a:
|
||
- nginx_config_owner_evidence
|
||
- route_smoke
|
||
- tls_acme_check
|
||
- rollback_owner
|
||
- post_check_plan
|
||
service_levels:
|
||
B0_ABORTED_BEFORE_REBOOT: preflight_no_go_no_runtime_write
|
||
B1_HOST_RECOVERY_ONLY: host_layer_recovered_service_not_fully_verified
|
||
B2_CORE_SERVICE_READY: core_service_available_cross_surface_checks_incomplete
|
||
B3_SERVICE_AVAILABLE_DEGRADED: cold_start_blocked_zero_warn_greater_than_zero
|
||
B4_FULL_STACK_GREEN: cold_start_warn_zero_blocked_zero_backup_offsite_db_alert_scheduler_green
|
||
B5_DR_COMPLETE: b4_plus_credential_escrow_missing_zero_restore_and_escrow_evidence_complete
|
||
timeline:
|
||
T+0: freeze_cd_runner_ai_auto_remediation_heavy_batch_preserve_evidence
|
||
T+5: decide_host_powered_booted_ready_enter_host_plan_b_if_needed
|
||
T+15: stop_if_188_data_or_110_registry_observability_unhealthy
|
||
T+30: route_green_only_guard_b2_ceiling
|
||
T+60: run_cold_start_scorecard_and_record_warn_blocked
|
||
T+120: open_incident_or_followup_no_unapproved_runtime_write
|
||
closeout_states:
|
||
RETURNED_TO_PLAN_A: blocker_cleared_and_full_plan_a_chain_verified
|
||
SERVICE_AVAILABLE_DEGRADED: service_available_but_warn_dr_escrow_or_governance_gate_open
|
||
OPEN_INCIDENT_REQUIRED: p0_host_data_k3s_gateway_backup_or_alert_hard_blocked
|
||
|
||
resource_guardrails:
|
||
"110":
|
||
cadvisor:
|
||
image: gcr.io/cadvisor/cadvisor:v0.47.0
|
||
cpus: 0.3
|
||
mem_limit: 512m
|
||
sentry_snuba_cold_start_consumers:
|
||
cpus: 0.5
|
||
persist_in: /opt/sentry/docker-compose.override.yml
|
||
sentry_self_hosted_memory_limits:
|
||
taskscheduler_mem_limit: 1g
|
||
relay_mem_limit: 3g
|
||
relay_memswap_limit: 4g
|
||
persist_in: /opt/sentry/docker-compose.override.yml
|
||
note: "taskscheduler 不得回退到 512m/1g；relay 不得回退到 2g 或以下造成長期 >85% memory-limit pressure；110 主機仍以 ClickHouse/Kafka/Snuba CPU caps 防止冷啟動過載。"
|
||
actions_runner_systemd:
|
||
cpu_quota: 200%
|
||
memory_max: 2G
|
||
watchdog: disabled
|
||
gitea:
|
||
cpus: 1.5
|
||
memory: 3G
|
||
restart_policy: always
|
||
note: "2026-07-02 live recovery 將 gitea 從 2 CPU cap 收斂到 1.5 CPU；不得回退到無限額或 3 CPU，rollback 只允許暫回 2 CPU 並需 Prometheus readback。"
|
||
stockplatform_v2_postgres:
|
||
cpus: 1.5
|
||
memory: 4G
|
||
memswap: 6G
|
||
restart_policy: unless-stopped
|
||
note: "2026-07-02 live recovery 將 stockplatform-v2-postgres-1 補上 Docker resource guardrail；不得回退到 nanocpus=0 / memory=0 / restart=no 造成 110 host pressure。"
|
||
"188":
|
||
ollama_systemd:
|
||
cpu_quota: 300%
|
||
memory_high: 20G
|
||
memory_max: 24G
|
||
max_loaded_models: 1
|
||
num_parallel: 1
|
||
note: "188 本機 Ollama 是 cold-start 依賴與 Open-WebUI local endpoint；不得維持 disabled/inactive，也不得保留 700%/45G 無節制 guardrail。"
|
||
litellm:
|
||
mode: optional_retired
|
||
note: "188 currently has no litellm container, unit, port 4000, or /opt/litellm tree. Do not hard-start a provider proxy without provider route/cost approval; P3 gate treats absent litellm as warning evidence."
|
||
momo_scheduler:
|
||
cpus: 2.0
|
||
memory: 2G
|
||
momo_db:
|
||
cpus: 2.0
|
||
memory: 4G
|
||
memswap: 6G
|
||
restart_policy: unless-stopped
|
||
note: "2026-07-01 live recovery 將 momo-db 補上 Docker resource guardrail；不得回退到 nanocpus=0 / memory=0 / memswap=0。"
|
||
signoz_clickhouse:
|
||
memory: 24G
|
||
note: do_not_lower_during_merge_backlog
|
||
|
||
authoritative_checks:
|
||
read_only_monitor:
|
||
command: bash scripts/reboot-recovery/full-stack-cold-start-check.sh --monitor-read-only --no-color
|
||
expected_for_cron: PASS>0 WARN=0 BLOCKED=0
|
||
release_gate:
|
||
command: SSH_BATCH_MODE=yes bash scripts/reboot-recovery/full-stack-cold-start-check.sh --send-alert-test
|
||
expected: PASS=64 WARN=0 BLOCKED=0
|
||
textfile_metric:
|
||
path: /home/wooo/node_exporter_textfiles/cold_start_recovery.prom
|
||
green_metric: awoooi_cold_start_last_result{host="110",scope="110_120_121_188",result="green"} 1
|
||
backup_baseline:
|
||
path: ops/reboot-recovery/full-stack-backup-baseline.yml
|
||
required_metrics:
|
||
- awoooi_backup_health_monitor_up
|
||
- awoooi_backup_job_fresh
|
||
- awoooi_backup_integrity_fresh
|
||
- awoooi_velero_restore_test_cron_present
|
||
- awoooi_velero_restore_test_last_success_fresh
|