diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 9bde1c08b..597c2f492 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -11,6 +11,33 @@ from src.services.reboot_auto_recovery_slo_scorecard import ( load_latest_reboot_auto_recovery_slo_scorecard, ) +EXPECTED_REBOOT_SLO_BLOCKERS = [ + "all_required_hosts_not_in_10_minute_reboot_window", + "backup_core_green_not_1", + "host_188_service_green_not_1", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "local_disk_free_below_minimum", + "post_start_blocked_not_zero", + "product_data_green_not_1", + "reboot_event_required_host_unreachable", + "service_green_not_1", + "wazuh_dashboard_degraded", +] + +EXPECTED_DRILL_PREFLIGHT_BLOCKERS = [ + "service_green_not_ready", + "product_data_green_not_ready", + "backup_core_green_not_ready", + "host_188_service_green_not_ready", + "all_required_hosts_reachable_not_ready", + "latest_verify_only_metric_present_not_ready", + "stockplatform_freshness_ok_not_ready", + "stockplatform_ingestion_ok_not_ready", + "blocked_only_by_fresh_reboot_window_not_ready", +] + def test_reboot_auto_recovery_slo_scorecard_loader_exposes_stockplatform_gate(): payload = load_latest_reboot_auto_recovery_slo_scorecard() @@ -51,78 +78,75 @@ def _assert_reboot_slo_payload(payload: dict): assert payload["priority"] == "P0-006" assert payload["status"] == "blocked_reboot_auto_recovery_slo_not_ready" assert payload["safe_next_step"] == ( - "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_" - "event_or_approved_reboot_drill_to_prove_10_minute_slo" + "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_" + "rerun_scorecard_until_status_slo_ready" ) assert payload["can_claim_all_services_recovered_within_target"] is False - assert payload["active_blocker_count"] == 1 - assert payload["readiness_percent"] == 82 - assert payload["service_green"] is True - assert payload["product_data_green"] is True - assert payload["backup_core_green"] is True - assert payload["host_188_service_green"] is True - assert payload["observed_host_count"] == 4 + assert payload["active_blocker_count"] == 12 + assert payload["readiness_percent"] == 18 + assert payload["service_green"] is False + assert payload["product_data_green"] is False + assert payload["backup_core_green"] is False + assert payload["host_188_service_green"] is False + assert payload["observed_host_count"] == 7 assert payload["missing_host_count"] == 0 - assert payload["unreachable_host_count"] == 0 - assert payload["stale_host_count"] == 4 - assert payload["blocked_by_fresh_reboot_window_only"] is True - assert payload["latest_verify_only_metric_present"] is True + assert payload["unreachable_host_count"] == 1 + assert payload["stale_host_count"] == 5 + assert payload["blocked_by_fresh_reboot_window_only"] is False + assert payload["latest_verify_only_metric_present"] is False assert payload["latest_verify_only_metric_ready"] == 0 - assert payload["latest_verify_only_metric_blocker_count"] == 1 - assert payload["latest_verify_only_metric_max_host_uptime_seconds"] == 541538 - assert payload["latest_verify_only_metric_last_run_timestamp"] > 0 - assert payload["stockplatform_freshness_status"] == "ok" - assert payload["stockplatform_ingestion_status"] == "ok" + assert payload["latest_verify_only_metric_blocker_count"] == 0 + assert payload["latest_verify_only_metric_max_host_uptime_seconds"] == 0 + assert payload["latest_verify_only_metric_last_run_timestamp"] == 0 + assert payload["stockplatform_freshness_status"] == "not_configured" + assert payload["stockplatform_ingestion_status"] == "not_configured" assert payload["readback"]["workplan_id"] == "P0-006" assert payload["readback"]["target_minutes"] == 10 assert payload["readback"]["safe_next_step"] == ( - "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_" - "event_or_approved_reboot_drill_to_prove_10_minute_slo" + "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_" + "rerun_scorecard_until_status_slo_ready" ) - assert payload["readback"]["active_blocker_count"] == 1 - assert payload["readback"]["readiness_percent"] == 82 - assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is True - assert payload["readback"]["latest_verify_only_metric_present"] is True - assert payload["rollups"]["active_blocker_count"] == 1 - assert payload["rollups"]["readiness_percent"] == 82 - assert payload["rollups"]["observed_host_count"] == 4 + assert payload["readback"]["active_blocker_count"] == 12 + assert payload["readback"]["readiness_percent"] == 18 + assert payload["readback"]["blocked_by_fresh_reboot_window_only"] is False + assert payload["readback"]["latest_verify_only_metric_present"] is False + assert payload["rollups"]["active_blocker_count"] == 12 + assert payload["rollups"]["readiness_percent"] == 18 + assert payload["rollups"]["observed_host_count"] == 7 assert payload["rollups"]["missing_host_count"] == 0 - assert payload["rollups"]["unreachable_host_count"] == 0 - assert payload["rollups"]["stale_host_count"] == 4 - assert payload["rollups"]["service_green"] is True - assert payload["rollups"]["product_data_green"] is True - assert payload["rollups"]["backup_core_green"] is True - assert payload["rollups"]["host_188_service_green"] is True - assert payload["rollups"]["blocked_by_fresh_reboot_window_only"] is True - assert payload["rollups"]["latest_verify_only_metric_present"] is True + assert payload["rollups"]["unreachable_host_count"] == 1 + assert payload["rollups"]["stale_host_count"] == 5 + assert payload["rollups"]["service_green"] is False + assert payload["rollups"]["product_data_green"] is False + assert payload["rollups"]["backup_core_green"] is False + assert payload["rollups"]["host_188_service_green"] is False + assert payload["rollups"]["blocked_by_fresh_reboot_window_only"] is False + assert payload["rollups"]["latest_verify_only_metric_present"] is False assert payload["rollups"]["latest_verify_only_metric_ready"] == 0 - assert payload["rollups"]["latest_verify_only_metric_blocker_count"] == 1 - assert payload["rollups"]["stockplatform_freshness_status"] == "ok" - assert payload["rollups"]["stockplatform_ingestion_status"] == "ok" - assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 0 - assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 0 - assert payload["rollups"]["stockplatform_final_retry_window_passed"] is True + assert payload["rollups"]["latest_verify_only_metric_blocker_count"] == 0 + assert payload["rollups"]["stockplatform_freshness_status"] == "not_configured" + assert payload["rollups"]["stockplatform_ingestion_status"] == "not_configured" + assert payload["rollups"]["stockplatform_freshness_blocker_count"] == 1 + assert payload["rollups"]["stockplatform_ingestion_blocker_count"] == 1 + assert payload["rollups"]["stockplatform_final_retry_window_passed"] is False assert ( payload["rollups"]["stockplatform_controlled_recovery_gate_required"] is False ) - assert payload["active_blockers"] == ["host_boot_observation_older_than_target_window"] + assert payload["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS stockplatform = payload["stockplatform_data_freshness"] assert stockplatform["freshness_endpoint_readback_present"] is True assert stockplatform["ingestion_endpoint_readback_present"] is True - assert stockplatform["freshness_blockers"] == [] - assert stockplatform["ingestion_blockers"] == [] - assert stockplatform["margin_short_recovery"]["status"] == "recovered" - assert stockplatform["margin_short_recovery"]["successful_source_run_ids"] == [ - 3389, - 3390, - ] - assert stockplatform["ai_recommendations_recovery"]["status"] == "recovered" - assert stockplatform["eod_window"]["final_retry_window_passed"] is True + assert stockplatform["freshness_blockers"] == ["postgres_not_ready"] + assert stockplatform["ingestion_blockers"] == ["postgres_not_ready"] + assert stockplatform["margin_short_recovery"]["status"] == "not_verified" + assert stockplatform["margin_short_recovery"]["successful_source_run_ids"] == [] + assert stockplatform["ai_recommendations_recovery"]["status"] == "not_verified" + assert stockplatform["eod_window"]["final_retry_window_passed"] is False assert stockplatform["controlled_recovery_gate"]["required"] is False assert ( stockplatform["controlled_recovery_gate"]["status"] - == "not_required_freshness_recovered" + == "not_required_yet" ) assert "manual_db_update" in stockplatform["controlled_recovery_gate"][ "forbidden_actions" @@ -142,10 +166,10 @@ def _assert_drill_preflight_payload(payload: dict): == "reboot_auto_recovery_drill_preflight_readback_v1" ) assert payload["priority"] == "P0-006" - assert payload["status"] == "ready_for_break_glass_reboot_drill_authorization" - assert payload["preflight_ready"] is True - assert payload["preflight_blocker_count"] == 0 - assert payload["active_blockers"] == [] + assert payload["status"] == "blocked_reboot_drill_preflight_not_ready" + assert payload["preflight_ready"] is False + assert payload["preflight_blocker_count"] == 9 + assert payload["active_blockers"] == EXPECTED_DRILL_PREFLIGHT_BLOCKERS assert payload["break_glass_authorization_required"] is True assert payload["execution_authorized_by_this_endpoint"] is False assert payload["safe_next_step"] == ( @@ -153,12 +177,20 @@ def _assert_drill_preflight_payload(payload: dict): "real_all_host_reboot_event_then_rerun_verify_only" ) target_selector = payload["target_selector"] - assert target_selector["required_host_aliases"] == ["110", "120", "121", "188"] - assert target_selector["required_host_count"] == 4 - assert target_selector["observed_host_count"] == 4 + assert target_selector["required_host_aliases"] == [ + "110", + "111", + "112", + "120", + "121", + "188", + "99", + ] + assert target_selector["required_host_count"] == 7 + assert target_selector["observed_host_count"] == 7 assert target_selector["missing_host_count"] == 0 - assert target_selector["unreachable_host_count"] == 0 - assert target_selector["stale_host_count"] == 4 + assert target_selector["unreachable_host_count"] == 1 + assert target_selector["stale_host_count"] == 5 assert set(payload["preconditions"]) == { "source_controls_present", "service_green", @@ -172,18 +204,28 @@ def _assert_drill_preflight_payload(payload: dict): "stockplatform_ingestion_ok", "blocked_only_by_fresh_reboot_window", } - assert all(payload["preconditions"].values()) + assert payload["preconditions"] == { + "source_controls_present": True, + "service_green": False, + "product_data_green": False, + "backup_core_green": False, + "host_188_service_green": False, + "all_required_hosts_observed": True, + "all_required_hosts_reachable": False, + "latest_verify_only_metric_present": False, + "stockplatform_freshness_ok": False, + "stockplatform_ingestion_ok": False, + "blocked_only_by_fresh_reboot_window": False, + } current = payload["current_readback"] assert current["scorecard_status"] == "blocked_reboot_auto_recovery_slo_not_ready" - assert current["readiness_percent"] == 82 - assert current["active_blocker_count"] == 1 - assert current["active_blockers"] == [ - "host_boot_observation_older_than_target_window" - ] + assert current["readiness_percent"] == 18 + assert current["active_blocker_count"] == 12 + assert current["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS assert current["latest_verify_only_metric_ready"] == 0 - assert current["latest_verify_only_metric_blocker_count"] == 1 - assert current["stockplatform_freshness_status"] == "ok" - assert current["stockplatform_ingestion_status"] == "ok" + assert current["latest_verify_only_metric_blocker_count"] == 0 + assert current["stockplatform_freshness_status"] == "not_configured" + assert current["stockplatform_ingestion_status"] == "not_configured" check_mode = payload["check_mode"] assert check_mode["verify_only_available"] is True assert ( @@ -201,17 +243,18 @@ def _assert_drill_preflight_payload(payload: dict): assert rollback["rollback_required_for_this_endpoint"] is False readback = payload["readback"] assert readback["workplan_id"] == "P0-006" - assert readback["preflight_ready"] is True - assert readback["target_selector"]["required_host_count"] == 4 - assert readback["current_readback"]["active_blocker_count"] == 1 + assert readback["preflight_ready"] is False + assert readback["target_selector"]["required_host_count"] == 7 + assert readback["current_readback"]["active_blocker_count"] == 12 assert readback["safe_next_step"] == payload["safe_next_step"] rollups = payload["rollups"] - assert rollups["preflight_ready"] is True - assert rollups["preflight_ready_count"] == 1 - assert rollups["preflight_blocker_count"] == 0 - assert rollups["target_required_host_count"] == 4 - assert rollups["target_observed_host_count"] == 4 - assert rollups["blocked_only_by_fresh_reboot_window"] is True + assert rollups["preflight_ready"] is False + assert rollups["preflight_ready_count"] == 0 + assert rollups["preflight_blocker_count"] == 9 + assert rollups["target_required_host_count"] == 7 + assert rollups["target_observed_host_count"] == 7 + assert rollups["target_unreachable_host_count"] == 1 + assert rollups["blocked_only_by_fresh_reboot_window"] is False assert rollups["host_reboot_authorized_by_this_endpoint"] is False boundaries = payload["operation_boundaries"] assert boundaries["host_reboot_authorized_by_this_endpoint"] is False diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 8baa4fc17..cd7c98ed7 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -12,6 +12,25 @@ **邊界**：只讀 public Gitea / production HTTP 與本機 source；未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / `gh` / GitHub API；未 workflow_dispatch；未 SSH / Docker / Nginx / K3s / DB / firewall runtime 寫入。 +## 2026-06-30 — 20:46/20:53 全主機重啟後 SLO / Harbor / Stock fail-closed readback + +**照主線更新的真相**： +- 最新 Gitea SSH `main` 是 `01deee4a7 fix(runner): classify harbor 110 repair waiting runs`；Public Gitea queue 讀到 latest CD `#4062` failure，classifier=`harbor_registry_public_route_unavailable`、status code `502`、Harbor controlled repair attempted=`true`、skip reason=`not_110_host`、Harbor login attempts=`12`。 +- 最新 scheduled repair run 是 `harbor-110-local-repair.yaml #4063`，status=`Waiting`；queue classifier 已回 `blocked_harbor_110_repair_workflow_waiting`，表示下一步是讓 110-local repair lane 接到可用 runner / local execution，而不是重複泛化成單純 Harbor 502。 +- Live runtime probe 仍回：`https://registry.wooo.work/v2/` 502、`http://192.168.0.110:5000/v2/` 502、`https://harbor.wooo.work/api/v2.0/health` 502、`https://signoz.wooo.work/` 502、StockPlatform freshness HTTP 200 但 `status=not_configured` / blocker `postgres_not_ready`。 +- `post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260630-live-main/summary.txt` 回 `POST_START_PASS=33 WARN=6 BLOCKED=8`、`SERVICE_GREEN=0`、`PRODUCT_DATA_GREEN=0`、`BACKUP_CORE_GREEN=0`、`DR_ESCROW_EVIDENCE_UNKNOWN=1`、`HOST_188_SERVICE_GREEN=0`、`HOST_188_HYGIENE_BLOCKED=1`、`OVERALL_DECLARATION=SERVICE_BLOCKED`。 +- `full-stack-cold-start-check.sh --monitor-read-only --no-color` 回 `PASS=68 WARN=4 BLOCKED=4`，明確 blockers 是 110 registry `/v2`、110 SSH read-only check、K3s registry pull refused by `110:5000`、SignOz TLS / 502。 +- SLO artifact `/tmp/awoooi-reboot-slo-live-20260630-2045/scorecard.json` 回 `status=blocked_reboot_auto_recovery_slo_not_ready`、`can_claim_all_services_recovered_within_target=false`、active blockers `12`；host probe 觀察到 99/110/112/120/121/188 可達、111 不可達、99 uptime unknown、188 `awoooi-startup.service=failed` / systemd `degraded`、110/112/120/121/188 已超過 10 分鐘視窗。 +- `docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json` 已更新成 7-host fail-closed 真相；API contract tests 改為鎖住 readiness `18%`、active blockers `12`、Stock `not_configured`、drill preflight `blocked_reboot_drill_preflight_not_ready`。 + +**下一個固定 P0 順序**： +1. 110 control path / Harbor registry `/v2`：先讓 110-local repair workflow 或 110 console/local script 真正執行並讀回 `/v2` 200/401；否則 CD 無法恢復最新版本。 +2. 111 / 99 VMware：補齊 VM power / autostart readback，讓 all-host required probe 不再缺 111 / uptime unknown。 +3. StockPlatform freshness：在 110/Harbor/CD 路徑恢復後查 Postgres schema / migration / compose truth，禁止 fake freshness、manual DB rows、truncate/restore。 +4. Backup / DR / Telegram：110 backup-status 可讀後重跑 backup core、offsite、credential escrow evidence 與 alert delivery readback。 + +**邊界**：未 workflow_dispatch，未 SSH 寫主機，未重啟主機，未 restart Docker daemon / host Nginx / K3s / DB / Redis / firewall，未 prune / restore / DB write，未讀 secret / token / raw sessions / SQLite / `.env`，未使用 GitHub / `gh` / GitHub API。 + ## 2026-06-30 — 20:45 Harbor 110 repair waiting readback classifier **照主線修正的問題**： diff --git a/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json b/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json index 73eef3e74..29fa884b9 100644 --- a/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json +++ b/docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json @@ -1,112 +1,218 @@ { "active_blockers": [ - "host_boot_observation_older_than_target_window" + "all_required_hosts_not_in_10_minute_reboot_window", + "backup_core_green_not_1", + "host_188_service_green_not_1", + "host_boot_observation_older_than_target_window", + "host_unreachable_after_reboot", + "host_uptime_unknown", + "local_disk_free_below_minimum", + "post_start_blocked_not_zero", + "product_data_green_not_1", + "reboot_event_required_host_unreachable", + "service_green_not_1", + "wazuh_dashboard_degraded" ], "can_claim_all_services_recovered_within_target": false, "capacity": { "checked": true, - "free_gib": 4.454, + "free_gib": 0.751, "min_free_gib": 2.0 }, - "generated_at": "2026-06-29T23:45:54+08:00", + "generated_at": "2026-06-30T20:44:37+08:00", "host_boot_detection": { "host_rows": [ { - "alias": "110", - "boot_id": "a3dfae32-3762-4394-86fa-a342aea07df5", + "alias": "99", + "boot_id": "reachable_unknown_boot", "reachable": true, - "startup_active": "inactive", - "startup_enabled": "enabled", + "startup_active": "unknown", + "startup_enabled": "unknown", + "startup_unit": "vmware-host-autostart", + "systemd_state": "ping_reachable", + "target": "192.168.0.99", + "uptime_seconds": -1 + }, + { + "alias": "110", + "boot_id": "node_exporter_1782811060", + "reachable": true, + "startup_active": "unknown", + "startup_enabled": "unknown", "startup_unit": "awoooi-startup-110.service", - "systemd_state": "degraded", + "systemd_state": "node_exporter", "target": "wooo@192.168.0.110", - "uptime_seconds": 534656 + "uptime_seconds": 12406 + }, + { + "alias": "111", + "boot_id": "unknown", + "reachable": false, + "startup_active": "unknown", + "startup_enabled": "unknown", + "startup_unit": "vm-host-boot", + "systemd_state": "unknown", + "target": "192.168.0.111", + "uptime_seconds": -1 + }, + { + "alias": "112", + "boot_id": "node_exporter_1782810990", + "reachable": true, + "startup_active": "unknown", + "startup_enabled": "unknown", + "startup_unit": "vm-host-boot", + "systemd_state": "node_exporter", + "target": "192.168.0.112", + "uptime_seconds": 12484 }, { "alias": "120", - "boot_id": "866d621e-bad0-4096-938e-b103db8e5e03", + "boot_id": "eeb5049d-2c82-46f8-b9ea-be3875842795", "reachable": true, "startup_active": "active", "startup_enabled": "enabled", "startup_unit": "k3s.service", "systemd_state": "running", "target": "wooo@192.168.0.120", - "uptime_seconds": 534648 + "uptime_seconds": 12479 }, { "alias": "121", - "boot_id": "119c4ea7-8b49-45aa-b60e-bcd4b5dd0979", + "boot_id": "777fc552-1582-48bc-b09e-e00cca5cc2ce", "reachable": true, "startup_active": "active", "startup_enabled": "enabled", "startup_unit": "k3s.service", "systemd_state": "running", "target": "wooo@192.168.0.121", - "uptime_seconds": 534605 + "uptime_seconds": 12480 }, { "alias": "188", - "boot_id": "9cc1f1fc-b7cc-42c1-bc83-d495f0e3c863", + "boot_id": "ae363b77-286f-4c56-84b6-c77e530d9e87", "reachable": true, - "startup_active": "inactive", + "startup_active": "failed", "startup_enabled": "enabled", "startup_unit": "awoooi-startup.service", - "systemd_state": "running", + "systemd_state": "degraded", "target": "ollama@192.168.0.188", - "uptime_seconds": 534664 + "uptime_seconds": 12420 } ], - "max_observed_uptime_seconds": 541538, + "max_observed_uptime_seconds": 12484, "missing_hosts": [], "observed_hosts": [ "110", + "111", + "112", "120", "121", - "188" + "188", + "99" ], "required_hosts": [ "110", + "111", + "112", "120", "121", - "188" + "188", + "99" ], "stale_hosts": [ "110", + "112", "120", "121", "188" ], - "unknown_uptime_hosts": [], - "unreachable_hosts": [] + "unknown_uptime_hosts": [ + "99" + ], + "unreachable_hosts": [ + "111" + ] }, "post_reboot_readiness": { - "backup_core_green": true, - "host_188_service_green": true, - "next_required_gates": "credential_escrow_evidence", - "overall_declaration": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED", - "post_start_blocked": 0, - "post_start_result": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED", - "product_data_green": true, - "service_green": true, + "backup_core_green": false, + "host_188_service_green": false, + "next_required_gates": "product_data_freshness_recovery,backup_core_readback_recovery,credential_escrow_evidence,host_188_hygiene_maintenance_window", + "overall_declaration": "SERVICE_BLOCKED", + "post_start_blocked": 8, + "post_start_result": "BLOCKED", + "product_data_green": false, + "service_green": false, "summary_present": true, - "wazuh_dashboard_degraded": false + "wazuh_dashboard_degraded": true }, - "safe_next_step": "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_event_or_approved_reboot_drill_to_prove_10_minute_slo", + "reboot_event_detection": { + "all_required_hosts_in_reboot_window": false, + "all_required_hosts_observed": false, + "changed_boot_id_hosts": [], + "fresh_boot_hosts": [ + "99" + ], + "missing_hosts": [], + "readback_present": true, + "reboot_detected": true, + "rebooted_hosts": [ + "99" + ], + "state_file": "/tmp/awoooi-reboot-slo-live-20260630-2045/reboot-event-state.json", + "state_written": false, + "unreachable_hosts": [ + "111" + ] + }, + "safe_next_step": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard_until_status_slo_ready", "schema_version": "awoooi_reboot_auto_recovery_slo_scorecard_v1", + "sla_recovery_eta": { + "deadline_status": "target_window_elapsed", + "estimated_recovery_status": "target_window_elapsed_or_not_detected", + "fixed_triage_order": [ + "99_vmware_autostart_and_vm_power", + "host_boot_event_and_node_exporter", + "public_routes_and_maintenance_fallback", + "awoooi_k3s_workloads_and_registry", + "stockplatform_public_api_and_freshness", + "backup_health_and_offsite_evidence", + "telegram_alert_delivery_readback" + ], + "next_update_due_seconds": 60, + "target_minutes": 10, + "target_seconds_remaining": 0 + }, "source_controls": { "cold_start_textfile_exporter_source_present": true, "host_110_startup_unit_source_present": true, "host_188_startup_unit_source_present": true, "host_boot_probe_source_present": true, "post_reboot_summary_source_present": true, + "public_maintenance_fallback_source_present": true, + "reboot_event_detector_source_present": true, "slo_exporter_source_present": true, "slo_installer_source_present": true, "slo_systemd_service_source_present": true, - "slo_systemd_timer_source_present": true + "slo_systemd_timer_source_present": true, + "telegram_reboot_backup_alert_rules_source_present": true, + "windows_99_vmware_autostart_source_present": true }, "status": "blocked_reboot_auto_recovery_slo_not_ready", "stockplatform_data_freshness": { - "blocked_sources": [], + "ai_recommendations_recovery": { + "cleared_blocker": "ai_recommendations_stale", + "status": "not_verified" + }, + "blocked_sources": [ + { + "latest_date": null, + "notes": "PostgreSQL 結構尚未就緒：OperationalError", + "row_count": 0, + "source": "postgres", + "status": "not_configured" + } + ], "controlled_recovery_gate": { "allowed_actions": [ "inspect_existing_ingestion_readback", @@ -122,114 +228,36 @@ "reboot_or_service_restart_from_reboot_slo_lane" ], "required": false, - "status": "not_required_freshness_recovered", + "status": "not_required_yet", "target_selector": "stockplatform-v2:system_freshness:core.margin_short_daily,ai.recommendations" }, "eod_window": { - "classification": "recovered_after_final_retry_window", + "classification": "blocked_non_current_trading_day", "final_retry_window_end_local": "23:35", - "final_retry_window_passed": true, + "final_retry_window_passed": false, "first_full_window_end_local": "19:15", - "next_action": "rerun_slo_verify_only_after_next_fresh_all_host_reboot_event_or_approved_reboot_drill", + "next_action": "inspect_trading_calendar_and_ingestion_logs", "pending": false }, - "freshness_blockers": [], - "freshness_endpoint_readback_present": true, - "freshness_status": "ok", - "ingestion_blockers": [], - "ingestion_endpoint_readback_present": true, - "ingestion_status": "ok", - "latest_source_runs": [ - { - "source_run_id": 3392, - "source_name": "intelligence_security_linker", - "target_date": null, - "status": "succeeded", - "started_at": "2026-06-29T15:00:09.333642Z", - "finished_at": "2026-06-29T15:00:09.333642Z" - }, - { - "source_run_id": 3391, - "source_name": "intelligence_reports_import", - "target_date": null, - "status": "succeeded", - "started_at": "2026-06-29T15:00:08.472808Z", - "finished_at": "2026-06-29T15:00:08.472808Z" - }, - { - "source_run_id": 3389, - "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", - "status": "succeeded", - "started_at": "2026-06-29T13:05:13.341357Z", - "finished_at": "2026-06-29T13:05:13.341357Z" - }, - { - "source_run_id": 3390, - "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", - "status": "succeeded", - "started_at": "2026-06-29T13:05:13.341357Z", - "finished_at": "2026-06-29T13:05:13.341357Z" - }, - { - "source_run_id": 3388, - "source_name": "intelligence_security_linker", - "target_date": null, - "status": "succeeded", - "started_at": "2026-06-29T13:00:08.792550Z", - "finished_at": "2026-06-29T13:00:08.792550Z" - }, - { - "source_run_id": 3387, - "source_name": "intelligence_reports_import", - "target_date": null, - "status": "succeeded", - "started_at": "2026-06-29T13:00:07.822700Z", - "finished_at": "2026-06-29T13:00:07.822700Z" - }, - { - "source_run_id": 3385, - "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", - "status": "official_pending", - "started_at": "2026-06-29T12:05:16.716460Z", - "finished_at": "2026-06-29T12:05:16.716460Z" - }, - { - "source_run_id": 3386, - "source_name": "official_margin_short_daily", - "target_date": "2026-06-29", - "status": "official_pending", - "started_at": "2026-06-29T12:05:16.716460Z", - "finished_at": "2026-06-29T12:05:16.716460Z" - } + "freshness_blockers": [ + "postgres_not_ready" ], - "latest_trading_date": "2026-06-29", - "second_retry_checked_at": "2026-06-29T20:21:35+08:00", - "third_retry_checked_at": "2026-06-29T21:15:30+08:00", + "freshness_endpoint_readback_present": true, + "freshness_status": "not_configured", + "ingestion_blockers": [ + "postgres_not_ready" + ], + "ingestion_endpoint_readback_present": true, + "ingestion_status": "not_configured", + "latest_source_runs": [], + "latest_trading_date": "None", "margin_short_recovery": { - "status": "recovered", "cleared_blocker": "core_margin_short_daily_missing", "cleared_ingestion_blocker": "core.margin_short_daily_incomplete", - "successful_source_run_ids": [ - 3389, - 3390 - ] - }, - "ai_recommendations_recovery": { - "status": "recovered", - "cleared_blocker": "ai_recommendations_stale" - }, - "final_retry_checked_at": "2026-06-29T23:45:54+08:00" + "status": "not_verified", + "successful_source_run_ids": [] + } }, "target_minutes": 10, - "target_seconds": 600, - "latest_verify_only_metric": { - "ready": 0, - "blocker_count": 1, - "max_host_uptime_seconds": 541538, - "last_run_timestamp": 1782738942, - "artifact_dir": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-211422" - } + "target_seconds": 600 } diff --git a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md index 7ee0bf790..b7d36adb1 100644 --- a/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md +++ b/docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md @@ -3,7 +3,7 @@ > Owner: SRE / DevOps commander > Timezone: Asia/Taipei > Baseline: 2026-06-04 15:00 live read-only checks. Do not reuse the 2026-05-29 baseline without rerunning checks. -> Scope: 110 / 120 / 121 / 188. 112 is Kali and is intentionally excluded from this recovery wave. +> Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 for reboot auto-detection and 10-minute SLO. 112 remains Kali, but the post-2026-06-30 all-host recovery probe tracks it as a required boot/power signal. --- @@ -15,13 +15,13 @@ | 優先 | 狀態 | 工作項 | 2026-06-30 證據 | 下一步 / 完成條件 | |------|------|--------|------------------|-------------------| -| P0-1 | BLOCKED | 全主機 cold-start / 10 分鐘自動恢復 SLO | 20:18 `post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260630-201642/summary.txt` 回 `POST_START_PASS=33 WARN=6 BLOCKED=8`、`SERVICE_GREEN=0`、`OVERALL_DECLARATION=SERVICE_BLOCKED`；110 registry `/v2`、110 SSH / backup / CPU / runner readback、K3s registry pull refused、SignOz 502/TLS、Stock `postgres_not_ready`、188 hygiene 仍 blocked。summary SSH 已 bounded，不再無限卡住。 | 先修第一個 runtime blocker：110 control path / Harbor registry `/v2`。重跑同一 summary 到 `SERVICE_GREEN=1` 且 `POST_START_BLOCKED=0`；不可只用 route 200 宣稱恢復。 | +| P0-1 | BLOCKED | 全主機 cold-start / 10 分鐘自動恢復 SLO | 20:46 `post-reboot-readiness-summary.sh --no-color` artifact `/tmp/awoooi-post-reboot-readiness-20260630-live-main/summary.txt` 回 `POST_START_PASS=33 WARN=6 BLOCKED=8`、`SERVICE_GREEN=0`、`PRODUCT_DATA_GREEN=0`、`BACKUP_CORE_GREEN=0`、`HOST_188_SERVICE_GREEN=0`、`OVERALL_DECLARATION=SERVICE_BLOCKED`；`full-stack-cold-start-check.sh --monitor-read-only --no-color` 回 `PASS=68 WARN=4 BLOCKED=4`；SLO scorecard `/tmp/awoooi-reboot-slo-live-20260630-2045/scorecard.json` 回 `can_claim_all_services_recovered_within_target=false`、active blockers `12`、readiness `18%`。111 不可達、99 uptime unknown、188 startup failed/degraded、110/112/120/121/188 已超過 10 分鐘視窗。 | 先修第一個 runtime blocker：110 control path / Harbor registry `/v2`。重跑同一 summary / cold-start / SLO scorecard 到 `SERVICE_GREEN=1`、`POST_START_BLOCKED=0`、`PASS` 無 BLOCKED、all-host required observed/reachable 且 `awoooi_reboot_auto_recovery_slo_ready=1`；不可只用 route 200 宣稱恢復。 | | P0-2 | DONE_THIS_INCIDENT | 使用者可見 502：Tsenyang | `www.tsenyang.com` / `tsenyang.com` 由 502 恢復為 200；188 `tsenyang-website` container running；local `127.0.0.1:3000` 回 200。 | 下次同類 502 先查 release symlink / image / container；不先動 Nginx、DNS、DB、主機重啟。 | | P0-3 | BLOCKED | StockPlatform data freshness | public `/healthz`、`/api/healthz` 回 200；freshness / ingestion 回 `not_configured`、`postgres_not_ready`。 | 恢復 110 control path 後，read-only 查 `/home/wooo/stockplatform-v2` compose / DB schema / migration status；禁止 fake freshness、manual DB rows、restore/prune。 | -| P0-4 | BLOCKED | AWOOOI production 版本最新性 | Gitea `main` 已多次前進但 production runtime readback 仍為 `7890778b83`，`runtime_build_readback_status=runtime_build_diverges_from_committed_deploy_readback`。Public Gitea visible run `cd.yaml #4043` 是 Failure；jobs API 與 visible run `head_sha` 不一致，已標 `cd_jobs_stale_or_mismatched`。 | 補 deploy marker / runtime SHA / endpoint readback 一致；未一致前不可宣稱 AWOOOI 最新。 | -| P0-5 | BLOCKED | 110 control path | `diagnose-110-ssh-publickey-auth.sh`：node-exporter / SSH banner 正常；`NODE_LOAD_CLASSIFIER=high_load`、`NODE_PROCS_BLOCKED=0`；`wooo` publickey `publickey_offer_timeout`，`root` publickey `permission_denied`，`git` / `ollama` `preauth_timeout`。 | 集中查 110 sshd publickey auth / authorized_keys / PAM / account lookup path，並把 load / runner pressure 視為同一 blocker 的共因；可在 110 console/local 跑 `repair-110-ssh-publickey-auth-local.sh --check` / `--apply` 修 metadata 權限。恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | +| P0-4 | BLOCKED | AWOOOI production 版本最新性 | Gitea SSH `main` 最新是 `49a9f7309`，但 public CD `#4061` 失敗，classifier=`harbor_registry_public_route_unavailable`、status code `502`、controlled repair attempted=`true`、skip reason=`not_110_host`；production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 仍回 2026-06-29 舊資料，不能當現在真相。 | 補 deploy marker / runtime SHA / endpoint readback 一致；Harbor `/v2` 恢復前 CD 無法把最新 source 發到 production，未一致前不可宣稱 AWOOOI 最新。 | +| P0-5 | BLOCKED | 110 control path / Harbor registry `/v2` | 20:46 live probe：`https://registry.wooo.work/v2/` 502、`http://192.168.0.110:5000/v2/` 502、`https://harbor.wooo.work/api/v2.0/health` 502；CD `#4061` self-heal skip reason=`not_110_host`；110 SSH read-only / backup-status / CPU / controlled runner readback 仍 timeout 或不可確認。 | 讓 110-local repair workflow 或 110 console/local script 真正執行 `recover-110-control-path-and-harbor-local.sh --check` / `--apply-all`，並讀回 public/internal `/v2` 為 `200/401`。恢復 SSH read-only command path 後才能驗證 Stock DB、Gitea dump、110 backup completeness。 | | P0-6 | BLOCKED_BACKUP_COMPLETENESS | Gitea repo visibility 與完整備份 | Gitea version API 200；public repo search 只列 4 個 public repo；`stockplatform-v2` public page/API 404，但 internal `git ls-remote` 成功；188 `/home/ollama/backup/110/gitea` 起初為空。已建立 verified emergency bundle `/home/ollama/backup/110/gitea/git-bundles/20260630-190931`：4 個 public/internal repo bundle verify + checksum 成功，`AwoooGo`、`stockplatform-v2`、`vibework` 因 private auth fail-closed。20:18 summary 因 110 `backup-status` 不可讀回，`BACKUP_CORE_GREEN=0`、`DR_ESCROW_BLOCKED=1`、`DR_ESCROW_EVIDENCE_UNKNOWN=1`。 | 188 `gitea_repo_mirror_from_110` subtree metric / alert 已補；下一步仍是恢復 110 SSH command path 後跑正式 `gitea dump`、private repo 非互動備份、repo count、backup-status 與 restore drill readback。unknown 不得當作 backup / DR green。 | -| P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`；最新只讀 readback：99 ping OK、RDP 3389 OK、SSH 22 OK、WinRM 5985 fail，`administrator@192.168.0.99` SSH publickey denied；VM host 111 仍不可達。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence。 | +| P0-7 | SOURCE_READY_RUNTIME_BLOCKED | 99 VMware / VM autostart | repo 已有 `windows99-vmware-autostart.ps1`；20:46 host probe 讀到 99 ping reachable 但 `boot_id=reachable_unknown_boot` / uptime unknown，111 不可達，112/120/121/188 可讀，188 startup unit failed。先前只讀 readback 顯示 99 RDP 3389 / SSH 22 可達、WinRM 5985 fail，`administrator@192.168.0.99` SSH publickey denied。 | 恢復 99 可控通道或由 console 套用腳本；完成後讀回 111/188/120/121/112 boot evidence，要求 all-host required observed/reachable 且 99 不再是 unknown uptime。 | | P0-8 | SOURCE_READY_RUNTIME_BLOCKED | 502 maintenance fallback / Telegram / backup alert | L0/L1 fallback runbook、Nginx snippet、reboot / backup alert rules 已在 source；runtime 尚需部署與外部 L1 provider readback。 | L0 以測試 vhost 驗證 `X-AWOOOI-Fallback`；L1 需外部雲端/CDN probe；Telegram 以脫敏 alert receipt 驗證。 | 本次核心經驗： @@ -30,6 +30,7 @@ - 備份心跳 fresh 不等於完整備份；Gitea 子樹 / emergency bundle / dump / repo count / restore drill 必須單獨告警。bundle 只保 Git history，不等於 Gitea 全量 dump。 - 使用者可見 502 優先於資料 freshness；先恢復靜態/容器服務，再回到資料層與版本一致性。 - 版本最新性要同時看 source SHA、deploy marker、runtime SHA 與 public endpoint；不能只看 Gitea main。 +- 2026-06-30 實測證明：source 上有 reboot detector / alert / VMware autostart / maintenance fallback 並不等於 runtime 已達標；必須同時讀回 all-host probe、SLO metric、public `/v2`、Stock freshness、backup-status、Telegram delivery。 | Area | Status | Completion | Evidence | |------|--------|------------|----------|