diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index d9cbbfad9..18116af0d 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -1420,7 +1420,9 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) - if "local_disk_free_below_minimum" in active_blockers: return "host_capacity_blocked" if any( - blocker.startswith("host_") or blocker.startswith("awooop_") + blocker.startswith("host_") + or blocker.startswith("awooop_") + or blocker.startswith("conversation_event_hot_path_") for blocker in active_blockers ): return "host_pressure_blocked" @@ -1668,15 +1670,27 @@ def _active_blocker_action_row( "host_110_gitea_cpu_pressure", "host_188_postgres_cpu_pressure", "awooop_conversation_event_hot_path_index_drift_detected", + "conversation_event_hot_path_index_migration_source_missing", }: category = "host_cpu_pressure" owner_lane = "host_pressure_controller" - next_safe_action = ( - "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" - "verifier_green_no_restart" - ) - post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" - evidence_inputs = ["host_pressure"] + if blocker == "conversation_event_hot_path_index_migration_source_missing": + next_safe_action = ( + "restore_conversation_event_hot_path_migration_source_then_rerun_" + "host_pressure_and_reboot_slo_scorecard_no_restart" + ) + post_verifier = ( + "source_controls_conversation_event_hot_path_index_migration_" + "and_host_pressure_readback" + ) + evidence_inputs = ["source_controls", "runtime_metric_readback"] + else: + next_safe_action = ( + "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" + "verifier_green_no_restart" + ) + post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" + evidence_inputs = ["host_pressure"] controlled_apply_mode = "playbook_check_mode_then_controlled_apply_if_low_risk" forbidden_actions.extend( ["unmask_legacy_runner_or_restore_generic_labels", "kill_process_without_gate"] diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 9a2c3c002..9692f1d34 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -267,6 +267,26 @@ def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_runtime_metrics( assert "backup_core_green_not_1" not in payload["active_blockers"] assert "service_green_not_1" not in payload["active_blockers"] assert payload["active_blocker_action_matrix"]["item_count"] == 7 + action_by_blocker = { + item["blocker"]: item + for item in payload["active_blocker_action_matrix"]["items"] + } + hot_path_source_action = action_by_blocker[ + "conversation_event_hot_path_index_migration_source_missing" + ] + assert hot_path_source_action["category"] == "host_cpu_pressure" + assert hot_path_source_action["owner_lane"] == "host_pressure_controller" + assert hot_path_source_action["evidence_inputs"] == [ + "source_controls", + "runtime_metric_readback", + ] + assert hot_path_source_action["next_safe_action"] == ( + "restore_conversation_event_hot_path_migration_source_then_rerun_" + "host_pressure_and_reboot_slo_scorecard_no_restart" + ) + assert payload["active_blocker_action_matrix"]["category_counts"][ + "host_cpu_pressure" + ] == 1 assert payload["readback"]["active_blocker_count"] == 7 assert payload["readback"]["runtime_metric_readback_present"] is True assert payload["rollups"]["active_blocker_count"] == 7 diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 53f5fec85..a6c8da2fb 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,22 @@ +## 2026-07-03 — 00:55 P0-006 runtime blocker action-matrix classification + +**完成內容**: +- Production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 在 deploy marker `cb39fc2bd chore(cd): deploy c0f43ae [skip ci]` 後已讀到 action matrix,active blockers 由 `11` 收斂為 `7`。 +- 7 個 blockers 為:`all_required_hosts_not_in_10_minute_reboot_window`、`conversation_event_hot_path_index_migration_source_missing`、`host_boot_observation_older_than_target_window`、`host_unreachable_after_reboot`、`host_uptime_unknown`、`reboot_event_required_host_unreachable`、`windows99_vmware_autostart_readback_missing`。 +- 發現 `conversation_event_hot_path_index_migration_source_missing` 尚落在 `reboot_slo_unknown`;已把 API / CLI action matrix 同步分類為 `host_cpu_pressure`,owner lane `host_pressure_controller`,evidence inputs `source_controls` + `runtime_metric_readback`,next action `restore_conversation_event_hot_path_migration_source_then_rerun_host_pressure_and_reboot_slo_scorecard_no_restart`。 +- `FULL-STACK-COLD-START-SOP.md` 升到 v1.99,固定 runtime overlay blocker 不得留 unknown;此路徑只授權 source-control / verifier / check-mode,不授權 destructive migration、kill process、Docker / DB / K3s / Nginx restart 或 reboot。 + +**驗證**: +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py -q -p no:cacheprovider`:`9 passed`,鎖住 `conversation_event_hot_path_index_migration_source_missing` 的 `category=host_cpu_pressure`、`owner_lane=host_pressure_controller` 與 evidence inputs。 +- `python3.11 -m pytest scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/tests/test_reboot_event_detector.py -q -p no:cacheprovider`:`15 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py scripts/reboot-recovery/reboot-event-detector.py`:通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`:`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- `git diff --check`:通過。 + +**仍維持**: +- P0-006 仍未達 10 分鐘全主機自動恢復 SLO;primary 仍是 host reboot detection lane,Windows99 VMware verifier 仍缺 readback,111 unreachable / host uptime unknown 類 blocker 仍需繼續 verify-only 收斂。 +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth;未使用 GitHub / gh;未 workflow_dispatch;未重啟 host / VM / service;未 Docker / Nginx / K3s / DB / firewall restart;未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 00:43 P0-006 reboot-event detector unknown uptime false-fresh fix **完成內容**: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index e4921fe61..02ba75682 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,6 +1,6 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.98 +> Version: v1.99 > Last updated: 2026-07-03 Asia/Taipei > Scope: 110 / 120 / 121 / 188 full-stack reboot recovery. 112 Kali is recorded as P3 optional and is not part of this recovery path. @@ -30,6 +30,8 @@ v1.97 reboot SLO action-matrix routing rule:重啟後不得只把 active block v1.98 reboot-event detector unknown-uptime rule:`reboot-event-detector.py` 不得把 `uptime_seconds=unknown`、`boot_id=unknown` 或 `boot_id=reachable_unknown_boot` 當作 fresh reboot 或 boot id changed。只有 `reachable=true` 且 `uptime_seconds >= 0` 且 `uptime_seconds <= target_seconds` 才能進 `fresh_boot_hosts`;只有前後兩個 boot id 都不是 placeholder 時才能判定 `boot_id_changed`。若 99 只剩 ping / TCP reachable、111 unreachable、188 degraded 或 startup failed,必須維持 `all_required_hosts_in_reboot_window=false` 與 SLO blocked;不得用 ping-only host 當 10 分鐘內重啟證據。 +v1.99 runtime blocker action-matrix classification rule:Prometheus / runtime overlay 帶回的 `conversation_event_hot_path_index_migration_source_missing` 不得留在 `reboot_slo_unknown`。此 blocker 屬於 `host_cpu_pressure`,owner lane 固定為 `host_pressure_controller`,next action 固定為 `restore_conversation_event_hot_path_migration_source_then_rerun_host_pressure_and_reboot_slo_scorecard_no_restart`;evidence inputs 必須包含 `source_controls` 與 `runtime_metric_readback`。此規則只授權 source-control / verifier 修復與 check-mode,仍不得直接做 DB destructive migration、kill process、Docker / DB / K3s / Nginx restart 或 reboot。 + 2026-07-02 110 control-path / Harbor recovery receipt rule:若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure,但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`,則該 Gitea Harbor repair 失敗只能列為 historical queue metadata,不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier,並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示:public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`;active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure,以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker,non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker;non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule:110 runner 壓力事故後,legacy / generic runner 仍必須 fail-closed;但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail,且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時,才可保留 drain config / binary / unit,並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service;若 registration 缺失,readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走,優先修 source enforcer / unit guardrail,不要手工反覆補同一組 artifact。 diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index ff2eea3a6..e90ad58b6 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -1185,7 +1185,12 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str return "backup_readback_blocked" if "local_disk_free_below_minimum" in active_blockers: return "host_capacity_blocked" - if any(blocker.startswith("host_") or blocker.startswith("awooop_") for blocker in active_blockers): + if any( + blocker.startswith("host_") + or blocker.startswith("awooop_") + or blocker.startswith("conversation_event_hot_path_") + for blocker in active_blockers + ): return "host_pressure_blocked" return "slo_blocked" @@ -1456,15 +1461,27 @@ def active_blocker_action_row( "host_110_gitea_cpu_pressure", "host_188_postgres_cpu_pressure", "awooop_conversation_event_hot_path_index_drift_detected", + "conversation_event_hot_path_index_migration_source_missing", }: category = "host_cpu_pressure" owner_lane = "host_pressure_controller" - next_safe_action = ( - "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" - "verifier_green_no_restart" - ) - post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" - evidence_inputs = ["host_pressure"] + if blocker == "conversation_event_hot_path_index_migration_source_missing": + next_safe_action = ( + "restore_conversation_event_hot_path_migration_source_then_rerun_" + "host_pressure_and_reboot_slo_scorecard_no_restart" + ) + post_verifier = ( + "source_controls_conversation_event_hot_path_index_migration_" + "and_host_pressure_readback" + ) + evidence_inputs = ["source_controls", "runtime_metric_readback"] + else: + next_safe_action = ( + "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" + "verifier_green_no_restart" + ) + post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" + evidence_inputs = ["host_pressure"] controlled_apply_mode = "playbook_check_mode_then_controlled_apply_if_low_risk" forbidden_actions.extend( ["unmask_legacy_runner_or_restore_generic_labels", "kill_process_without_gate"]