diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index 10e10a553..88be898df 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -487,8 +487,28 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: readiness_percent=readiness_percent, can_claim_slo=can_claim_slo, ) + active_blocker_action_matrix = _build_active_blocker_action_matrix( + active_blockers=active_blockers, + primary_blocker=sop_progress["primary_blocker"], + ) rollups = { "active_blocker_count": active_blocker_count, + "active_blocker_action_count": active_blocker_action_matrix["item_count"], + "active_blocker_action_category_counts": active_blocker_action_matrix[ + "category_counts" + ], + "active_blocker_action_owner_lane_counts": active_blocker_action_matrix[ + "owner_lane_counts" + ], + "primary_blocker_owner_lane": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("owner_lane", ""), + "primary_blocker_action_category": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("category", ""), + "telegram_active_blocker_alert_required_count": active_blocker_action_matrix[ + "telegram_alert_required_count" + ], "readiness_percent": readiness_percent, "completed_check_count": completed_check_count, "required_check_count": len(required_checks), @@ -694,7 +714,17 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: "safe_next_step": safe_next_step, "next_safe_action": sop_progress["next_safe_action"], "primary_blocker": sop_progress["primary_blocker"], + "primary_blocker_owner_lane": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("owner_lane", ""), + "primary_blocker_action_category": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("category", ""), "active_blocker_count": active_blocker_count, + "active_blocker_action_count": active_blocker_action_matrix["item_count"], + "telegram_active_blocker_alert_required_count": ( + active_blocker_action_matrix["telegram_alert_required_count"] + ), "readiness_percent": readiness_percent, "blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only, "latest_verify_only_metric_present": latest_verify_only_metric_present, @@ -740,6 +770,7 @@ def _build_payload(scorecard: dict[str, Any], path: Path) -> dict[str, Any]: ], }, "reboot_sop_progress": sop_progress, + "active_blocker_action_matrix": active_blocker_action_matrix, "controlled_service_data_backup_readback": ( controlled_service_data_backup_readback ), @@ -1080,8 +1111,8 @@ def _reboot_sop_current_phase(active_blockers: list[str], can_claim_slo: bool) - "reboot_event_required_host_unreachable", "fresh_all_host_reboot_event_missing", "all_required_hosts_not_in_10_minute_reboot_window", - "windows99_vmware_autostart_readback_missing", "windows99_remote_execution_channel_unavailable", + "windows99_vmware_autostart_readback_missing", "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", @@ -1129,8 +1160,8 @@ def _reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "all_required_hosts_not_in_10_minute_reboot_window", "host_boot_observation_older_than_target_window", "host_uptime_unknown", - "windows99_vmware_autostart_readback_missing", "windows99_remote_execution_channel_unavailable", + "windows99_vmware_autostart_readback_missing", "windows99_vmrun_missing", "windows99_vmware_vmx_missing", "windows99_vmware_autostart_config_not_ready", @@ -1185,6 +1216,228 @@ def _reboot_sop_eta_or_wait_reason( return "eta_unavailable" +def _build_active_blocker_action_matrix( + *, + active_blockers: list[str], + primary_blocker: str, +) -> dict[str, Any]: + items = [ + _active_blocker_action_row( + blocker=blocker, + priority_order=index, + is_primary=blocker == primary_blocker, + ) + for index, blocker in enumerate(active_blockers, start=1) + ] + primary_action = next( + (item for item in items if item.get("is_primary") is True), + {}, + ) + return { + "schema_version": "reboot_active_blocker_action_matrix_v1", + "item_count": len(items), + "telegram_alert_required_count": sum( + 1 for item in items if item.get("telegram_alert_required") is True + ), + "category_counts": _count_by(items, "category"), + "owner_lane_counts": _count_by(items, "owner_lane"), + "primary_blocker_action": primary_action, + "items": items, + } + + +def _active_blocker_action_row( + *, + blocker: str, + priority_order: int, + is_primary: bool, +) -> dict[str, Any]: + category = "reboot_slo_unknown" + owner_lane = "sre_reboot_slo_scorecard" + severity = "critical" + next_safe_action = ( + "read_active_blocker_evidence_then_rerun_reboot_slo_scorecard_no_reboot" + ) + post_verifier = ( + "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py " + "--output " + ) + evidence_inputs = ["scorecard.active_blockers"] + controlled_apply_mode = "verifier_only_from_reboot_slo_lane" + forbidden_actions = [ + "host_reboot", + "service_restart", + "docker_daemon_restart", + "node_drain", + "database_write_or_restore", + "secret_value_read", + "workflow_dispatch", + "github_api", + "force_push_or_ref_delete", + ] + + if blocker.startswith("windows99_"): + category = "windows99_vmware_autostart" + owner_lane = "windows99_console_or_no_secret_management_channel" + next_safe_action = ( + "collect_windows99_vmware_autostart_verify_readback_then_rerun_" + "reboot_scorecard_no_secret_no_reboot" + ) + if blocker == "windows99_remote_execution_channel_unavailable": + next_safe_action = ( + "restore_windows99_no_secret_management_channel_or_collect_local_" + "console_verify_readback_then_rerun_reboot_scorecard_no_reboot" + ) + post_verifier = ( + "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh " + "--check && rerun_reboot_auto_recovery_slo_scorecard" + ) + evidence_inputs = [ + "windows99_management_channel", + "windows99_vmware_autostart", + "windows99_verify_collection", + ] + controlled_apply_mode = "console_verify_or_no_secret_collector_only" + forbidden_actions.extend( + [ + "windows_password_or_secret_collection", + "vm_power_change", + "windows_update_policy_apply_from_scorecard", + "manual_registry_edit", + ] + ) + elif blocker in { + "all_host_reboot_detection_missing", + "stateful_reboot_event_detection_missing", + "host_boot_probe_missing_hosts", + "host_unreachable_after_reboot", + "host_boot_observation_older_than_target_window", + "host_uptime_unknown", + "reboot_event_missing_required_hosts", + "reboot_event_required_host_unreachable", + "fresh_all_host_reboot_event_missing", + "all_required_hosts_not_in_10_minute_reboot_window", + }: + category = "host_boot_detection" + owner_lane = "reboot_event_detector_and_host_probe" + next_safe_action = ( + "rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot" + ) + post_verifier = ( + "scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh && " + "scripts/reboot-recovery/reboot-event-detector.py" + ) + evidence_inputs = ["host_boot_detection", "reboot_event_detection"] + elif blocker in { + "post_reboot_summary_missing", + "post_start_blocked_not_zero", + "service_green_not_1", + "host_188_service_green_not_1", + }: + category = "post_reboot_service_readiness" + owner_lane = "service_readiness_and_k3s_workloads" + next_safe_action = ( + "collect_post_reboot_readiness_summary_and_public_health_no_restart" + ) + post_verifier = "scripts/reboot-recovery/post-reboot-readiness-summary.sh" + evidence_inputs = ["post_reboot_readiness", "public_health_readback"] + elif blocker == "backup_core_green_not_1" or "backup" in blocker: + category = "backup_observability" + owner_lane = "backup_health_and_restore_drill" + next_safe_action = ( + "collect_backup_health_restore_drill_and_offsite_readback_no_restore" + ) + post_verifier = "scripts/ops/backup-health-textfile-exporter.py --help" + evidence_inputs = [ + "post_reboot_readiness.backup_core_green", + "backup_health_textfile", + "restore_drill_readback", + ] + elif blocker == "wazuh_dashboard_degraded": + category = "security_observability" + owner_lane = "wazuh_dashboard_and_security_readback" + severity = "warning" + next_safe_action = "collect_wazuh_dashboard_readback_no_service_restart" + post_verifier = "read_wazuh_dashboard_or_registry_health_readback" + evidence_inputs = ["post_reboot_readiness.wazuh_dashboard_degraded"] + elif blocker in _PUBLIC_MAINTENANCE_BLOCKERS or blocker.startswith("public_"): + category = "public_maintenance_fallback" + owner_lane = "public_route_maintenance_fallback" + next_safe_action = ( + "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload" + ) + post_verifier = ( + "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py" + ) + evidence_inputs = ["public_maintenance_fallback"] + controlled_apply_mode = "check_mode_then_separate_edge_or_nginx_apply" + elif "stockplatform" in blocker or "product_data" in blocker: + category = "product_data_freshness" + owner_lane = "stockplatform_freshness_and_ingestion" + next_safe_action = ( + "inspect_stockplatform_freshness_ingestion_readback_then_check_mode_" + "recovery_if_required_no_manual_db_write" + ) + post_verifier = "stockplatform_public_api_freshness_and_ingestion_readback" + evidence_inputs = ["stockplatform_data_freshness"] + controlled_apply_mode = "check_mode_then_controlled_data_recovery_lane" + forbidden_actions.extend( + ["manual_zero_fill", "copy_previous_trading_day_data"] + ) + elif blocker in { + "host_pressure_high_load", + "host_container_cpu_attribution_stale", + "host_110_gitea_cpu_pressure", + "host_188_postgres_cpu_pressure", + "awooop_conversation_event_hot_path_index_drift_detected", + }: + category = "host_cpu_pressure" + owner_lane = "host_pressure_controller" + next_safe_action = ( + "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" + "verifier_green_no_restart" + ) + post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" + evidence_inputs = ["host_pressure"] + controlled_apply_mode = "playbook_check_mode_then_controlled_apply_if_low_risk" + forbidden_actions.extend( + ["unmask_legacy_runner_or_restore_generic_labels", "kill_process_without_gate"] + ) + elif blocker == "local_disk_free_below_minimum": + category = "host_capacity" + owner_lane = "capacity_and_disk_hygiene" + next_safe_action = "collect_disk_usage_inventory_and_prepare_cleanup_dry_run" + post_verifier = "df_and_capacity_scorecard_readback" + evidence_inputs = ["capacity"] + controlled_apply_mode = "cleanup_dry_run_then_controlled_apply" + + return { + "blocker": blocker, + "priority_order": priority_order, + "is_primary": is_primary, + "category": category, + "owner_lane": owner_lane, + "telegram_alert": "RebootAutoRecoveryActiveBlocker", + "telegram_alert_required": True, + "telegram_severity": severity, + "metric_name": "awoooi_reboot_auto_recovery_slo_active_blocker", + "evidence_inputs": evidence_inputs, + "next_safe_action": next_safe_action, + "post_verifier": post_verifier, + "controlled_apply_mode": controlled_apply_mode, + "controlled_apply_authorized_by_scorecard": False, + "forbidden_actions": _unique_strings(forbidden_actions), + } + + +def _count_by(items: list[dict[str, Any]], key: str) -> dict[str, int]: + counts: dict[str, int] = {} + for item in items: + value = str(item.get(key) or "unknown") + counts[value] = counts.get(value, 0) + 1 + return counts + + def _require_operation_boundaries(payload: dict[str, Any], label: str) -> None: boundaries = _dict(payload.get("operation_boundaries")) forbidden_true = [ diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 696f6ceaa..0c497a434 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -474,6 +474,43 @@ def _assert_reboot_slo_payload(payload: dict): is False ) assert payload["active_blockers"] == EXPECTED_REBOOT_SLO_BLOCKERS + matrix = payload["active_blocker_action_matrix"] + assert matrix["schema_version"] == "reboot_active_blocker_action_matrix_v1" + assert matrix["item_count"] == 11 + assert matrix["telegram_alert_required_count"] == 11 + assert matrix["primary_blocker_action"]["blocker"] == ( + "reboot_event_required_host_unreachable" + ) + assert matrix["primary_blocker_action"]["category"] == "host_boot_detection" + assert matrix["primary_blocker_action"]["owner_lane"] == ( + "reboot_event_detector_and_host_probe" + ) + assert matrix["category_counts"]["host_boot_detection"] == 5 + assert matrix["category_counts"]["windows99_vmware_autostart"] == 2 + assert matrix["category_counts"]["backup_observability"] == 1 + action_by_blocker = {item["blocker"]: item for item in matrix["items"]} + assert action_by_blocker["windows99_remote_execution_channel_unavailable"][ + "owner_lane" + ] == "windows99_console_or_no_secret_management_channel" + assert action_by_blocker["backup_core_green_not_1"]["category"] == ( + "backup_observability" + ) + assert action_by_blocker["wazuh_dashboard_degraded"][ + "telegram_severity" + ] == "warning" + assert action_by_blocker["reboot_event_required_host_unreachable"][ + "controlled_apply_authorized_by_scorecard" + ] is False + assert "host_reboot" in action_by_blocker[ + "reboot_event_required_host_unreachable" + ]["forbidden_actions"] + assert payload["readback"]["active_blocker_action_count"] == 11 + assert payload["readback"]["primary_blocker_owner_lane"] == ( + "reboot_event_detector_and_host_probe" + ) + assert payload["rollups"]["active_blocker_action_category_counts"][ + "windows99_vmware_autostart" + ] == 2 progress = payload["reboot_sop_progress"] assert progress["current_phase"] == "host_boot_detection_blocked" assert progress["eta_or_wait_reason"] == ( diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 05ca31045..5dc376e10 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,25 @@ +## 2026-07-03 — 00:36 P0-006 reboot SLO action-matrix alert routing + +**完成內容**： +- 先對齊 Gitea 主線 `f700489bb fix(reboot): load runtime slo scorecard artifact`，保留 production API 從 runtime `scorecard.json` 讀取 reboot SLO truth 的修正，再接續本輪 action matrix。 +- `apps/api/src/services/reboot_auto_recovery_slo_scorecard.py` 與 `scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py` 新增 `active_blocker_action_matrix`：每個 active blocker 固定輸出 `category`、`owner_lane`、`telegram_severity`、`evidence_inputs`、`next_safe_action`、`post_verifier`、`controlled_apply_mode`、`forbidden_actions` 與 `controlled_apply_authorized_by_scorecard=false`。 +- `scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh` 將 active blocker metric 擴充為 `blocker/category/owner_lane/severity/primary` labels；`ops/monitoring/alerts-unified.yml` 的 `RebootAutoRecoveryActiveBlocker` 會在 Telegram / Alertmanager annotation 直接帶出 action lane，不再只給總 blocker count。 +- 優先序固定：`windows99_remote_execution_channel_unavailable` 高於 `windows99_vmware_autostart_readback_missing`，避免 99 no-secret collection channel 尚未恢復時只等待 VMware verifier；host reboot detection、backup observability、Wazuh security readback、public maintenance fallback、product freshness、host CPU pressure、capacity 都有固定 owner lane 與 post-verifier。 +- `FULL-STACK-COLD-START-SOP.md` 升到 v1.97，明確禁止把 scorecard 當作重啟、VM power change、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete 或 secret 讀取授權；scorecard 只負責 verifier / check-mode 分流。 + +**驗證**： +- `DATABASE_URL=postgresql+asyncpg://test:test@localhost/test PYTHONPATH=apps/api python3.11 -m pytest apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py -q -p no:cacheprovider`：`20 passed`。 +- `python3.11 -m py_compile apps/api/src/services/reboot_auto_recovery_slo_scorecard.py scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py`：通過。 +- `bash -n scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh`：通過。 +- `python3.11 ops/runner/guard-gitea-runner-pressure.py --root .`：`GITEA_RUNNER_PRESSURE_GUARD_OK workflow_files=12 scheduled_workflows=4 auto_branch_events_on_110=0 generic_runner_labels=0`。 +- API local readback：`status=blocked_reboot_auto_recovery_slo_not_ready`、`active_blockers=11`、`active_blocker_action_matrix.item_count=11`、primary `reboot_event_required_host_unreachable`、primary lane `reboot_event_detector_and_host_probe`、category counts `host_boot_detection=5 / windows99_vmware_autostart=2 / backup_observability=1 / post_reboot_service_readiness=2 / security_observability=1`。 +- CLI fail-closed readback：空輸入維持 `blocked_reboot_auto_recovery_slo_not_ready`、`active_blocker_action_matrix.item_count=11`、primary `all_host_reboot_detection_missing`、primary lane `reboot_event_detector_and_host_probe`。 + +**仍維持**： +- Production `#4476` 已部署 source `7e39dd2880`、deploy marker `90c0d5b30`，且 `/api/v1/agents/delivery-closure-workbench` 讀回 `matches_main=true`；本輪 action matrix 尚需下一次 Gitea CD 後才會成為 production API truth。 +- P0-006 仍 fail-closed：最新 production reboot SLO scorecard 仍有 `active_blocker_count=11`，包含 99 remote execution channel、VMware autostart readback、host boot detection、service/backup/Wazuh 類 blocker；不得宣稱 10 分鐘全主機自動恢復 SLO 已完成。 +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未重啟 host / VM / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 00:11 P0-006 latest CD profile / Windows99 live probe readback **完成內容**： diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 7578b0950..b5159acbb 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.96 -> Last updated: 2026-07-02 Asia/Taipei +> Version: v1.97 +> Last updated: 2026-07-03 Asia/Taipei > Scope: 110 / 120 / 121 / 188 full-stack reboot recovery. 112 Kali is recorded as P3 optional and is not part of this recovery path. --- @@ -26,6 +26,8 @@ v1.95 Windows 99 management-channel readback rule：若 99 可 ping / RDP / TCP v1.96 reboot SLO per-blocker alert projection rule：重啟後不得只看 `awoooi_reboot_auto_recovery_slo_blocker_count` 或 `RebootAutoRecoverySLOMissed` 總告警。`scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh` 必須從同一份 `scorecard.json` 產生 `awoooi_reboot_auto_recovery_slo_active_blocker{blocker=...}` 與 `awoooi_reboot_auto_recovery_slo_primary_blocker{blocker=...}` textfile metrics；Prometheus 必須載入 `RebootAutoRecoveryActiveBlocker` 與 `RebootAutoRecoveryActiveBlockerMetricMissing`。Telegram / Alertmanager 看到 blocker label 後，先按 `blocker` 指向的固定 lane 處理：`windows99_*` 走 no-secret collector 或 console verify、`backup_*` 走 backup-status / exporter readback、`wazuh_dashboard_degraded` 走 Wazuh registry/API 只讀、`host_*` / `reboot_event_*` 走 host-probe / reboot-event-state，同時禁止 reboot、Docker / systemd / Nginx / DB / firewall restart、restore、prune、delete。若 blocker_count 大於 0 但 active_blocker metric 缺失，視為監控鏈缺口，不可宣稱「有告警」已足夠。 +v1.97 reboot SLO action-matrix routing rule：重啟後不得只把 active blocker 丟給人工判讀。`reboot-auto-recovery-slo-scorecard.py`、production `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 與 exporter artifact 必須輸出 `active_blocker_action_matrix`，每個 blocker 固定含 `category`、`owner_lane`、`telegram_severity`、`evidence_inputs`、`next_safe_action`、`post_verifier`、`controlled_apply_mode` 與 `forbidden_actions`。`awoooi_reboot_auto_recovery_slo_active_blocker` metric 必須帶 `category`、`owner_lane`、`severity`、`primary` labels，Telegram / Alertmanager 優先讀這些 labels；若 labels 缺失，只能判為 exporter/action-matrix drift。`windows99_remote_execution_channel_unavailable` 必須排在 `windows99_vmware_autostart_readback_missing` 前，因為未恢復 no-secret collection channel 時不可只等 VMware verifier。所有 action matrix row 的 `controlled_apply_authorized_by_scorecard=false`，表示 scorecard 只授權 verifier / check-mode 路由，不授權重啟、VM power change、Docker / Nginx / K3s / DB / firewall restart、restore、prune、delete 或 secret 讀取；低風險 controlled apply 必須進各自 lane 的 check-mode、rollback、post-verifier。 + 2026-07-02 110 control-path / Harbor recovery receipt rule：若 Gitea Harbor repair queue 仍保留 `harbor_110_remote_ssh_publickey_auth_stalled`、remote-control unavailable、jobs stale 或 historical failure，但同一輪本地證據同時證明 `wooo` command path ready、110 local Harbor `/v2/` ready、public/internal registry `/v2/` 回 `401`，則該 Gitea Harbor repair 失敗只能列為 historical queue metadata，不得再當成 current SSH blocker。必須用 `/api/v1/agents/harbor-registry-controlled-recovery-receipt` 或同等 validator 合併 `diagnose-110-ssh-publickey-auth.sh`、`recover-110-control-path-and-harbor-local.sh --check`、public Gitea queue readback 與 registry `/v2/` verifier，並把機器可讀結果寫入 `docs/operations/harbor-110-control-path-recovery-readback-2026-07-02.snapshot.json` 類型的 snapshot。2026-07-02 live receipt 顯示：public/internal registry `/v2/` 均為 `401`、latest visible CD `#4335` 為 `Success`、Gitea Harbor repair failure 已是 `historical_after_latest_cd_success=true`；active blockers 收斂為 110 controlled CD lane config / binary / registration / service guardrail、active action container pressure，以及 Gitea CD jobs head-SHA / stale readback mismatch。若 local-console output 只有 `AWOOOI_110_CONTROLLED_CD_LANE_READY` marker，non110 runner parser 不得從 110 `BLOCKER` 行推導 non110 blocker；non110 只有看到 `AWOOOI_NON110_RUNNER_READY` marker 才能列入 active blocker。 2026-07-02 110 controlled CD lane fail-closed enforcer staging rule：110 runner 壓力事故後，legacy / generic runner 仍必須 fail-closed；但 `awoooi-cd-lane-drain.service` 的非 secret staging artifact 不得再被 enforcer 無差別封回 stub。`scripts/reboot-recovery/enforce-110-runner-failclosed.sh` 只有在 `config.yaml` 符合 `capacity <= 1`、只含 `awoooi-host:host` 與 `awoooi-ubuntu:docker://192.168.0.110:5000/awoooi/ci-runner:act-22.04`、binary 是 executable ELF、systemd unit 具備 `ConditionPathExists=/home/wooo/awoooi-cd-lane-drain/data/.runner`、`CPUAccounting` / `MemoryAccounting` / `TasksAccounting` / `NoNewPrivileges` 等 guardrail，且 service `inactive`、`MainPID=0`、未 enabled / 未 masked 時，才可保留 drain config / binary / unit，並輸出 `CONTROLLED_DRAIN_STAGING_ALLOWED=1` 與 textfile metric。此 staging 規則不得讀 token、不得讀 `.runner` 內容、不得註冊 runner、不得啟動 service；若 registration 缺失，readiness verifier 仍必須只留下 `controlled_cd_lane_registration_missing` / `controlled_cd_lane_service_not_active` 類 blocker。若 `CONTROLLED_DRAIN_STAGING_ALLOWED=0` 且 config / binary 又被搬走，優先修 source enforcer / unit guardrail，不要手工反覆補同一組 artifact。 diff --git a/ops/monitoring/alerts-unified.yml b/ops/monitoring/alerts-unified.yml index c508f2c70..cd06ea79b 100644 --- a/ops/monitoring/alerts-unified.yml +++ b/ops/monitoring/alerts-unified.yml @@ -223,8 +223,8 @@ groups: auto_repair: "false" annotations: summary: "重啟恢復 SLO blocker 未清除：{{ $labels.blocker }}" - description: "reboot-auto-recovery scorecard 仍有 active blocker={{ $labels.blocker }}；不可只看總 blocker_count，必須對應處理 99/VMware、主機可達、service/backup/Wazuh 或 public fallback 的具體證據。" - runbook: "先讀 110 最新 /home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/scorecard.json、summary.txt、host-probe.txt 與同 run 的 public-maintenance/windows99 readback；Windows99 只走 no-secret collector 或 console verify，backup 只讀 backup-status/exporter，Wazuh 只讀 registry/API；禁止 reboot、Docker/systemd/Nginx/DB/firewall restart、restore/prune/delete。" + description: "reboot-auto-recovery scorecard 仍有 active blocker={{ $labels.blocker }}；category={{ $labels.category }}、owner_lane={{ $labels.owner_lane }}、primary={{ $labels.primary }}。不可只看總 blocker_count，必須依 action matrix 處理 99/VMware、主機可達、service/backup/Wazuh、CPU pressure 或 public fallback 的具體證據。" + runbook: "先讀 110 最新 /home/wooo/reboot-recovery/reboot-auto-recovery-slo-*/scorecard.json 的 active_blocker_action_matrix，再讀 summary.txt、host-probe.txt 與同 run 的 public-maintenance/windows99 readback；Windows99 只走 no-secret collector 或 console verify，backup 只讀 backup-status/exporter，Wazuh 只讀 registry/API；禁止 reboot、Docker/systemd/Nginx/DB/firewall restart、restore/prune/delete。" - alert: RebootAutoRecoveryActiveBlockerMetricMissing expr: | diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 2ad93d2e3..4663c1aa3 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -155,6 +155,15 @@ import json, sys payload = json.load(open(sys.argv[1], encoding="utf-8")) target_minutes = str(sys.argv[2]) scope = "99_110_111_112_120_121_188" +matrix = payload.get("active_blocker_action_matrix") or {} +items = matrix.get("items") if isinstance(matrix, dict) else [] +if not isinstance(items, list): + items = [] +actions_by_blocker = { + str(item.get("blocker") or ""): item + for item in items + if isinstance(item, dict) and item.get("blocker") +} def label(value: object) -> str: @@ -168,16 +177,27 @@ def label(value: object) -> str: for blocker in payload.get("active_blockers") or []: if blocker: + action = actions_by_blocker.get(str(blocker), {}) print( "awoooi_reboot_auto_recovery_slo_active_blocker" - f'{{scope="{scope}",target_minutes="{label(target_minutes)}",blocker="{label(blocker)}"}} 1' + f'{{scope="{scope}",target_minutes="{label(target_minutes)}",' + f'blocker="{label(blocker)}",' + f'category="{label(action.get("category"))}",' + f'owner_lane="{label(action.get("owner_lane"))}",' + f'severity="{label(action.get("telegram_severity"))}",' + f'primary="{label(str(action.get("is_primary") is True).lower())}"}} 1' ) primary_blocker = payload.get("primary_blocker") or "" if primary_blocker: + action = actions_by_blocker.get(str(primary_blocker), {}) print( "awoooi_reboot_auto_recovery_slo_primary_blocker" - f'{{scope="{scope}",target_minutes="{label(target_minutes)}",blocker="{label(primary_blocker)}"}} 1' + f'{{scope="{scope}",target_minutes="{label(target_minutes)}",' + f'blocker="{label(primary_blocker)}",' + f'category="{label(action.get("category"))}",' + f'owner_lane="{label(action.get("owner_lane"))}",' + f'severity="{label(action.get("telegram_severity"))}"}} 1' ) PY )" @@ -190,9 +210,9 @@ awoooi_reboot_auto_recovery_slo_ready{scope="99_110_111_112_120_121_188",target_ # HELP awoooi_reboot_auto_recovery_slo_blocker_count Number of active fail-closed blockers for the reboot recovery SLO. # TYPE awoooi_reboot_auto_recovery_slo_blocker_count gauge awoooi_reboot_auto_recovery_slo_blocker_count{scope="99_110_111_112_120_121_188",target_minutes="$TARGET_MINUTES"} $blocker_count -# HELP awoooi_reboot_auto_recovery_slo_active_blocker Active fail-closed reboot SLO blockers projected from scorecard active_blockers. +# HELP awoooi_reboot_auto_recovery_slo_active_blocker Active fail-closed reboot SLO blockers projected from scorecard active_blockers with category and owner lane labels. # TYPE awoooi_reboot_auto_recovery_slo_active_blocker gauge -# HELP awoooi_reboot_auto_recovery_slo_primary_blocker Primary reboot SLO blocker selected by the fixed SOP priority order. +# HELP awoooi_reboot_auto_recovery_slo_primary_blocker Primary reboot SLO blocker selected by the fixed SOP priority order with category and owner lane labels. # TYPE awoooi_reboot_auto_recovery_slo_primary_blocker gauge # HELP awoooi_reboot_auto_recovery_slo_max_host_uptime_seconds Max observed host uptime in the boot probe. # TYPE awoooi_reboot_auto_recovery_slo_max_host_uptime_seconds gauge diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py index a49318fc1..ff2eea3a6 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py @@ -104,6 +104,17 @@ def strings(value: Any) -> list[str]: return [str(item) for item in value if item not in (None, "")] +def unique_strings(values: list[str]) -> list[str]: + seen: set[str] = set() + unique: list[str] = [] + for value in values: + if value in seen: + continue + seen.add(value) + unique.append(value) + return unique + + def csv_strings(value: str | None) -> list[str]: if not value or value in {"none", "unknown"}: return [] @@ -1147,6 +1158,7 @@ def reboot_sop_current_phase(active_blockers: list[str], can_claim: bool) -> str "reboot_event_required_host_unreachable", "fresh_all_host_reboot_event_missing", "all_required_hosts_not_in_10_minute_reboot_window", + "windows99_remote_execution_channel_unavailable", "windows99_vmware_autostart_readback_missing", "windows99_vmrun_missing", "windows99_vmware_vmx_missing", @@ -1189,6 +1201,7 @@ def reboot_sop_primary_blocker(active_blockers: list[str]) -> str: "all_required_hosts_not_in_10_minute_reboot_window", "host_boot_observation_older_than_target_window", "host_uptime_unknown", + "windows99_remote_execution_channel_unavailable", "windows99_vmware_autostart_readback_missing", "windows99_vmrun_missing", "windows99_vmware_vmx_missing", @@ -1273,6 +1286,224 @@ def build_reboot_sop_progress( } +def build_active_blocker_action_matrix( + active_blockers: list[str], + primary_blocker: str, +) -> dict[str, Any]: + items = [ + active_blocker_action_row( + blocker=blocker, + priority_order=index, + is_primary=blocker == primary_blocker, + ) + for index, blocker in enumerate(active_blockers, start=1) + ] + primary_action = next((item for item in items if item.get("is_primary") is True), {}) + return { + "schema_version": "reboot_active_blocker_action_matrix_v1", + "item_count": len(items), + "telegram_alert_required_count": sum( + 1 for item in items if item.get("telegram_alert_required") is True + ), + "category_counts": count_by(items, "category"), + "owner_lane_counts": count_by(items, "owner_lane"), + "primary_blocker_action": primary_action, + "items": items, + } + + +def active_blocker_action_row( + *, + blocker: str, + priority_order: int, + is_primary: bool, +) -> dict[str, Any]: + category = "reboot_slo_unknown" + owner_lane = "sre_reboot_slo_scorecard" + severity = "critical" + next_safe_action = ( + "read_active_blocker_evidence_then_rerun_reboot_slo_scorecard_no_reboot" + ) + post_verifier = ( + "scripts/reboot-recovery/reboot-auto-recovery-slo-scorecard.py " + "--output " + ) + evidence_inputs = ["scorecard.active_blockers"] + controlled_apply_mode = "verifier_only_from_reboot_slo_lane" + forbidden_actions = [ + "host_reboot", + "service_restart", + "docker_daemon_restart", + "node_drain", + "database_write_or_restore", + "secret_value_read", + "workflow_dispatch", + "github_api", + "force_push_or_ref_delete", + ] + + if blocker.startswith("windows99_"): + category = "windows99_vmware_autostart" + owner_lane = "windows99_console_or_no_secret_management_channel" + next_safe_action = ( + "collect_windows99_vmware_autostart_verify_readback_then_rerun_" + "reboot_scorecard_no_secret_no_reboot" + ) + if blocker == "windows99_remote_execution_channel_unavailable": + next_safe_action = ( + "restore_windows99_no_secret_management_channel_or_collect_local_" + "console_verify_readback_then_rerun_reboot_scorecard_no_reboot" + ) + post_verifier = ( + "bash scripts/reboot-recovery/collect-windows99-vmware-verify.sh " + "--check && rerun_reboot_auto_recovery_slo_scorecard" + ) + evidence_inputs = [ + "windows99_management_channel", + "windows99_vmware_autostart", + "windows99_verify_collection", + ] + controlled_apply_mode = "console_verify_or_no_secret_collector_only" + forbidden_actions.extend( + [ + "windows_password_or_secret_collection", + "vm_power_change", + "windows_update_policy_apply_from_scorecard", + "manual_registry_edit", + ] + ) + elif blocker in { + "all_host_reboot_detection_missing", + "stateful_reboot_event_detection_missing", + "host_boot_probe_missing_hosts", + "host_unreachable_after_reboot", + "host_boot_observation_older_than_target_window", + "host_uptime_unknown", + "reboot_event_missing_required_hosts", + "reboot_event_required_host_unreachable", + "fresh_all_host_reboot_event_missing", + "all_required_hosts_not_in_10_minute_reboot_window", + }: + category = "host_boot_detection" + owner_lane = "reboot_event_detector_and_host_probe" + next_safe_action = ( + "rerun_reboot_event_detector_and_host_probe_verify_only_no_reboot" + ) + post_verifier = ( + "scripts/reboot-recovery/reboot-auto-recovery-host-probe.sh && " + "scripts/reboot-recovery/reboot-event-detector.py" + ) + evidence_inputs = ["host_boot_detection", "reboot_event_detection"] + elif blocker in { + "post_reboot_summary_missing", + "post_start_blocked_not_zero", + "service_green_not_1", + "host_188_service_green_not_1", + }: + category = "post_reboot_service_readiness" + owner_lane = "service_readiness_and_k3s_workloads" + next_safe_action = ( + "collect_post_reboot_readiness_summary_and_public_health_no_restart" + ) + post_verifier = "scripts/reboot-recovery/post-reboot-readiness-summary.sh" + evidence_inputs = ["post_reboot_readiness", "public_health_readback"] + elif blocker == "backup_core_green_not_1" or "backup" in blocker: + category = "backup_observability" + owner_lane = "backup_health_and_restore_drill" + next_safe_action = ( + "collect_backup_health_restore_drill_and_offsite_readback_no_restore" + ) + post_verifier = "scripts/ops/backup-health-textfile-exporter.py --help" + evidence_inputs = [ + "post_reboot_readiness.backup_core_green", + "backup_health_textfile", + "restore_drill_readback", + ] + elif blocker == "wazuh_dashboard_degraded": + category = "security_observability" + owner_lane = "wazuh_dashboard_and_security_readback" + severity = "warning" + next_safe_action = "collect_wazuh_dashboard_readback_no_service_restart" + post_verifier = "read_wazuh_dashboard_or_registry_health_readback" + evidence_inputs = ["post_reboot_readiness.wazuh_dashboard_degraded"] + elif blocker.startswith("public_"): + category = "public_maintenance_fallback" + owner_lane = "public_route_maintenance_fallback" + next_safe_action = ( + "run_public_maintenance_fallback_probe_and_rerun_scorecard_no_reload" + ) + post_verifier = ( + "python3 scripts/reboot-recovery/public-maintenance-fallback-probe.py" + ) + evidence_inputs = ["public_maintenance_fallback"] + controlled_apply_mode = "check_mode_then_separate_edge_or_nginx_apply" + elif "stockplatform" in blocker or "product_data" in blocker: + category = "product_data_freshness" + owner_lane = "stockplatform_freshness_and_ingestion" + next_safe_action = ( + "inspect_stockplatform_freshness_ingestion_readback_then_check_mode_" + "recovery_if_required_no_manual_db_write" + ) + post_verifier = "stockplatform_public_api_freshness_and_ingestion_readback" + evidence_inputs = ["stockplatform_data_freshness"] + controlled_apply_mode = "check_mode_then_controlled_data_recovery_lane" + forbidden_actions.extend( + ["manual_zero_fill", "copy_previous_trading_day_data"] + ) + elif blocker in { + "host_pressure_high_load", + "host_container_cpu_attribution_stale", + "host_110_gitea_cpu_pressure", + "host_188_postgres_cpu_pressure", + "awooop_conversation_event_hot_path_index_drift_detected", + }: + category = "host_cpu_pressure" + owner_lane = "host_pressure_controller" + next_safe_action = ( + "collect_sanitized_host_pressure_and_apply_allowed_playbook_if_" + "verifier_green_no_restart" + ) + post_verifier = "host_pressure_readback_and_reboot_slo_scorecard" + evidence_inputs = ["host_pressure"] + controlled_apply_mode = "playbook_check_mode_then_controlled_apply_if_low_risk" + forbidden_actions.extend( + ["unmask_legacy_runner_or_restore_generic_labels", "kill_process_without_gate"] + ) + elif blocker == "local_disk_free_below_minimum": + category = "host_capacity" + owner_lane = "capacity_and_disk_hygiene" + next_safe_action = "collect_disk_usage_inventory_and_prepare_cleanup_dry_run" + post_verifier = "df_and_capacity_scorecard_readback" + evidence_inputs = ["capacity"] + controlled_apply_mode = "cleanup_dry_run_then_controlled_apply" + + return { + "blocker": blocker, + "priority_order": priority_order, + "is_primary": is_primary, + "category": category, + "owner_lane": owner_lane, + "telegram_alert": "RebootAutoRecoveryActiveBlocker", + "telegram_alert_required": True, + "telegram_severity": severity, + "metric_name": "awoooi_reboot_auto_recovery_slo_active_blocker", + "evidence_inputs": evidence_inputs, + "next_safe_action": next_safe_action, + "post_verifier": post_verifier, + "controlled_apply_mode": controlled_apply_mode, + "controlled_apply_authorized_by_scorecard": False, + "forbidden_actions": unique_strings(forbidden_actions), + } + + +def count_by(items: list[dict[str, Any]], key: str) -> dict[str, int]: + counts: dict[str, int] = {} + for item in items: + value = str(item.get(key) or "unknown") + counts[value] = counts.get(value, 0) + 1 + return counts + + def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: """Add machine-readable P0-006 readback fields to the source scorecard.""" active_blockers = strings(payload.get("active_blockers")) @@ -1322,6 +1553,10 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "host_boot_observation_older_than_target_window" ] sop_progress = build_reboot_sop_progress(payload, active_blockers, readiness_percent) + active_blocker_action_matrix = build_active_blocker_action_matrix( + active_blockers, + sop_progress["primary_blocker"], + ) source_control_ready_count = sum(1 for value in controls.values() if value) source_controls_present = ( bool(controls) and source_control_ready_count == len(controls) @@ -1333,6 +1568,22 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: rollups = { "active_blocker_count": len(active_blockers), + "active_blocker_action_count": active_blocker_action_matrix["item_count"], + "active_blocker_action_category_counts": active_blocker_action_matrix[ + "category_counts" + ], + "active_blocker_action_owner_lane_counts": active_blocker_action_matrix[ + "owner_lane_counts" + ], + "primary_blocker_owner_lane": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("owner_lane", ""), + "primary_blocker_action_category": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("category", ""), + "telegram_active_blocker_alert_required_count": active_blocker_action_matrix[ + "telegram_alert_required_count" + ], "readiness_percent": readiness_percent, "completed_check_count": completed_check_count, "required_check_count": required_check_count, @@ -1464,8 +1715,18 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: "safe_next_step": str(payload.get("safe_next_step") or ""), "next_safe_action": sop_progress["next_safe_action"], "primary_blocker": sop_progress["primary_blocker"], + "primary_blocker_owner_lane": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("owner_lane", ""), + "primary_blocker_action_category": active_blocker_action_matrix[ + "primary_blocker_action" + ].get("category", ""), "active_blockers": active_blockers, "active_blocker_count": len(active_blockers), + "active_blocker_action_count": active_blocker_action_matrix["item_count"], + "telegram_active_blocker_alert_required_count": active_blocker_action_matrix[ + "telegram_alert_required_count" + ], "readiness_percent": readiness_percent, "blocked_by_fresh_reboot_window_only": blocked_by_fresh_reboot_window_only, "required_checks": required_checks, @@ -1565,6 +1826,7 @@ def enrich_machine_readback(payload: dict[str, Any]) -> dict[str, Any]: payload["primary_blocker"] = sop_progress["primary_blocker"] payload["next_safe_action"] = sop_progress["next_safe_action"] payload["reboot_sop_progress"] = sop_progress + payload["active_blocker_action_matrix"] = active_blocker_action_matrix payload["readback"] = readback payload["rollups"] = rollups payload["summary"] = summary diff --git a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py index f2cf6e045..5587e970c 100644 --- a/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py +++ b/scripts/reboot-recovery/tests/test_reboot_auto_recovery_slo_scorecard.py @@ -388,6 +388,8 @@ def test_green_summary_and_recent_all_host_probe_can_claim_slo(tmp_path: Path) - ] assert payload["windows99_vmware_autostart"]["verify_ready"] is True assert payload["windows99_verify_collection"]["collection_blockers"] == [] + assert payload["active_blocker_action_matrix"]["item_count"] == 0 + assert payload["active_blocker_action_matrix"]["items"] == [] assert ( payload["windows99_verify_collection"]["post_verifier"] == "rerun_reboot_auto_recovery_slo_scorecard_with_windows99_vmware_file_no_secret_no_reboot" @@ -488,6 +490,28 @@ def test_windows99_management_channel_unavailable_is_visible(tmp_path: Path) -> "windows99_remote_execution_channel_unavailable", "windows99_vmware_autostart_readback_missing", ] + matrix = payload["active_blocker_action_matrix"] + assert matrix["schema_version"] == "reboot_active_blocker_action_matrix_v1" + assert matrix["item_count"] == 2 + assert matrix["telegram_alert_required_count"] == 2 + assert matrix["category_counts"] == {"windows99_vmware_autostart": 2} + assert matrix["primary_blocker_action"]["blocker"] == ( + "windows99_remote_execution_channel_unavailable" + ) + assert matrix["primary_blocker_action"]["owner_lane"] == ( + "windows99_console_or_no_secret_management_channel" + ) + assert matrix["items"][0]["telegram_alert"] == ( + "RebootAutoRecoveryActiveBlocker" + ) + assert matrix["items"][0]["controlled_apply_authorized_by_scorecard"] is False + assert "windows_password_or_secret_collection" in matrix["items"][0][ + "forbidden_actions" + ] + assert payload["readback"]["active_blocker_action_count"] == 2 + assert payload["readback"]["primary_blocker_owner_lane"] == ( + "windows99_console_or_no_secret_management_channel" + ) assert payload["safe_next_step"] == ( "restore_windows99_no_secret_management_channel_or_collect_local_console_" "verify_readback_then_rerun_reboot_scorecard_no_reboot"