diff --git a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py index de2ae3a24..c146cf2f2 100644 --- a/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py +++ b/apps/api/src/services/reboot_auto_recovery_slo_scorecard.py @@ -51,6 +51,21 @@ _PROMETHEUS_RUNTIME_QUERIES = { ), "windows99_missing_vmx_alias": "awoooi_windows99_vmware_missing_vmx_alias", "windows99_powered_off_alias": "awoooi_windows99_vmware_powered_off_alias", + "windows99_no_secret_collector_readback_present": ( + "awoooi_windows99_no_secret_collector_readback_present" + ), + "windows99_no_secret_collector_status": ( + "awoooi_windows99_no_secret_collector_status" + ), + "windows99_no_secret_collector_ssh_batchmode_auth_ready": ( + "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready" + ), + "windows99_no_secret_collector_port_22_open": ( + "awoooi_windows99_no_secret_collector_port_22_open" + ), + "windows99_no_secret_collector_remote_verify_attempted": ( + "awoooi_windows99_no_secret_collector_remote_verify_attempted" + ), } _PROMETHEUS_RUNTIME_COMBINED_QUERY = ( '{__name__=~"' @@ -81,6 +96,13 @@ _PROMETHEUS_SOURCE_CONTROLLED_BLOCKERS = { "conversation_event_hot_path_index_migration_source_present" ), } +_WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS = { + "blocked_ssh_publickey_auth_missing": "windows99_ssh_publickey_auth_missing", + "blocked_ssh_port_closed": "windows99_ssh_port_closed", + "blocked_local_verify_script_missing": "windows99_local_verify_script_missing", + "blocked_remote_verify_output_invalid": "windows99_remote_verify_output_invalid", + "blocked_remote_verify_command_failed": "windows99_remote_verify_command_failed", +} def load_latest_reboot_auto_recovery_slo_scorecard( @@ -288,6 +310,7 @@ def _load_reboot_slo_prometheus_metric_readback( "primary_blocker", "windows99_missing_vmx_alias", "windows99_powered_off_alias", + "windows99_no_secret_collector_status", } } missing_vmx_results = results_by_name.get( @@ -353,6 +376,18 @@ def _load_reboot_slo_prometheus_metric_readback( powered_off_results, "alias", ) + collector_status_results = results_by_name.get( + _PROMETHEUS_RUNTIME_QUERIES["windows99_no_secret_collector_status"], + [], + ) + windows99_collector_status = _prometheus_first_label_value( + collector_status_results, + "status", + ) + windows99_collector_safe_next_step = _prometheus_first_label_value( + collector_status_results, + "safe_next_step", + ) return { "schema_version": "reboot_slo_prometheus_metric_readback_v1", "readback_present": bool(combined_results), @@ -368,6 +403,48 @@ def _load_reboot_slo_prometheus_metric_readback( "windows99_windows_update_no_auto_reboot_ready": windows99_update_ready, "windows99_missing_vmx_aliases": windows99_missing_vmx_aliases, "windows99_powered_off_aliases": windows99_powered_off_aliases, + "windows99_no_secret_collector_readback_present": ( + _prometheus_first_int( + scalar_results.get( + "windows99_no_secret_collector_readback_present" + ) + or [], + default=0, + ) + == 1 + ), + "windows99_no_secret_collector_status": windows99_collector_status, + "windows99_no_secret_collector_safe_next_step": ( + windows99_collector_safe_next_step + ), + "windows99_no_secret_collector_ssh_batchmode_auth_ready": ( + _prometheus_first_int( + scalar_results.get( + "windows99_no_secret_collector_ssh_batchmode_auth_ready" + ) + or [], + default=0, + ) + == 1 + ), + "windows99_no_secret_collector_port_22_open": ( + _prometheus_first_int( + scalar_results.get("windows99_no_secret_collector_port_22_open") + or [], + default=0, + ) + == 1 + ), + "windows99_no_secret_collector_remote_verify_attempted": ( + _prometheus_first_int( + scalar_results.get( + "windows99_no_secret_collector_remote_verify_attempted" + ) + or [], + default=0, + ) + == 1 + ), } @@ -411,6 +488,20 @@ def _prometheus_label_values( ) +def _prometheus_first_label_value( + results: list[dict[str, Any]], + label_name: str, +) -> str: + for result in results: + value = _prometheus_first_int([result], default=0) + if value != 1: + continue + label = str(_dict(result.get("metric")).get(label_name) or "") + if label: + return label + return "" + + def _annotate_prometheus_metric_readback( payload: dict[str, Any], metric_readback: dict[str, Any], @@ -459,9 +550,114 @@ def _annotate_prometheus_metric_readback( rollups["runtime_metric_source_control_reconciled_blocker_count"] = 0 rollups["runtime_metric_runtime_readback_reconciled_blocker_count"] = 0 rollups["runtime_metric_runtime_readback_added_blocker_count"] = 0 + _apply_prometheus_windows99_no_secret_collector_readback( + payload, + metric_readback, + ) _apply_prometheus_windows99_vmware_readback(payload, metric_readback) +def _apply_prometheus_windows99_no_secret_collector_readback( + payload: dict[str, Any], + metric_readback: dict[str, Any], +) -> None: + if metric_readback.get("readback_present") is not True: + return + if ( + metric_readback.get("windows99_no_secret_collector_readback_present") + is not True + ): + return + + status = str( + metric_readback.get("windows99_no_secret_collector_status") or "unknown" + ) + safe_next_step = str( + metric_readback.get("windows99_no_secret_collector_safe_next_step") or "" + ) + ssh_ready = ( + metric_readback.get( + "windows99_no_secret_collector_ssh_batchmode_auth_ready" + ) + is True + ) + port_22_open = ( + metric_readback.get("windows99_no_secret_collector_port_22_open") is True + ) + remote_attempted = ( + metric_readback.get( + "windows99_no_secret_collector_remote_verify_attempted" + ) + is True + ) + blocker = _WINDOWS99_COLLECTOR_BLOCKERS_BY_STATUS.get(status) + if not blocker and status.startswith("blocked_"): + blocker = "windows99_no_secret_collector_not_ready" + + collection = _dict(payload.setdefault("windows99_verify_collection", {})) + available_channels = _strings(collection.get("available_collection_channels")) + if ssh_ready: + available_channels.append("no_secret_ssh_batchmode_verify_collector") + collector_blockers = _strings(collection.get("collection_blockers")) + if blocker: + collector_blockers.append(blocker) + collector = _dict(collection.get("no_secret_collector")) + collector.update( + { + "schema_version": "windows99_vmware_verify_collector_v1", + "readback_present": True, + "status": status, + "safe_next_step": safe_next_step, + "ssh_batchmode_auth_ready": ssh_ready, + "port_22_open": port_22_open, + "remote_verify_attempted": remote_attempted, + "source": "prometheus_textfile_metric", + } + ) + collection.update( + { + "no_secret_collector_readback_present": True, + "no_secret_collector_status": status, + "no_secret_collector_safe_next_step": safe_next_step, + "no_secret_collector_ssh_batchmode_auth_ready": ssh_ready, + "no_secret_collector_port_22_open": port_22_open, + "no_secret_collector_remote_verify_attempted": remote_attempted, + "no_secret_collector": collector, + "available_collection_channels": _unique_strings(available_channels), + "collection_blockers": _unique_strings(collector_blockers), + } + ) + + readback = _dict(payload.setdefault("readback", {})) + readback["windows99_no_secret_collector_readback_present"] = True + readback["windows99_no_secret_collector_status"] = status + readback["windows99_no_secret_collector_ssh_batchmode_auth_ready"] = ssh_ready + readback["windows99_no_secret_collector_port_22_open"] = port_22_open + readback["windows99_no_secret_collector_remote_verify_attempted"] = ( + remote_attempted + ) + readback["windows99_no_secret_collector_safe_next_step"] = safe_next_step + readback["windows99_available_collection_channels"] = collection[ + "available_collection_channels" + ] + + rollups = _dict(payload.setdefault("rollups", {})) + rollups["windows99_no_secret_collector_readback_present"] = True + rollups["windows99_no_secret_collector_status"] = status + rollups["windows99_no_secret_collector_ssh_batchmode_auth_ready"] = ssh_ready + rollups["windows99_no_secret_collector_port_22_open"] = port_22_open + rollups["windows99_no_secret_collector_remote_verify_attempted"] = ( + remote_attempted + ) + rollups["windows99_no_secret_collector_safe_next_step"] = safe_next_step + rollups["windows99_available_collection_channels"] = collection[ + "available_collection_channels" + ] + rollups["windows99_verify_collection_blocker_count"] = len( + _strings(collection.get("collection_blockers")) + ) + + def _apply_prometheus_windows99_vmware_readback( payload: dict[str, Any], metric_readback: dict[str, Any], diff --git a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py index 7b070007c..974d8266e 100644 --- a/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py +++ b/apps/api/tests/test_reboot_auto_recovery_slo_scorecard_api.py @@ -684,6 +684,49 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): } for alias in ["111", "112", "120", "121", "188"] ], + { + "metric": { + "__name__": ( + "awoooi_windows99_no_secret_collector_readback_present" + ), + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": "awoooi_windows99_no_secret_collector_status", + "status": "blocked_ssh_publickey_auth_missing", + "safe_next_step": ( + "select_existing_authorized_public_key_user_or_set_" + "WINDOWS99_SSH_USERS_then_rerun_collector_no_password" + ), + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": ( + "awoooi_windows99_no_secret_collector_" + "ssh_batchmode_auth_ready" + ), + }, + "value": [1783010479, "0"], + }, + { + "metric": { + "__name__": "awoooi_windows99_no_secret_collector_port_22_open", + }, + "value": [1783010479, "1"], + }, + { + "metric": { + "__name__": ( + "awoooi_windows99_no_secret_collector_" + "remote_verify_attempted" + ), + }, + "value": [1783010479, "0"], + }, ] monkeypatch.setattr( @@ -715,6 +758,67 @@ def test_reboot_slo_prometheus_readback_uses_single_combined_query(monkeypatch): "121", "188", ] + assert payload["windows99_no_secret_collector_readback_present"] is True + assert ( + payload["windows99_no_secret_collector_status"] + == "blocked_ssh_publickey_auth_missing" + ) + assert payload["windows99_no_secret_collector_port_22_open"] is True + assert ( + payload["windows99_no_secret_collector_ssh_batchmode_auth_ready"] is False + ) + assert ( + payload["windows99_no_secret_collector_remote_verify_attempted"] is False + ) + + +def test_reboot_auto_recovery_slo_scorecard_overlays_prometheus_collector_blocker(): + metric_readback = dict(PROMETHEUS_RUNTIME_READBACK) + metric_readback.update( + { + "windows99_vmware_readback_present": False, + "windows99_no_secret_collector_readback_present": True, + "windows99_no_secret_collector_status": ( + "blocked_ssh_publickey_auth_missing" + ), + "windows99_no_secret_collector_safe_next_step": ( + "select_existing_authorized_public_key_user_or_set_" + "WINDOWS99_SSH_USERS_then_rerun_collector_no_password" + ), + "windows99_no_secret_collector_ssh_batchmode_auth_ready": False, + "windows99_no_secret_collector_port_22_open": True, + "windows99_no_secret_collector_remote_verify_attempted": False, + } + ) + + payload = load_latest_reboot_auto_recovery_slo_scorecard( + prometheus_metric_readback=metric_readback + ) + + collection = payload["windows99_verify_collection"] + assert collection["no_secret_collector_readback_present"] is True + assert ( + collection["no_secret_collector_status"] + == "blocked_ssh_publickey_auth_missing" + ) + assert collection["no_secret_collector_port_22_open"] is True + assert collection["no_secret_collector_ssh_batchmode_auth_ready"] is False + assert collection["no_secret_collector_remote_verify_attempted"] is False + assert ( + collection["no_secret_collector"]["source"] + == "prometheus_textfile_metric" + ) + assert "windows99_ssh_publickey_auth_missing" in collection[ + "collection_blockers" + ] + assert payload["readback"]["windows99_no_secret_collector_status"] == ( + "blocked_ssh_publickey_auth_missing" + ) + assert payload["readback"]["windows99_no_secret_collector_port_22_open"] is True + assert payload["rollups"]["windows99_no_secret_collector_status"] == ( + "blocked_ssh_publickey_auth_missing" + ) + assert payload["rollups"]["windows99_no_secret_collector_readback_present"] is True def test_reboot_auto_recovery_slo_scorecard_endpoint_returns_readback(monkeypatch): diff --git a/docs/LOGBOOK.md b/docs/LOGBOOK.md index 716f57117..1190af384 100644 --- a/docs/LOGBOOK.md +++ b/docs/LOGBOOK.md @@ -1,3 +1,13 @@ +## 2026-07-03 — 08:23 Windows99 no-secret collector Prometheus overlay + +**完成內容**： +- `reboot-auto-recovery-slo-exporter.sh` 會把 `windows99_vmware_verify_collector` 的 no-secret collector 狀態投影成 Prometheus textfile metrics：readback present、status label、SSH BatchMode auth、port 22 與 remote verify attempted。 +- `/api/v1/agents/reboot-auto-recovery-slo-scorecard` 在 runtime scorecard 檔案未掛載到 API container 時，也能從 Prometheus overlay 讀回 `blocked_ssh_publickey_auth_missing`、safe next step 與 collector booleans，並同步到 `windows99_verify_collection`、readback 與 rollups。 +- 此 overlay 只補 collector evidence，不會把 VMX / guest power / VMware Verify ready 改成 green，也不會清除既有 Windows99 autostart blockers。 + +**仍維持**： +- 未讀 secret / token / `.env` / raw sessions / SQLite / auth；未使用 GitHub / gh；未 workflow_dispatch；未重啟 host / VM / service；未 Docker / Nginx / K3s / DB / firewall restart；未 DROP / TRUNCATE / restore / prune / delete / force push。 + ## 2026-07-03 — 07:55 Drift fingerprint readback degraded fallback **完成內容**： diff --git a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh index 1875a2680..7448aa5ed 100755 --- a/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh +++ b/scripts/reboot-recovery/reboot-auto-recovery-slo-exporter.sh @@ -363,6 +363,7 @@ import json, sys payload = json.load(open(sys.argv[1], encoding="utf-8")) windows99 = payload.get("windows99_vmware_autostart") or {} +collector = payload.get("windows99_vmware_verify_collector") or {} def label(value: object) -> str: @@ -402,6 +403,28 @@ for alias in windows99.get("powered_off_aliases") or []: "awoooi_windows99_vmware_powered_off_alias" f'{{scope="windows99",alias="{label(alias)}"}} 1' ) +collector_present = collector.get("readback_present") is True +gauge("awoooi_windows99_no_secret_collector_readback_present", collector_present) +gauge( + "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready", + collector.get("ssh_batchmode_auth_ready") is True, +) +gauge( + "awoooi_windows99_no_secret_collector_port_22_open", + collector.get("port_22_open") is True, +) +gauge( + "awoooi_windows99_no_secret_collector_remote_verify_attempted", + collector.get("remote_verify_attempted") is True, +) +collector_status = str(collector.get("status") or "unknown") +collector_safe_next_step = str(collector.get("safe_next_step") or "") +if collector_present: + print( + "awoooi_windows99_no_secret_collector_status" + f'{{scope="windows99",status="{label(collector_status)}",' + f'safe_next_step="{label(collector_safe_next_step)}"}} 1' + ) PY )" active_blocker_metrics="$(python3 - "$scorecard_file" "$TARGET_MINUTES" <<'PY' @@ -497,6 +520,16 @@ awoooi_public_route_unreachable_without_external_l1_fallback{scope="public_route # TYPE awoooi_windows99_vmware_missing_vmx_alias gauge # HELP awoooi_windows99_vmware_powered_off_alias Required Windows99 VMware guests observed powered off. # TYPE awoooi_windows99_vmware_powered_off_alias gauge +# HELP awoooi_windows99_no_secret_collector_readback_present Whether the no-secret Windows99 collector produced readback. +# TYPE awoooi_windows99_no_secret_collector_readback_present gauge +# HELP awoooi_windows99_no_secret_collector_status Current no-secret Windows99 collector status as labels. +# TYPE awoooi_windows99_no_secret_collector_status gauge +# HELP awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready Whether no-secret SSH BatchMode publickey auth is ready. +# TYPE awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready gauge +# HELP awoooi_windows99_no_secret_collector_port_22_open Whether Windows99 SSH port 22 is reachable. +# TYPE awoooi_windows99_no_secret_collector_port_22_open gauge +# HELP awoooi_windows99_no_secret_collector_remote_verify_attempted Whether the remote no-secret verify command was attempted. +# TYPE awoooi_windows99_no_secret_collector_remote_verify_attempted gauge METRICS if [ -n "$windows99_vmware_metrics" ]; then printf '%s\n' "$windows99_vmware_metrics" >>"$tmp_metric" diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index f9300dcdd..f46dda528 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -124,6 +124,13 @@ def test_reboot_p0_contract_has_maintenance_and_telegram_alerts() -> None: assert "--public-maintenance-file" in exporter assert "awoooi_public_route_raw_5xx_without_maintenance_fallback" in exporter assert "awoooi_reboot_auto_recovery_slo_active_blocker" in exporter + assert "awoooi_windows99_no_secret_collector_readback_present" in exporter + assert "awoooi_windows99_no_secret_collector_status" in exporter + assert ( + "awoooi_windows99_no_secret_collector_ssh_batchmode_auth_ready" in exporter + ) + assert "awoooi_windows99_no_secret_collector_port_22_open" in exporter + assert "awoooi_windows99_no_secret_collector_remote_verify_attempted" in exporter assert 'blocker="{label(blocker)}"' in exporter assert "$labels.blocker" in alerts