fix(reboot): expose windows99 management channel readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 1s
CD Pipeline / cancel-stale-cd (push) Has been skipped
CD Pipeline / tests (push) Failing after 41s
CD Pipeline / build-and-deploy (push) Has been skipped
CD Pipeline / post-deploy-checks (push) Has been skipped

This commit is contained in:
Your Name
2026-07-02 15:19:09 +08:00
parent 25148d0752
commit aaa617f00f
14 changed files with 983 additions and 195 deletions

View File

@@ -2,27 +2,25 @@
"active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
"backup_core_green_not_1",
"host_188_service_green_not_1",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"local_disk_free_below_minimum",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"reboot_event_required_host_unreachable",
"service_green_not_1",
"wazuh_dashboard_degraded",
"windows99_remote_execution_channel_unavailable",
"windows99_vmware_autostart_readback_missing"
],
"can_claim_all_services_recovered_within_target": false,
"capacity": {
"checked": true,
"free_gib": 0.751,
"free_gib": 4.021,
"min_free_gib": 2.0
},
"current_phase": "host_boot_detection_blocked",
"eta_or_wait_reason": "target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe",
"generated_at": "2026-06-30T20:44:37+08:00",
"generated_at": "2026-07-02T15:08:44+08:00",
"host_boot_detection": {
"host_rows": [
{
@@ -38,14 +36,14 @@
},
{
"alias": "110",
"boot_id": "node_exporter_1782811060",
"boot_id": "2b07b8f1-d818-48be-a4bc-a05aedcca658",
"reachable": true,
"startup_active": "unknown",
"startup_enabled": "unknown",
"startup_active": "inactive",
"startup_enabled": "enabled",
"startup_unit": "awoooi-startup-110.service",
"systemd_state": "node_exporter",
"systemd_state": "running",
"target": "wooo@192.168.0.110",
"uptime_seconds": 12406
"uptime_seconds": 86200
},
{
"alias": "111",
@@ -67,7 +65,7 @@
"startup_unit": "vm-host-boot",
"systemd_state": "node_exporter",
"target": "192.168.0.112",
"uptime_seconds": 12484
"uptime_seconds": 165145
},
{
"alias": "120",
@@ -78,7 +76,7 @@
"startup_unit": "k3s.service",
"systemd_state": "running",
"target": "wooo@192.168.0.120",
"uptime_seconds": 12479
"uptime_seconds": 165139
},
{
"alias": "121",
@@ -89,7 +87,7 @@
"startup_unit": "k3s.service",
"systemd_state": "running",
"target": "wooo@192.168.0.121",
"uptime_seconds": 12480
"uptime_seconds": 165140
},
{
"alias": "188",
@@ -100,10 +98,10 @@
"startup_unit": "awoooi-startup.service",
"systemd_state": "degraded",
"target": "ollama@192.168.0.188",
"uptime_seconds": 12420
"uptime_seconds": 165079
}
],
"max_observed_uptime_seconds": 12484,
"max_observed_uptime_seconds": 165145,
"missing_hosts": [],
"observed_hosts": [
"110",
@@ -162,67 +160,70 @@
"rerun_host_pressure_and_cold_start_scorecard_after_apply"
]
},
"next_safe_action": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"next_safe_action": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"post_reboot_readiness": {
"backup_core_green": false,
"host_188_service_green": false,
"next_required_gates": "product_data_freshness_recovery,backup_core_readback_recovery,credential_escrow_evidence,host_188_hygiene_maintenance_window",
"host_188_service_green": true,
"next_required_gates": "backup_core_readback_recovery,credential_escrow_evidence,host_188_hygiene_maintenance_window",
"overall_declaration": "SERVICE_BLOCKED",
"post_start_blocked": 8,
"post_start_blocked": 1,
"post_start_result": "BLOCKED",
"product_data_green": false,
"product_data_green": true,
"service_green": false,
"summary_present": true,
"wazuh_dashboard_degraded": true
},
"primary_blocker": "reboot_event_required_host_unreachable",
"readback": {
"active_blocker_count": 13,
"active_blocker_count": 11,
"active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
"backup_core_green_not_1",
"host_188_service_green_not_1",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"local_disk_free_below_minimum",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"reboot_event_required_host_unreachable",
"service_green_not_1",
"wazuh_dashboard_degraded",
"windows99_remote_execution_channel_unavailable",
"windows99_vmware_autostart_readback_missing"
],
"blocked_by_fresh_reboot_window_only": false,
"current_phase": "host_boot_detection_blocked",
"eta_or_wait_reason": "target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe",
"host_reboot_authorized_by_this_scorecard": false,
"next_safe_action": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"next_safe_action": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"primary_blocker": "reboot_event_required_host_unreachable",
"readiness_percent": 15,
"readiness_percent": 43,
"required_checks": {
"backup_core_green": false,
"can_claim_slo": false,
"fresh_reboot_window_observed": false,
"host_188_service_green": false,
"product_data_green": false,
"host_188_service_green": true,
"product_data_green": true,
"required_hosts_observed": true,
"required_hosts_reachable": false,
"service_green": false,
"source_controls_present": true,
"stockplatform_freshness_ok": false,
"stockplatform_ingestion_ok": false,
"stockplatform_freshness_ok": true,
"stockplatform_ingestion_ok": true,
"windows99_management_channel_ready": false,
"windows99_vmware_autostart_ready": false,
"windows_update_no_auto_reboot_ready": false
},
"runtime_write_authorized_by_this_scorecard": false,
"safe_next_step": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"safe_next_step": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"secret_value_collection_allowed": false,
"source_controls_present": true,
"source_id": "reboot_auto_recovery_slo_scorecard",
"status": "blocked_reboot_auto_recovery_slo_not_ready",
"target_minutes": 10,
"windows99_remote_execution_channel_ready": false,
"windows99_ssh_batch_status": "permission_denied",
"windows99_update_no_auto_reboot_ready": false,
"windows99_verify_collection_can_collect_no_secret": true,
"windows99_verify_collection_status": "blocked_windows99_verify_output_missing_host_reachable",
"windows99_vmware_verify_ready": false,
"workflow_trigger_authorized_by_this_scorecard": false,
"workplan_id": "P0-006",
@@ -241,27 +242,25 @@
"rebooted_hosts": [
"99"
],
"state_file": "/tmp/awoooi-reboot-slo-live-20260630-2045/reboot-event-state.json",
"state_written": false,
"state_file": "/tmp/awoooi-reboot-slo-20260702-150844/reboot-event-state.json",
"state_written": true,
"unreachable_hosts": [
"111"
]
},
"reboot_sop_progress": {
"active_blocker_count": 13,
"active_blocker_count": 11,
"active_blockers": [
"all_required_hosts_not_in_10_minute_reboot_window",
"backup_core_green_not_1",
"host_188_service_green_not_1",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"local_disk_free_below_minimum",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"reboot_event_required_host_unreachable",
"service_green_not_1",
"wazuh_dashboard_degraded",
"windows99_remote_execution_channel_unavailable",
"windows99_vmware_autostart_readback_missing"
],
"current_phase": "host_boot_detection_blocked",
@@ -275,36 +274,37 @@
"backup_health_and_offsite_evidence",
"telegram_alert_delivery_readback"
],
"next_safe_action": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"next_safe_action": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"primary_blocker": "reboot_event_required_host_unreachable",
"readiness_percent": 15
"readiness_percent": 43
},
"required_checks": {
"backup_core_green": false,
"can_claim_slo": false,
"fresh_reboot_window_observed": false,
"host_188_service_green": false,
"product_data_green": false,
"host_188_service_green": true,
"product_data_green": true,
"required_hosts_observed": true,
"required_hosts_reachable": false,
"service_green": false,
"source_controls_present": true,
"stockplatform_freshness_ok": false,
"stockplatform_ingestion_ok": false,
"stockplatform_freshness_ok": true,
"stockplatform_ingestion_ok": true,
"windows99_management_channel_ready": false,
"windows99_vmware_autostart_ready": false,
"windows_update_no_auto_reboot_ready": false
},
"rollups": {
"active_blocker_count": 13,
"active_blocker_count": 11,
"backup_core_green": false,
"blocked_by_fresh_reboot_window_only": false,
"can_claim_all_services_recovered_within_target": false,
"capacity_below_minimum": true,
"capacity_below_minimum": false,
"capacity_checked": true,
"capacity_free_gib": 0.751,
"capacity_free_gib": 4.021,
"capacity_min_free_gib": 2.0,
"completed_check_count": 2,
"host_188_service_green": false,
"completed_check_count": 6,
"host_188_service_green": true,
"latest_verify_only_metric_blocker_count": 0,
"latest_verify_only_metric_last_run_timestamp": 0,
"latest_verify_only_metric_max_host_uptime_seconds": 0,
@@ -312,31 +312,45 @@
"latest_verify_only_metric_ready": 0,
"missing_host_count": 0,
"observed_host_count": 7,
"post_start_blocked": 8,
"product_data_green": false,
"readiness_percent": 15,
"required_check_count": 13,
"post_start_blocked": 1,
"product_data_green": true,
"readiness_percent": 43,
"required_check_count": 14,
"service_green": false,
"source_control_count": 15,
"source_control_ready_count": 15,
"source_control_count": 16,
"source_control_ready_count": 16,
"source_controls_present": true,
"stale_host_count": 5,
"stockplatform_freshness_blocker_count": 1,
"stockplatform_freshness_status": "not_configured",
"stockplatform_ingestion_blocker_count": 1,
"stockplatform_ingestion_status": "not_configured",
"stockplatform_freshness_blocker_count": 0,
"stockplatform_freshness_status": "ok",
"stockplatform_ingestion_blocker_count": 0,
"stockplatform_ingestion_status": "ok",
"unknown_uptime_host_count": 1,
"unreachable_host_count": 1,
"windows99_can_collect_vmware_verify_without_secret": false,
"windows99_host99_reachable": true,
"windows99_host99_uptime_known": false,
"windows99_host_reachable": true,
"windows99_management_readback_present": true,
"windows99_rdp_console_reachable": true,
"windows99_remote_execution_channel_ready": false,
"windows99_ssh_batch_ready": false,
"windows99_ssh_batch_status": "permission_denied",
"windows99_update_no_auto_reboot_ready": false,
"windows99_verify_collection_blocker_count": 2,
"windows99_verify_collection_can_collect_no_secret": true,
"windows99_verify_collection_status": "blocked_windows99_verify_output_missing_host_reachable",
"windows99_vmrun_present": false,
"windows99_vmware_config_ready": false,
"windows99_vmware_missing_vmx_count": 5,
"windows99_vmware_power_ready": false,
"windows99_vmware_powered_off_count": 5,
"windows99_vmware_readback_present": false,
"windows99_vmware_verify_ready": false
"windows99_vmware_verify_ready": false,
"windows99_winrm_http_open": false,
"windows99_winrm_https_open": false
},
"safe_next_step": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"safe_next_step": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"schema_version": "awoooi_reboot_auto_recovery_slo_scorecard_v1",
"sla_recovery_eta": {
"deadline_status": "target_window_elapsed",
@@ -369,23 +383,16 @@
"slo_systemd_service_source_present": true,
"slo_systemd_timer_source_present": true,
"telegram_reboot_backup_alert_rules_source_present": true,
"windows99_management_channel_probe_source_present": true,
"windows_99_vmware_autostart_source_present": true
},
"status": "blocked_reboot_auto_recovery_slo_not_ready",
"stockplatform_data_freshness": {
"ai_recommendations_recovery": {
"cleared_blocker": "ai_recommendations_stale",
"status": "not_verified"
"status": "recovered"
},
"blocked_sources": [
{
"latest_date": null,
"notes": "PostgreSQL 結構尚未就緒OperationalError",
"row_count": 0,
"source": "postgres",
"status": "not_configured"
}
],
"blocked_sources": [],
"controlled_recovery_gate": {
"allowed_actions": [
"inspect_existing_ingestion_readback",
@@ -401,29 +408,106 @@
"reboot_or_service_restart_from_reboot_slo_lane"
],
"required": false,
"status": "not_required_yet",
"status": "not_required_freshness_recovered",
"target_selector": "stockplatform-v2:system_freshness:core.margin_short_daily,ai.recommendations"
},
"eod_window": {
"classification": "unknown",
"classification": "ok",
"final_retry_window_end_local": "unknown",
"final_retry_window_passed": false,
"first_full_window_end_local": "unknown",
"next_action": "unknown",
"next_action": "none",
"pending": false
},
"freshness_blockers": [
"postgres_not_ready"
],
"freshness_blockers": [],
"freshness_endpoint_readback_present": true,
"freshness_status": "not_configured",
"ingestion_blockers": [
"postgres_not_ready"
],
"freshness_status": "ok",
"ingestion_blockers": [],
"ingestion_endpoint_readback_present": true,
"ingestion_status": "not_configured",
"latest_source_runs": [],
"latest_trading_date": "None",
"ingestion_status": "ok",
"latest_source_runs": [
{
"finished_at": "2026-07-02T07:00:09.686543Z",
"source_name": "intelligence_security_linker",
"source_run_id": 3508,
"started_at": "2026-07-02T07:00:09.686543Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T07:00:07.375654Z",
"source_name": "intelligence_reports_import",
"source_run_id": 3507,
"started_at": "2026-07-02T07:00:07.375654Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T05:00:09.044775Z",
"source_name": "intelligence_security_linker",
"source_run_id": 3506,
"started_at": "2026-07-02T05:00:09.044775Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T05:00:07.871117Z",
"source_name": "intelligence_reports_import",
"source_run_id": 3505,
"started_at": "2026-07-02T05:00:07.871117Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T03:00:09.959576Z",
"source_name": "intelligence_security_linker",
"source_run_id": 3504,
"started_at": "2026-07-02T03:00:09.959576Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T03:00:07.617332Z",
"source_name": "intelligence_reports_import",
"source_run_id": 3503,
"started_at": "2026-07-02T03:00:07.617332Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T01:00:13.407344Z",
"source_name": "intelligence_security_linker",
"source_run_id": 3502,
"started_at": "2026-07-02T01:00:13.407344Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-02T01:00:11.723462Z",
"source_name": "intelligence_reports_import",
"source_run_id": 3501,
"started_at": "2026-07-02T01:00:11.723462Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-01T23:00:10.305951Z",
"source_name": "intelligence_security_linker",
"source_run_id": 3500,
"started_at": "2026-07-01T23:00:10.305951Z",
"status": "succeeded",
"target_date": null
},
{
"finished_at": "2026-07-01T23:00:09.148428Z",
"source_name": "intelligence_reports_import",
"source_run_id": 3499,
"started_at": "2026-07-01T23:00:09.148428Z",
"status": "succeeded",
"target_date": null
}
],
"latest_trading_date": "2026-07-01",
"margin_short_recovery": {
"cleared_blocker": "core_margin_short_daily_missing",
"cleared_ingestion_blocker": "core.margin_short_daily_incomplete",
@@ -433,25 +517,27 @@
},
"summary": {
"github_api_used": false,
"reboot_auto_recovery_active_blocker_count": 13,
"reboot_auto_recovery_active_blocker_count": 11,
"reboot_auto_recovery_backup_core_green": false,
"reboot_auto_recovery_can_claim_slo": false,
"reboot_auto_recovery_current_phase": "host_boot_detection_blocked",
"reboot_auto_recovery_eta_or_wait_reason": "target_window_elapsed_eta_unavailable_until_fresh_all_host_reboot_event_and_probe",
"reboot_auto_recovery_host_188_service_green": false,
"reboot_auto_recovery_next_safe_action": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"reboot_auto_recovery_host_188_service_green": true,
"reboot_auto_recovery_next_safe_action": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"reboot_auto_recovery_observed_host_count": 7,
"reboot_auto_recovery_primary_blocker": "reboot_event_required_host_unreachable",
"reboot_auto_recovery_product_data_green": false,
"reboot_auto_recovery_readiness_percent": 15,
"reboot_auto_recovery_safe_next_step": "collect_windows99_vmware_autostart_verify_readback_then_rerun_all_host_reboot_scorecard_no_secret_no_reboot",
"reboot_auto_recovery_product_data_green": true,
"reboot_auto_recovery_readiness_percent": 43,
"reboot_auto_recovery_safe_next_step": "restore_windows99_no_secret_management_channel_or_collect_local_console_verify_readback_then_rerun_reboot_scorecard_no_reboot",
"reboot_auto_recovery_service_green": false,
"reboot_auto_recovery_source_controls_present": true,
"reboot_auto_recovery_stale_host_count": 5,
"reboot_auto_recovery_status": "blocked_reboot_auto_recovery_slo_not_ready",
"reboot_auto_recovery_stockplatform_freshness_status": "not_configured",
"reboot_auto_recovery_stockplatform_ingestion_status": "not_configured",
"reboot_auto_recovery_stockplatform_freshness_status": "ok",
"reboot_auto_recovery_stockplatform_ingestion_status": "ok",
"reboot_auto_recovery_windows99_update_no_auto_reboot_ready": false,
"reboot_auto_recovery_windows99_verify_collection_can_collect_no_secret": true,
"reboot_auto_recovery_windows99_verify_collection_status": "blocked_windows99_verify_output_missing_host_reachable",
"reboot_auto_recovery_windows99_vmware_verify_ready": false,
"reboot_auto_recovery_workplan_id": "P0-006",
"runtime_write_authorized": false,
@@ -460,6 +546,95 @@
},
"target_minutes": 10,
"target_seconds": 600,
"windows99_management_channel": {
"blockers": [
"windows99_remote_execution_channel_unavailable",
"windows99_winrm_unavailable",
"windows99_ssh_batch_denied"
],
"can_collect_vmware_verify_without_secret": false,
"forbidden_actions": [
"read_windows_password",
"read_secret_value",
"start_vm",
"reboot_host",
"restart_service",
"write_windows_policy"
],
"generated_at": "2026-07-02T15:08:44+08:00",
"host": "192.168.0.99",
"host_reachable": true,
"rdp_console_reachable": true,
"readback_present": true,
"remote_execution_channel_ready": false,
"schema_version": "windows99_management_channel_readback_v1",
"ssh_batch": {
"checked": true,
"ready": false,
"status": "permission_denied"
},
"ssh_user": "administrator",
"tcp_ports": {
"135": "open",
"22": "open",
"3389": "open",
"445": "open",
"5985": "timeout",
"5986": "timeout"
},
"winrm_http_open": false,
"winrm_https_open": false
},
"windows99_verify_collection": {
"can_collect_no_secret_verify": true,
"collection_blockers": [
"windows99_vmware_autostart_readback_missing",
"windows99_uptime_unknown"
],
"expected_no_secret_output_fields": [
"VMRUN_PRESENT",
"VMX alias=<alias> present=<0|1>",
"VMWARE_SERVICE name=<name> ok=<0|1>",
"VMWARE_AUTOSTART_TASK name=AWOOOI-Start-VMware-VMs ok=<0|1>",
"WINDOWS_UPDATE_POLICY name=<name> ok=<0|1>",
"VM_POWER alias=<alias> running=<0|1>",
"VMWARE_AUTOSTART_CONFIG_READY",
"VMWARE_AUTOSTART_POWER_READY",
"WINDOWS_UPDATE_NO_AUTO_REBOOT_READY",
"VMWARE_AUTOSTART_VERIFY_READY"
],
"forbidden_actions": [
"windows_password_or_secret_collection",
"host_reboot",
"vm_power_change",
"windows_update_policy_apply",
"manual_registry_edit",
"service_restart",
"github_api"
],
"host99_reachable": true,
"host99_uptime_known": false,
"no_secret_verify_command": "powershell -ExecutionPolicy Bypass -File .\\windows99-vmware-autostart.ps1 -Mode Verify",
"post_verifier": "rerun_reboot_auto_recovery_slo_scorecard_with_windows99_vmware_file_no_secret_no_reboot",
"readback_present": false,
"required_vm_aliases": [
"111",
"112",
"120",
"121",
"188"
],
"safe_collection_channels": [
"authorized_windows99_console_verify_stdout_only",
"existing_management_channel_verify_mode_only",
"committed_no_secret_artifact_file_then_scorecard_rerun"
],
"schema_version": "windows99_vmware_verify_collection_packet_v1",
"status": "blocked_windows99_verify_output_missing_host_reachable",
"target_host": "192.168.0.99",
"target_host_alias": "99",
"verify_ready": false
},
"windows99_vmware_autostart": {
"blockers": [
"windows99_vmware_autostart_readback_missing"