wooo/awoooi
1
1
Fork 0
Code Issues 2 Pull Requests 1 Actions 3 Packages Projects Releases Wiki Activity

fix(recovery): update reboot slo fail-closed readback
Some checks failed
CD Pipeline / workflow-shape (push) Successful in 0s
Details
CD Pipeline / cancel-stale-cd (push) Has been skipped
Details
CD Pipeline / tests (push) Failing after 28s
Details
CD Pipeline / build-and-deploy (push) Has been skipped
Details
CD Pipeline / post-deploy-checks (push) Has been skipped
Details

Browse Source
This commit is contained in:
Your Name
2026-06-30 20:56:29 +08:00
parent 92d0290734
commit 789e78a17f
4 changed files with 306 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files

290
docs/operations/awoooi-reboot-auto-recovery-slo-scorecard.snapshot.json
View File

@@ -1,112 +1,218 @@
{
"active_blockers": [
"host_boot_observation_older_than_target_window"
"all_required_hosts_not_in_10_minute_reboot_window",
"backup_core_green_not_1",
"host_188_service_green_not_1",
"host_boot_observation_older_than_target_window",
"host_unreachable_after_reboot",
"host_uptime_unknown",
"local_disk_free_below_minimum",
"post_start_blocked_not_zero",
"product_data_green_not_1",
"reboot_event_required_host_unreachable",
"service_green_not_1",
"wazuh_dashboard_degraded"
],
"can_claim_all_services_recovered_within_target": false,
"capacity": {
"checked": true,
"free_gib": 4.454,
"free_gib": 0.751,
"min_free_gib": 2.0
},
"generated_at": "2026-06-29T23:45:54+08:00",
"generated_at": "2026-06-30T20:44:37+08:00",
"host_boot_detection": {
"host_rows": [
{
"alias": "110",
"boot_id": "a3dfae32-3762-4394-86fa-a342aea07df5",
"alias": "99",
"boot_id": "reachable_unknown_boot",
"reachable": true,
"startup_active": "inactive",
"startup_enabled": "enabled",
"startup_active": "unknown",
"startup_enabled": "unknown",
"startup_unit": "vmware-host-autostart",
"systemd_state": "ping_reachable",
"target": "192.168.0.99",
"uptime_seconds": -1
},
{
"alias": "110",
"boot_id": "node_exporter_1782811060",
"reachable": true,
"startup_active": "unknown",
"startup_enabled": "unknown",
"startup_unit": "awoooi-startup-110.service",
"systemd_state": "degraded",
"systemd_state": "node_exporter",
"target": "wooo@192.168.0.110",
"uptime_seconds": 534656
"uptime_seconds": 12406
},
{
"alias": "111",
"boot_id": "unknown",
"reachable": false,
"startup_active": "unknown",
"startup_enabled": "unknown",
"startup_unit": "vm-host-boot",
"systemd_state": "unknown",
"target": "192.168.0.111",
"uptime_seconds": -1
},
{
"alias": "112",
"boot_id": "node_exporter_1782810990",
"reachable": true,
"startup_active": "unknown",
"startup_enabled": "unknown",
"startup_unit": "vm-host-boot",
"systemd_state": "node_exporter",
"target": "192.168.0.112",
"uptime_seconds": 12484
},
{
"alias": "120",
"boot_id": "866d621e-bad0-4096-938e-b103db8e5e03",
"boot_id": "eeb5049d-2c82-46f8-b9ea-be3875842795",
"reachable": true,
"startup_active": "active",
"startup_enabled": "enabled",
"startup_unit": "k3s.service",
"systemd_state": "running",
"target": "wooo@192.168.0.120",
"uptime_seconds": 534648
"uptime_seconds": 12479
},
{
"alias": "121",
"boot_id": "119c4ea7-8b49-45aa-b60e-bcd4b5dd0979",
"boot_id": "777fc552-1582-48bc-b09e-e00cca5cc2ce",
"reachable": true,
"startup_active": "active",
"startup_enabled": "enabled",
"startup_unit": "k3s.service",
"systemd_state": "running",
"target": "wooo@192.168.0.121",
"uptime_seconds": 534605
"uptime_seconds": 12480
},
{
"alias": "188",
"boot_id": "9cc1f1fc-b7cc-42c1-bc83-d495f0e3c863",
"boot_id": "ae363b77-286f-4c56-84b6-c77e530d9e87",
"reachable": true,
"startup_active": "inactive",
"startup_active": "failed",
"startup_enabled": "enabled",
"startup_unit": "awoooi-startup.service",
"systemd_state": "running",
"systemd_state": "degraded",
"target": "ollama@192.168.0.188",
"uptime_seconds": 534664
"uptime_seconds": 12420
}
],
"max_observed_uptime_seconds": 541538,
"max_observed_uptime_seconds": 12484,
"missing_hosts": [],
"observed_hosts": [
"110",
"111",
"112",
"120",
"121",
"188"
"188",
"99"
],
"required_hosts": [
"110",
"111",
"112",
"120",
"121",
"188"
"188",
"99"
],
"stale_hosts": [
"110",
"112",
"120",
"121",
"188"
],
"unknown_uptime_hosts": [],
"unreachable_hosts": []
"unknown_uptime_hosts": [
"99"
],
"unreachable_hosts": [
"111"
]
},
"post_reboot_readiness": {
"backup_core_green": true,
"host_188_service_green": true,
"next_required_gates": "credential_escrow_evidence",
"overall_declaration": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED",
"post_start_blocked": 0,
"post_start_result": "FULL_STACK_GREEN_DR_ESCROW_BLOCKED",
"product_data_green": true,
"service_green": true,
"backup_core_green": false,
"host_188_service_green": false,
"next_required_gates": "product_data_freshness_recovery,backup_core_readback_recovery,credential_escrow_evidence,host_188_hygiene_maintenance_window",
"overall_declaration": "SERVICE_BLOCKED",
"post_start_blocked": 8,
"post_start_result": "BLOCKED",
"product_data_green": false,
"service_green": false,
"summary_present": true,
"wazuh_dashboard_degraded": false
"wazuh_dashboard_degraded": true
},
"safe_next_step": "timer_and_service_data_readback_green_wait_for_next_all_host_reboot_event_or_approved_reboot_drill_to_prove_10_minute_slo",
"reboot_event_detection": {
"all_required_hosts_in_reboot_window": false,
"all_required_hosts_observed": false,
"changed_boot_id_hosts": [],
"fresh_boot_hosts": [
"99"
],
"missing_hosts": [],
"readback_present": true,
"reboot_detected": true,
"rebooted_hosts": [
"99"
],
"state_file": "/tmp/awoooi-reboot-slo-live-20260630-2045/reboot-event-state.json",
"state_written": false,
"unreachable_hosts": [
"111"
]
},
"safe_next_step": "deploy_boot_triggered_slo_timer_and_collect_all_host_boot_probe_then_rerun_scorecard_until_status_slo_ready",
"schema_version": "awoooi_reboot_auto_recovery_slo_scorecard_v1",
"sla_recovery_eta": {
"deadline_status": "target_window_elapsed",
"estimated_recovery_status": "target_window_elapsed_or_not_detected",
"fixed_triage_order": [
"99_vmware_autostart_and_vm_power",
"host_boot_event_and_node_exporter",
"public_routes_and_maintenance_fallback",
"awoooi_k3s_workloads_and_registry",
"stockplatform_public_api_and_freshness",
"backup_health_and_offsite_evidence",
"telegram_alert_delivery_readback"
],
"next_update_due_seconds": 60,
"target_minutes": 10,
"target_seconds_remaining": 0
},
"source_controls": {
"cold_start_textfile_exporter_source_present": true,
"host_110_startup_unit_source_present": true,
"host_188_startup_unit_source_present": true,
"host_boot_probe_source_present": true,
"post_reboot_summary_source_present": true,
"public_maintenance_fallback_source_present": true,
"reboot_event_detector_source_present": true,
"slo_exporter_source_present": true,
"slo_installer_source_present": true,
"slo_systemd_service_source_present": true,
"slo_systemd_timer_source_present": true
"slo_systemd_timer_source_present": true,
"telegram_reboot_backup_alert_rules_source_present": true,
"windows_99_vmware_autostart_source_present": true
},
"status": "blocked_reboot_auto_recovery_slo_not_ready",
"stockplatform_data_freshness": {
"blocked_sources": [],
"ai_recommendations_recovery": {
"cleared_blocker": "ai_recommendations_stale",
"status": "not_verified"
},
"blocked_sources": [
{
"latest_date": null,
"notes": "PostgreSQL 結構尚未就緒：OperationalError",
"row_count": 0,
"source": "postgres",
"status": "not_configured"
}
],
"controlled_recovery_gate": {
"allowed_actions": [
"inspect_existing_ingestion_readback",
@@ -122,114 +228,36 @@
"reboot_or_service_restart_from_reboot_slo_lane"
],
"required": false,
"status": "not_required_freshness_recovered",
"status": "not_required_yet",
"target_selector": "stockplatform-v2:system_freshness:core.margin_short_daily,ai.recommendations"
},
"eod_window": {
"classification": "recovered_after_final_retry_window",
"classification": "blocked_non_current_trading_day",
"final_retry_window_end_local": "23:35",
"final_retry_window_passed": true,
"final_retry_window_passed": false,
"first_full_window_end_local": "19:15",
"next_action": "rerun_slo_verify_only_after_next_fresh_all_host_reboot_event_or_approved_reboot_drill",
"next_action": "inspect_trading_calendar_and_ingestion_logs",
"pending": false
},
"freshness_blockers": [],
"freshness_endpoint_readback_present": true,
"freshness_status": "ok",
"ingestion_blockers": [],
"ingestion_endpoint_readback_present": true,
"ingestion_status": "ok",
"latest_source_runs": [
{
"source_run_id": 3392,
"source_name": "intelligence_security_linker",
"target_date": null,
"status": "succeeded",
"started_at": "2026-06-29T15:00:09.333642Z",
"finished_at": "2026-06-29T15:00:09.333642Z"
},
{
"source_run_id": 3391,
"source_name": "intelligence_reports_import",
"target_date": null,
"status": "succeeded",
"started_at": "2026-06-29T15:00:08.472808Z",
"finished_at": "2026-06-29T15:00:08.472808Z"
},
{
"source_run_id": 3389,
"source_name": "official_margin_short_daily",
"target_date": "2026-06-29",
"status": "succeeded",
"started_at": "2026-06-29T13:05:13.341357Z",
"finished_at": "2026-06-29T13:05:13.341357Z"
},
{
"source_run_id": 3390,
"source_name": "official_margin_short_daily",
"target_date": "2026-06-29",
"status": "succeeded",
"started_at": "2026-06-29T13:05:13.341357Z",
"finished_at": "2026-06-29T13:05:13.341357Z"
},
{
"source_run_id": 3388,
"source_name": "intelligence_security_linker",
"target_date": null,
"status": "succeeded",
"started_at": "2026-06-29T13:00:08.792550Z",
"finished_at": "2026-06-29T13:00:08.792550Z"
},
{
"source_run_id": 3387,
"source_name": "intelligence_reports_import",
"target_date": null,
"status": "succeeded",
"started_at": "2026-06-29T13:00:07.822700Z",
"finished_at": "2026-06-29T13:00:07.822700Z"
},
{
"source_run_id": 3385,
"source_name": "official_margin_short_daily",
"target_date": "2026-06-29",
"status": "official_pending",
"started_at": "2026-06-29T12:05:16.716460Z",
"finished_at": "2026-06-29T12:05:16.716460Z"
},
{
"source_run_id": 3386,
"source_name": "official_margin_short_daily",
"target_date": "2026-06-29",
"status": "official_pending",
"started_at": "2026-06-29T12:05:16.716460Z",
"finished_at": "2026-06-29T12:05:16.716460Z"
}
"freshness_blockers": [
"postgres_not_ready"
],
"latest_trading_date": "2026-06-29",
"second_retry_checked_at": "2026-06-29T20:21:35+08:00",
"third_retry_checked_at": "2026-06-29T21:15:30+08:00",
"freshness_endpoint_readback_present": true,
"freshness_status": "not_configured",
"ingestion_blockers": [
"postgres_not_ready"
],
"ingestion_endpoint_readback_present": true,
"ingestion_status": "not_configured",
"latest_source_runs": [],
"latest_trading_date": "None",
"margin_short_recovery": {
"status": "recovered",
"cleared_blocker": "core_margin_short_daily_missing",
"cleared_ingestion_blocker": "core.margin_short_daily_incomplete",
"successful_source_run_ids": [
3389,
3390
]
},
"ai_recommendations_recovery": {
"status": "recovered",
"cleared_blocker": "ai_recommendations_stale"
},
"final_retry_checked_at": "2026-06-29T23:45:54+08:00"
"status": "not_verified",
"successful_source_run_ids": []
}
},
"target_minutes": 10,
"target_seconds": 600,
"latest_verify_only_metric": {
"ready": 0,
"blocker_count": 1,
"max_host_uptime_seconds": 541538,
"last_run_timestamp": 1782738942,
"artifact_dir": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260629-211422"
}
"target_seconds": 600
}