{ "schema_version": "agent99_full_host_reboot_drill_receipt_v2", "run_id": "agent99-20260711-1559", "status": "slo_breached_recovered_late", "timezone": "Asia/Taipei", "source_runtime": { "pre_drill_agent99_source_revision": "7cf83f7f59a5d72ce6c303f59c144bd5bf8566df", "final_agent99_source_revision": "442b319bb80e009291ced7afa4f704cbe3b64886", "followup_source_commits": [ "0a4ddb133e123d2d326984c932da978e0a38e77f", "442b319bb80e009291ced7afa4f704cbe3b64886", "801a50b017dd0fb975e675553075e67128202da2" ], "runtime_manifest_matched": true, "runtime_file_count": 14, "runtime_mismatch_count": 0, "required_tasks_healthy": "9/9", "startup_recovery_execution_limit": "PT20M", "agent99_deploy_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-173023.json", "production_principal_recover_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-173543.json" }, "preflight": { "all_required_hosts_reachable": true, "required_host_ids": [ "99", "110", "111", "112", "120", "121", "188" ], "core_routes_up": 6, "agent99_relay_ready": true, "agent99_queue_depth": 0 }, "timeline": { "observer_started_at": "2026-07-11T07:58:54Z", "first_service_route_down_at": "2026-07-11T07:59:51Z", "vm_guests_down_at": "2026-07-11T08:00:07Z", "windows99_observer_first_down_at": "2026-07-11T08:01:27Z", "windows99_actual_boot_at": "2026-07-11T16:08:03+08:00", "agent99_startup_recovery_started_at": "2026-07-11T16:08:57+08:00", "ten_minute_deadline_at": "2026-07-11T08:10:07Z", "agent99_startup_recovery_terminal": "terminated_by_pt10m_task_limit_before_terminal_breach_receipt", "vmware_autostart_task_ran_at": "2026-07-11T16:26:48+08:00", "host188_postgresql_ready_at": "2026-07-11T16:40:05+08:00", "host120_vix_soft_reboot_at": "2026-07-11T16:49:32+08:00", "host120_recovered_at": "2026-07-11T16:52:14+08:00", "host121_vix_soft_reboot_at": "2026-07-11T16:56:00+08:00", "host121_recovered_at": "2026-07-11T16:58:46+08:00", "core_routes_green_at": "2026-07-11T16:59:01+08:00", "signoz_bridge_green_at": "2026-07-11T17:02:00+08:00", "final_agent99_service_recovery_at": "2026-07-11T17:35:54+08:00", "all_public_surface_matrix_verified_at": "2026-07-11T17:37:00+08:00" }, "observer_evidence": { "initial_events": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559/events.tsv", "continued_summary": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559-part2/summary.txt", "continued_host_down_samples": 567, "continued_host_down_samples_authoritative": false, "continued_host_probe_false_negative_cause": "curl_telnet_waited_for_ssh_payload_and_returned_nonzero_after_tcp_connect", "corrected_observer": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-observer-fixed", "continued_route_down_samples": 486, "continued_recovery_transitions": 0, "secret_value_read": false, "remote_write_performed": false }, "recovery_evidence": { "latest_110_full_sop_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623", "post_start_result": "GREEN_WITH_EVIDENCE_WARNINGS", "service_green": true, "product_data_green": true, "stock_freshness_status": "ok", "backup_core_green": true, "host188_service_green": true, "host188_hygiene_green": true, "required_hosts_reachable": "7/7", "windows99_vmware_verify_ready": true, "windows99_vmware_running_guests": "5/5", "windows99_update_no_auto_reboot_ready": true, "scorecard_active_blockers": [ "all_required_hosts_not_in_10_minute_reboot_window", "fresh_all_host_reboot_event_missing", "host_boot_observation_older_than_target_window" ], "vmware_controlled_apply_evidence": "C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "vmware_controlled_apply_backup": "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "agent99_service_recovery_elapsed_seconds": 10.3, "agent99_service_recovery_completed": true, "agent99_coordinator_verified": true, "agent99_reboot_slo_claimed": false, "public_routes_green": "23/23", "public_surface_results": [ {"id": "awoooi-web", "url": "https://awoooi.wooo.work/", "http_status": 307}, {"id": "awoooi-api", "url": "https://awoooi.wooo.work/api/v1/health", "http_status": 200}, {"id": "iwooos", "url": "https://awoooi.wooo.work/zh-TW/iwooos", "http_status": 200}, {"id": "vibework", "url": "https://vibework.wooo.work/", "http_status": 200}, {"id": "awooogo", "url": "https://awooogo.wooo.work/", "http_status": 200}, {"id": "2026fifa", "url": "https://2026fifa.wooo.work/", "http_status": 200}, {"id": "agent", "url": "https://agent.wooo.work/", "http_status": 200}, {"id": "momo", "url": "https://mo.wooo.work/", "http_status": 302}, {"id": "momo-health", "url": "https://mo.wooo.work/health", "http_status": 200}, {"id": "stock", "url": "https://stock.wooo.work/", "http_status": 200}, {"id": "stock-health", "url": "https://stock.wooo.work/healthz", "http_status": 200}, {"id": "stock-api", "url": "https://stock.wooo.work/api/healthz", "http_status": 200}, {"id": "bitan", "url": "https://bitan.wooo.work/", "http_status": 200}, {"id": "tsenyang", "url": "https://tsenyang.com/", "http_status": 200}, {"id": "tsenyang-www", "url": "https://www.tsenyang.com/", "http_status": 200}, {"id": "vtuber", "url": "https://vtuber.wooo.work/", "http_status": 200}, {"id": "gitea", "url": "https://gitea.wooo.work/", "http_status": 200}, {"id": "harbor", "url": "https://harbor.wooo.work/", "http_status": 200}, {"id": "registry", "url": "https://registry.wooo.work/", "http_status": 200}, {"id": "sentry", "url": "https://sentry.wooo.work/", "http_status": 302}, {"id": "signoz", "url": "https://signoz.wooo.work/", "http_status": 200}, {"id": "langfuse", "url": "https://langfuse.wooo.work/", "http_status": 200}, {"id": "aiops", "url": "https://aiops.wooo.work/", "http_status": 307} ] }, "latest_readback": { "host_99": "up", "host_110": "up", "host_111": "up", "host_112": "up", "host_120": "up_k3s_active", "host_121": "up_k3s_active", "host_188": "up_startup_active", "awoooi_api_http_status": 200, "stock_freshness_http_status": 200, "momo_health_http_status": 200, "gitea_http_status": 200, "harbor_http_status": 200, "signoz_http_status": 200, "all_public_routes": "23/23_http_2xx_or_expected_3xx" }, "bounded_recovery_attempts": { "windows99_wol_packets_sent": 36, "windows99_wol_result": "no_immediate_management_readback", "windows99_intel_amt_probe": "timeout", "macos111_wol_packets_sent": 12, "macos111_wol_result": "no_immediate_network_readback", "cloud_management_result": "no_authorized_external_executor_available" }, "root_cause_classification": { "slo_result": "ten_minute_recovery_failed_even_though_windows99_booted_before_the_deadline", "primary_causes": [ "windows99_vmware_autostart_task_did_not_run_until_18_minutes_45_seconds_after_windows_boot", "agent99_startup_recovery_was_terminated_by_its_pt10m_scheduled_task_limit", "agent99_boot_event_was_consumed_before_terminal_recovery_and_later_runs_lost_the_fresh_reboot_scope", "host188_postgresql_was_not_ready_until_16_40_05_and_k3s_120_121_waited_on_192_168_0_188_5432" ], "observer_defect": "host TCP probe produced false negatives after switching from nc to curl telnet; route samples remain valid", "host120_121_secondary_reboots": "VMware VIX SoftReboot requests were issued from the Windows99 VMware control plane; no Agent99 or repo reset path exists and Windows process auditing was unavailable to attribute the actor", "public_experience_limit": "local edge fallback eventually worked for AWOOOI but Stock Gitea and Harbor initially exposed raw 502 because no external maintenance origin is deployed", "external_power_limit": "no verified off-LAN L0 supervisor or out-of-band power callback is deployed" }, "required_closure": [ "deploy_external_l0_supervisor_outside_the_local_failure_domain", "provision_verified_out_of_band_power_for_host99", "provision_external_maintenance_origin_or_cdn_failover", "audit_every_vmware_reset_stop_or_suspend_with_trace_id_and_verifier", "repeat_the_full_drill_and_close_within_600_seconds" ], "claims_verified": [ "all_required_hosts_recovered_late", "all_23_public_routes_are_currently_reachable", "agent99_runtime_is_integrated_with_the_full_sop_service_recovery_verifier", "windows99_vmware_autostart_and_update_policy_are_currently_green", "the_original_ten_minute_slo_breach_was_not_rewritten_as_success" ], "claims_forbidden": [ "all_services_recovered_within_ten_minutes_for_this_drill", "future_full_host_reboots_are_proven_to_recover_within_ten_minutes", "external_maintenance_failover_is_complete", "external_l0_power_recovery_is_deployed" ] }