178 lines
9.0 KiB
JSON
178 lines
9.0 KiB
JSON
{
|
|
"schema_version": "agent99_full_host_reboot_drill_receipt_v2",
|
|
"run_id": "agent99-20260711-1559",
|
|
"status": "slo_breached_recovered_late",
|
|
"timezone": "Asia/Taipei",
|
|
"source_runtime": {
|
|
"pre_drill_agent99_source_revision": "7cf83f7f59a5d72ce6c303f59c144bd5bf8566df",
|
|
"final_agent99_source_revision": "442b319bb80e009291ced7afa4f704cbe3b64886",
|
|
"followup_source_commits": [
|
|
"0a4ddb133e123d2d326984c932da978e0a38e77f",
|
|
"442b319bb80e009291ced7afa4f704cbe3b64886",
|
|
"801a50b017dd0fb975e675553075e67128202da2"
|
|
],
|
|
"runtime_manifest_matched": true,
|
|
"runtime_file_count": 14,
|
|
"runtime_mismatch_count": 0,
|
|
"required_tasks_healthy": "9/9",
|
|
"startup_recovery_execution_limit": "PT20M",
|
|
"agent99_deploy_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-173023.json",
|
|
"production_principal_recover_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-173543.json"
|
|
},
|
|
"preflight": {
|
|
"all_required_hosts_reachable": true,
|
|
"required_host_ids": [
|
|
"99",
|
|
"110",
|
|
"111",
|
|
"112",
|
|
"120",
|
|
"121",
|
|
"188"
|
|
],
|
|
"core_routes_up": 6,
|
|
"agent99_relay_ready": true,
|
|
"agent99_queue_depth": 0
|
|
},
|
|
"timeline": {
|
|
"observer_started_at": "2026-07-11T07:58:54Z",
|
|
"first_service_route_down_at": "2026-07-11T07:59:51Z",
|
|
"vm_guests_down_at": "2026-07-11T08:00:07Z",
|
|
"windows99_observer_first_down_at": "2026-07-11T08:01:27Z",
|
|
"windows99_actual_boot_at": "2026-07-11T16:08:03+08:00",
|
|
"agent99_startup_recovery_started_at": "2026-07-11T16:08:57+08:00",
|
|
"ten_minute_deadline_at": "2026-07-11T08:10:07Z",
|
|
"agent99_startup_recovery_terminal": "terminated_by_pt10m_task_limit_before_terminal_breach_receipt",
|
|
"vmware_autostart_task_ran_at": "2026-07-11T16:26:48+08:00",
|
|
"host188_postgresql_ready_at": "2026-07-11T16:40:05+08:00",
|
|
"host120_vix_soft_reboot_at": "2026-07-11T16:49:32+08:00",
|
|
"host120_recovered_at": "2026-07-11T16:52:14+08:00",
|
|
"host121_vix_soft_reboot_at": "2026-07-11T16:56:00+08:00",
|
|
"host121_recovered_at": "2026-07-11T16:58:46+08:00",
|
|
"core_routes_green_at": "2026-07-11T16:59:01+08:00",
|
|
"signoz_bridge_green_at": "2026-07-11T17:02:00+08:00",
|
|
"final_agent99_service_recovery_at": "2026-07-11T17:35:54+08:00",
|
|
"all_public_surface_matrix_verified_at": "2026-07-11T17:37:00+08:00"
|
|
},
|
|
"observer_evidence": {
|
|
"initial_events": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559/events.tsv",
|
|
"continued_summary": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559-part2/summary.txt",
|
|
"continued_host_down_samples": 567,
|
|
"continued_host_down_samples_authoritative": false,
|
|
"continued_host_probe_false_negative_cause": "curl_telnet_waited_for_ssh_payload_and_returned_nonzero_after_tcp_connect",
|
|
"corrected_observer": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-observer-fixed",
|
|
"continued_route_down_samples": 486,
|
|
"continued_recovery_transitions": 0,
|
|
"secret_value_read": false,
|
|
"remote_write_performed": false
|
|
},
|
|
"recovery_evidence": {
|
|
"latest_110_full_sop_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623",
|
|
"post_start_result": "GREEN_WITH_EVIDENCE_WARNINGS",
|
|
"service_green": true,
|
|
"product_data_green": true,
|
|
"stock_freshness_status": "ok",
|
|
"backup_core_green": true,
|
|
"host188_service_green": true,
|
|
"host188_hygiene_green": true,
|
|
"required_hosts_reachable": "7/7",
|
|
"windows99_vmware_verify_ready": true,
|
|
"windows99_vmware_running_guests": "5/5",
|
|
"windows99_update_no_auto_reboot_ready": true,
|
|
"scorecard_active_blockers": [
|
|
"all_required_hosts_not_in_10_minute_reboot_window",
|
|
"fresh_all_host_reboot_event_missing",
|
|
"host_boot_observation_older_than_target_window"
|
|
],
|
|
"vmware_controlled_apply_evidence": "C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json",
|
|
"vmware_controlled_apply_backup": "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505",
|
|
"agent99_service_recovery_elapsed_seconds": 10.3,
|
|
"agent99_service_recovery_completed": true,
|
|
"agent99_coordinator_verified": true,
|
|
"agent99_reboot_slo_claimed": false,
|
|
"public_routes_green": "23/23",
|
|
"public_surface_results": [
|
|
{"id": "awoooi-web", "url": "https://awoooi.wooo.work/", "http_status": 307},
|
|
{"id": "awoooi-api", "url": "https://awoooi.wooo.work/api/v1/health", "http_status": 200},
|
|
{"id": "iwooos", "url": "https://awoooi.wooo.work/zh-TW/iwooos", "http_status": 200},
|
|
{"id": "vibework", "url": "https://vibework.wooo.work/", "http_status": 200},
|
|
{"id": "awooogo", "url": "https://awooogo.wooo.work/", "http_status": 200},
|
|
{"id": "2026fifa", "url": "https://2026fifa.wooo.work/", "http_status": 200},
|
|
{"id": "agent", "url": "https://agent.wooo.work/", "http_status": 200},
|
|
{"id": "momo", "url": "https://mo.wooo.work/", "http_status": 302},
|
|
{"id": "momo-health", "url": "https://mo.wooo.work/health", "http_status": 200},
|
|
{"id": "stock", "url": "https://stock.wooo.work/", "http_status": 200},
|
|
{"id": "stock-health", "url": "https://stock.wooo.work/healthz", "http_status": 200},
|
|
{"id": "stock-api", "url": "https://stock.wooo.work/api/healthz", "http_status": 200},
|
|
{"id": "bitan", "url": "https://bitan.wooo.work/", "http_status": 200},
|
|
{"id": "tsenyang", "url": "https://tsenyang.com/", "http_status": 200},
|
|
{"id": "tsenyang-www", "url": "https://www.tsenyang.com/", "http_status": 200},
|
|
{"id": "vtuber", "url": "https://vtuber.wooo.work/", "http_status": 200},
|
|
{"id": "gitea", "url": "https://gitea.wooo.work/", "http_status": 200},
|
|
{"id": "harbor", "url": "https://harbor.wooo.work/", "http_status": 200},
|
|
{"id": "registry", "url": "https://registry.wooo.work/", "http_status": 200},
|
|
{"id": "sentry", "url": "https://sentry.wooo.work/", "http_status": 302},
|
|
{"id": "signoz", "url": "https://signoz.wooo.work/", "http_status": 200},
|
|
{"id": "langfuse", "url": "https://langfuse.wooo.work/", "http_status": 200},
|
|
{"id": "aiops", "url": "https://aiops.wooo.work/", "http_status": 307}
|
|
]
|
|
},
|
|
"latest_readback": {
|
|
"host_99": "up",
|
|
"host_110": "up",
|
|
"host_111": "up",
|
|
"host_112": "up",
|
|
"host_120": "up_k3s_active",
|
|
"host_121": "up_k3s_active",
|
|
"host_188": "up_startup_active",
|
|
"awoooi_api_http_status": 200,
|
|
"stock_freshness_http_status": 200,
|
|
"momo_health_http_status": 200,
|
|
"gitea_http_status": 200,
|
|
"harbor_http_status": 200,
|
|
"signoz_http_status": 200,
|
|
"all_public_routes": "23/23_http_2xx_or_expected_3xx"
|
|
},
|
|
"bounded_recovery_attempts": {
|
|
"windows99_wol_packets_sent": 36,
|
|
"windows99_wol_result": "no_immediate_management_readback",
|
|
"windows99_intel_amt_probe": "timeout",
|
|
"macos111_wol_packets_sent": 12,
|
|
"macos111_wol_result": "no_immediate_network_readback",
|
|
"cloud_management_result": "no_authorized_external_executor_available"
|
|
},
|
|
"root_cause_classification": {
|
|
"slo_result": "ten_minute_recovery_failed_even_though_windows99_booted_before_the_deadline",
|
|
"primary_causes": [
|
|
"windows99_vmware_autostart_task_did_not_run_until_18_minutes_45_seconds_after_windows_boot",
|
|
"agent99_startup_recovery_was_terminated_by_its_pt10m_scheduled_task_limit",
|
|
"agent99_boot_event_was_consumed_before_terminal_recovery_and_later_runs_lost_the_fresh_reboot_scope",
|
|
"host188_postgresql_was_not_ready_until_16_40_05_and_k3s_120_121_waited_on_192_168_0_188_5432"
|
|
],
|
|
"observer_defect": "host TCP probe produced false negatives after switching from nc to curl telnet; route samples remain valid",
|
|
"host120_121_secondary_reboots": "VMware VIX SoftReboot requests were issued from the Windows99 VMware control plane; no Agent99 or repo reset path exists and Windows process auditing was unavailable to attribute the actor",
|
|
"public_experience_limit": "local edge fallback eventually worked for AWOOOI but Stock Gitea and Harbor initially exposed raw 502 because no external maintenance origin is deployed",
|
|
"external_power_limit": "no verified off-LAN L0 supervisor or out-of-band power callback is deployed"
|
|
},
|
|
"required_closure": [
|
|
"deploy_external_l0_supervisor_outside_the_local_failure_domain",
|
|
"provision_verified_out_of_band_power_for_host99",
|
|
"provision_external_maintenance_origin_or_cdn_failover",
|
|
"audit_every_vmware_reset_stop_or_suspend_with_trace_id_and_verifier",
|
|
"repeat_the_full_drill_and_close_within_600_seconds"
|
|
],
|
|
"claims_verified": [
|
|
"all_required_hosts_recovered_late",
|
|
"all_23_public_routes_are_currently_reachable",
|
|
"agent99_runtime_is_integrated_with_the_full_sop_service_recovery_verifier",
|
|
"windows99_vmware_autostart_and_update_policy_are_currently_green",
|
|
"the_original_ten_minute_slo_breach_was_not_rewritten_as_success"
|
|
],
|
|
"claims_forbidden": [
|
|
"all_services_recovered_within_ten_minutes_for_this_drill",
|
|
"future_full_host_reboots_are_proven_to_recover_within_ten_minutes",
|
|
"external_maintenance_failover_is_complete",
|
|
"external_l0_power_recovery_is_deployed"
|
|
]
|
|
}
|