docs(recovery): record reboot breach and late recovery
This commit is contained in:
ogt
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "agent99_enterprise_ai_automation_work_items_v1",
|
||||
"generated_at": "2026-07-11T15:35:00+08:00",
|
||||
"status": "in_progress_p0_agent99_full_host_reboot_coordinator",
|
||||
"generated_at": "2026-07-11T17:40:00+08:00",
|
||||
"status": "in_progress_p0_reboot_slo_breached_recovered_late_l1_l2_integrated_l0_required",
|
||||
"scope_complete": false,
|
||||
"north_star": "Make Windows host 192.168.0.99 the policy-controlled on-prem AI execution node for all hosts, VMs, services, products, sites, tools, packages, alerts and recovery workflows, with external monitoring when host 99 is unavailable.",
|
||||
"canonical_sources": {
|
||||
@@ -19,20 +19,20 @@
|
||||
"current_p0": {
|
||||
"id": "AG99-P0-007",
|
||||
"title": "Agent99-coordinated all-host ten-minute reboot recovery state machine",
|
||||
"reason": "Live Recover falsely completed after only VM, five-host, Harbor, K3s and five-route checks; it did not gate closure on the full 110 cold-start scorecard, seven-host boot window, product data, backup, edge fallback or host-188 hygiene.",
|
||||
"implementation_status": "recovery_coordinator_source_test_and_windows_validate_green_runtime_promote_pending",
|
||||
"next_action": "Create a local commit, atomically promote it to Windows 99, verify the runtime manifest and production-principal Recover evidence, then execute the approved real all-host reboot drill and require fresh T+10 scorecard closure."
|
||||
"reason": "The approved drill breached T+10 and later recovered. Live Windows boot evidence corrected the observer false negative: host 99 booted before the deadline, but VMware startup was delayed, Agent99 was killed by its PT10M task limit, the reboot event was consumed early and dependent PostgreSQL/K3s services recovered late. Agent99 L1 and 110 L2 are now integrated in production; external L0 observation, out-of-band power and external maintenance remain required before the next 600-second drill can close.",
|
||||
"implementation_status": "slo_breached_recovered_late_agent99_l1_vmware_and_110_l2_runtime_verified_external_l0_missing",
|
||||
"next_action": "Deploy the external L0 supervisor outside the local failure domain, bind verified OOB power, maintenance and Agent99 resume callbacks, then repeat one incident-correlated full reboot drill and require zero blockers within 600 seconds."
|
||||
},
|
||||
"current_runtime_truth": {
|
||||
"observed_at": "2026-07-11T10:27:41+08:00",
|
||||
"observed_at": "2026-07-11T17:37:00+08:00",
|
||||
"awoooi_health_http_status": 200,
|
||||
"awoooi_health_state": "degraded",
|
||||
"awoooi_health_state": "recovered_late_currently_healthy",
|
||||
"awoooi_runtime_build_commit_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f",
|
||||
"gitea_main_observed_sha": "f102616f0",
|
||||
"gitea_main_deployed_source_sha": "4a4b95e5135e8cf71d807a54700a962f5490088f",
|
||||
"runtime_work_items_generated_at": "2026-07-10T19:40:00+08:00",
|
||||
"windows99_agent99_manifest_observed_at": "2026-07-11T15:20:33+08:00",
|
||||
"windows99_agent99_source_revision": "34e32f6ed6f29952181cd7aa43ca19210d25339b",
|
||||
"windows99_agent99_manifest_observed_at": "2026-07-11T17:31:00+08:00",
|
||||
"windows99_agent99_source_revision": "442b319bb80e009291ced7afa4f704cbe3b64886",
|
||||
"windows99_agent99_runtime_matched": true,
|
||||
"windows99_agent99_runtime_file_count": 14,
|
||||
"windows99_agent99_runtime_mismatch_count": 0,
|
||||
@@ -41,16 +41,27 @@
|
||||
"windows99_agent99_relay_listener_count": 1,
|
||||
"windows99_agent99_alert_incoming_count": 0,
|
||||
"windows99_agent99_command_pending_count": 0,
|
||||
"reboot_slo_status": "blocked_reboot_auto_recovery_slo_not_ready",
|
||||
"windows99_agent99_startup_recovery_execution_limit": "PT20M",
|
||||
"windows99_vmware_running_guests": "5/5",
|
||||
"public_surface_matrix": "23/23_http_2xx_or_expected_3xx",
|
||||
"reboot_slo_status": "slo_breached_recovered_late",
|
||||
"reboot_slo_readiness_percent": 87,
|
||||
"reboot_slo_primary_blocker": "fresh_all_host_reboot_event_missing",
|
||||
"reboot_slo_primary_blocker": "all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"reboot_slo_active_blockers": [
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"host_boot_observation_older_than_target_window",
|
||||
"fresh_all_host_reboot_event_missing"
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"host_boot_observation_older_than_target_window"
|
||||
],
|
||||
"reboot_program_remaining_gaps": [
|
||||
"external_l0_outage_supervisor_not_deployed",
|
||||
"windows99_out_of_band_power_control_missing",
|
||||
"external_public_maintenance_origin_not_deployed",
|
||||
"next_full_reboot_600_second_drill_not_yet_passed"
|
||||
],
|
||||
"reboot_drill_receipt": "docs/operations/agent99-full-host-reboot-drill-20260711.snapshot.json",
|
||||
"claims_forbidden": [
|
||||
"all_services_recovered_within_10_minutes",
|
||||
"all_services_recovered_within_10_minutes_for_this_drill",
|
||||
"future_full_reboots_are_proven_within_10_minutes",
|
||||
"all_products_are_latest",
|
||||
"agent99_is_a_complete_ai_agent",
|
||||
"http_200_means_full_stack_healthy",
|
||||
@@ -167,7 +178,7 @@
|
||||
]
|
||||
},
|
||||
"control_plane": [
|
||||
{"node": "external_watcher_edge", "responsibility": "Detect host 99 or all-site outage and keep alerts/status/maintenance available outside the local failure domain."},
|
||||
{"node": "external_watcher_edge", "responsibility": "Detect host 99 or all-site outage outside the LAN, preserve the incident durably, invoke allowlisted out-of-band power and maintenance callbacks, then enqueue Agent99 Recover when 99 returns."},
|
||||
{"node": "awoooi", "responsibility": "Canonical events, incidents, fingerprints, recurrence, product and SLO read models."},
|
||||
{"node": "awooop", "responsibility": "Work Items, Runs, approvals, candidates, rollback, verifier and asset ledger."},
|
||||
{"node": "iwooos_aisoc", "responsibility": "Security evidence, policy, investigation and controlled response."},
|
||||
@@ -430,20 +441,20 @@
|
||||
"owner_lane": "reboot_auto_recovery",
|
||||
"dependencies": ["AG99-P0-001", "AG99-P0-005", "AG99-P0-006"],
|
||||
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188", "all:public_surfaces"],
|
||||
"problem": "The deployed Agent99 Recover mode can falsely close after only VM, five-host, Harbor, K3s and route checks; full seven-host reboot-window, data, backup, edge and hygiene evidence is not yet part of its runtime outcome.",
|
||||
"problem": "The real drill breached the ten-minute SLO and later recovered. Host 99 actually booted before the deadline, but observer TCP logic produced false host-down samples, Agent99 was terminated by its PT10M task limit, the boot event was consumed before terminal recovery, VMware startup was delayed and PostgreSQL/K3s dependencies recovered late. L1/L2 are now integrated; L0 and a passing 600-second drill remain open.",
|
||||
"trigger": "Fresh boot identity, host down/up transition, Windows startup task or all-host reboot correlation.",
|
||||
"ai_role": "Dependency-aware reboot recovery orchestrator",
|
||||
"action_mode": "controlled_apply",
|
||||
"evidence_sources": ["boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness", "110 full-SOP artifact", "backup readback", "edge fallback", "host-188 hygiene", "Windows 99 runtime manifest"],
|
||||
"controlled_actions": ["detect reboot epoch", "start dependency phases", "correlate the latest 110 full-SOP artifact", "retry bounded stateless recovery", "fail closed without fresh reboot scorecard", "emit ETA and blockers"],
|
||||
"evidence_sources": ["off-LAN public probes", "out-of-band power receipt", "boot id", "uptime", "VM power", "SSH", "exporters", "service DAG", "public and data freshness", "110 full-SOP artifact", "backup readback", "edge fallback", "host-188 hygiene", "Windows 99 runtime manifest"],
|
||||
"controlled_actions": ["detect outage outside the LAN", "invoke allowlisted out-of-band power restoration", "activate external maintenance origin", "detect reboot epoch", "enqueue Agent99 Recover after 99 returns", "start dependency phases", "correlate the latest 110 full-SOP artifact", "retry bounded stateless recovery", "fail closed without fresh reboot scorecard", "emit ETA and blockers"],
|
||||
"prohibited_actions": ["blind host reboot", "declare green from route 200", "skip data and backup checks"],
|
||||
"rollback": "Stop at last verified phase, activate maintenance state and preserve dependent workloads.",
|
||||
"verifier": "Measure detection time, first availability and full green time; require seven host rows, service/data/backup, host-188 hygiene, edge fallback, VMware/update policy, fresh artifact, fresh reboot window, zero active blockers and can_claim_slo=true.",
|
||||
"verifier": "Measure off-LAN detection time, host-99 power-on receipt, first availability and full green time; require seven host rows, service/data/backup, host-188 hygiene, external maintenance lifecycle, VMware/update policy, fresh artifact, fresh reboot window, zero active blockers and can_claim_slo=true.",
|
||||
"callback": "Write reboot incident phase and SLO result to AWOOOI, AwoooP and Telegram.",
|
||||
"acceptance": "A fresh all-host reboot drill is detected and all required gates recover within 600 seconds or report a precise blocker and ETA.",
|
||||
"implementation_progress": "recoveryCoordinator source, config migration, outcome gate, no-secret live preflight and Windows staged parser/contract/synthetic validation are green; runtime promote and real reboot drill remain.",
|
||||
"evidence_refs": ["Windows 99 live preflight 2026-07-11T15:20:33+08:00", "Windows staged validation evidence C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-153425.json", "PowerShell synthetic recovery coordinator 12/12 green", "Python coordinator contract 6 passed", "Agent99 API contracts 45 passed", "reboot-recovery suite 131 passed", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841"],
|
||||
"next_action": "Create a local commit, atomically promote the tested package to Windows 99, run production-principal Recover baseline and live preflight, then execute the approved all-host reboot drill and require fresh T+10 closure."
|
||||
"implementation_progress": "Agent99 runtime revision 442b319bb80e009291ced7afa4f704cbe3b64886 is live with 14/14 matched files, nine tasks, PT20M Startup-Recovery and durable pending reboot state. Principal-aware VMware evidence and scorecard parsing are live. Production-principal Recover completed service recovery in 10.3 seconds with the full-SOP coordinator verified and rebootSloClaimed=false. Seven hosts and 23/23 public surfaces are currently green; the original T+10 breach remains and external L0/OOB/maintenance plus a passing repeat drill are still required.",
|
||||
"evidence_refs": ["Windows deployment C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-173023.json", "Windows production-principal Recover C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-173543.json", "Windows VMware controlled apply C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "source commits 0a4ddb133, 442b319bb and 801a50b01", "reboot-recovery suite 141 passed", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623", "docs/operations/agent99-full-host-reboot-drill-20260711.snapshot.json", "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-observer-fixed"],
|
||||
"next_action": "Deploy external L0 outside the local failure domain with OOB power, maintenance and Agent99 resume callbacks, then run one incident-correlated reboot drill and require L0, L1 and L2 to close with zero blockers within 600 seconds."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-008",
|
||||
@@ -454,19 +465,20 @@
|
||||
"owner_lane": "windows99_vmware",
|
||||
"dependencies": ["AG99-P0-003", "AG99-P0-005"],
|
||||
"scope_refs": ["host:99", "host:110", "host:111", "host:112", "host:120", "host:121", "host:188"],
|
||||
"problem": "The no-secret baseline is green, but automatic task execution and guest recovery still require proof during the next real Windows 99 reboot.",
|
||||
"problem": "The drill showed an 18-minute-45-second delay between Windows boot and the VMware autostart task, a principal visibility mismatch for the Kali VM and two later VIX SoftReboot actions without an attributable trace. Current VM power is green, but the next startup must prove timing and every reset/stop/suspend must be audited.",
|
||||
"trigger": "Windows startup, VMware exclusive-lock error, guest power mismatch, VMX drift or Windows Update policy drift.",
|
||||
"ai_role": "Windows and VMware control agent",
|
||||
"action_mode": "check_mode_then_controlled_apply",
|
||||
"evidence_sources": ["VMX inventory", "vmrun list", "VMware services", "scheduled tasks", "lock owners", "Windows Update policy"],
|
||||
"controlled_actions": ["repair stale lock after owner proof", "configure autostart order", "verify guest tools and network", "apply reversible update policy"],
|
||||
"prohibited_actions": ["delete unknown lock", "guess VMX path", "power off VM", "reboot Windows without break-glass"],
|
||||
"prohibited_actions": ["delete unknown lock", "guess VMX path", "power off VM", "reset stop or suspend a VM without trace and verifier", "reboot Windows without break-glass"],
|
||||
"rollback": "Restore previous task, service and policy exports; keep guest power unchanged during check mode.",
|
||||
"verifier": "All required VMX paths exist, guests report running and reachable, tasks survive reboot, update policy drift is zero.",
|
||||
"callback": "Update reboot SLO VMware and Windows policy gates.",
|
||||
"acceptance": "99 startup automatically launches VMware guests 110/188/112/120/121 in order, physical host 111 is verified separately, no exclusive-lock failure occurs and Windows Update cannot surprise-reboot the host.",
|
||||
"implementation_progress": "VMX inventory, autostart config, guest power readback, privileged no-secret collector and Windows Update no-auto-reboot policy are live green; 111 is corrected to a physical Mac readback lane.",
|
||||
"next_action": "Keep baseline read-only and verify actual task execution, all five guest boots and physical host 111 readback during AG99-P0-007 real reboot drill."
|
||||
"implementation_progress": "The controlled apply receipt windows99-vmware-autostart-apply-20260711-173505.json completed DryRun, backup, Apply and independent Verify. VMX process count remained 5, required guests 110/188/120/121/112 are running, the generated startup script has a process-aware duplicate-start guard, the task and Windows Update policy are green, and no reboot, VM power change or secret read occurred. VMware logs prove the 120/121 secondary reboots were VIX SoftReboot actions outside Agent99; actor attribution was unavailable because process auditing was not enabled.",
|
||||
"evidence_refs": ["C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json", "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505", "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623/windows99-vmware-verify.txt"],
|
||||
"next_action": "Keep guest power unchanged, add trace-correlated VMware control auditing, and on the next approved drill verify that the startup task runs promptly after Windows boot and all five guests plus physical host 111 recover inside the same 600-second window."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-009",
|
||||
@@ -477,7 +489,7 @@
|
||||
"owner_lane": "edge_external_observability",
|
||||
"dependencies": ["AG99-P0-002", "AG99-P0-006"],
|
||||
"scope_refs": ["host:99", "all:hosts", "all:public_surfaces"],
|
||||
"problem": "The 188 edge maintenance fallback is live green, but host 99 still cannot detect its own outage and no independent off-LAN watcher has closed the down/up lifecycle.",
|
||||
"problem": "The real drill proved the external lane is absent: the local edge and product origins failed together, no external Agent received the outage, no verified out-of-band power callback existed and several public products exposed raw failure instead of an external maintenance origin. Host 99 later returned, but that does not close the off-LAN dependency.",
|
||||
"trigger": "External synthetic failure, local recovery incident or raw public 5xx threshold breach.",
|
||||
"ai_role": "External availability observer and maintenance-state controller",
|
||||
"action_mode": "controlled_apply",
|
||||
@@ -488,8 +500,9 @@
|
||||
"verifier": "External probe sees maintenance content instead of raw 502 during outage and normal routes after recovery.",
|
||||
"callback": "Correlate external outage with reboot incident and Telegram lifecycle card.",
|
||||
"acceptance": "Down, recovering and recovered remain observable outside the LAN, and public users never receive unmanaged 502 for a known full-stack recovery.",
|
||||
"implementation_progress": "188 root-owned edge helper, exact sudo commands, canonical/live hash parity, nginx verifier and rollback are live; six public routes are 200 and raw 5xx count is zero.",
|
||||
"next_action": "Provision and verify an independent watcher outside 99/110/188, then exercise down, maintenance and recovered callbacks during the real reboot drill."
|
||||
"implementation_progress": "The local 188 edge helper was green before reboot but is in the same failure domain. A repo-owned off-LAN supervisor state machine, callback allowlist contract, systemd unit/timer and synthetic transition tests are now source-ready; no external runtime host, power callback or maintenance callback is deployed yet.",
|
||||
"evidence_refs": ["docs/operations/agent99-full-host-reboot-drill-20260711.snapshot.json", "scripts/reboot-recovery/external-l0-recovery-supervisor.py", "ops/reboot-recovery/external-l0-recovery-supervisor.example.json"],
|
||||
"next_action": "Deploy the supervisor to an authorized off-LAN host, bind host-local Telegram, maintenance, OOB power and Agent99 resume wrappers, then verify down, recovering and recovered callbacks with the same incident id."
|
||||
},
|
||||
{
|
||||
"id": "AG99-P0-010",
|
||||
|
||||
@@ -0,0 +1,177 @@
|
||||
{
|
||||
"schema_version": "agent99_full_host_reboot_drill_receipt_v2",
|
||||
"run_id": "agent99-20260711-1559",
|
||||
"status": "slo_breached_recovered_late",
|
||||
"timezone": "Asia/Taipei",
|
||||
"source_runtime": {
|
||||
"pre_drill_agent99_source_revision": "7cf83f7f59a5d72ce6c303f59c144bd5bf8566df",
|
||||
"final_agent99_source_revision": "442b319bb80e009291ced7afa4f704cbe3b64886",
|
||||
"followup_source_commits": [
|
||||
"0a4ddb133e123d2d326984c932da978e0a38e77f",
|
||||
"442b319bb80e009291ced7afa4f704cbe3b64886",
|
||||
"801a50b017dd0fb975e675553075e67128202da2"
|
||||
],
|
||||
"runtime_manifest_matched": true,
|
||||
"runtime_file_count": 14,
|
||||
"runtime_mismatch_count": 0,
|
||||
"required_tasks_healthy": "9/9",
|
||||
"startup_recovery_execution_limit": "PT20M",
|
||||
"agent99_deploy_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-deploy-20260711-173023.json",
|
||||
"production_principal_recover_evidence": "C:\\Wooo\\Agent99\\evidence\\agent99-Recover-20260711-173543.json"
|
||||
},
|
||||
"preflight": {
|
||||
"all_required_hosts_reachable": true,
|
||||
"required_host_ids": [
|
||||
"99",
|
||||
"110",
|
||||
"111",
|
||||
"112",
|
||||
"120",
|
||||
"121",
|
||||
"188"
|
||||
],
|
||||
"core_routes_up": 6,
|
||||
"agent99_relay_ready": true,
|
||||
"agent99_queue_depth": 0
|
||||
},
|
||||
"timeline": {
|
||||
"observer_started_at": "2026-07-11T07:58:54Z",
|
||||
"first_service_route_down_at": "2026-07-11T07:59:51Z",
|
||||
"vm_guests_down_at": "2026-07-11T08:00:07Z",
|
||||
"windows99_observer_first_down_at": "2026-07-11T08:01:27Z",
|
||||
"windows99_actual_boot_at": "2026-07-11T16:08:03+08:00",
|
||||
"agent99_startup_recovery_started_at": "2026-07-11T16:08:57+08:00",
|
||||
"ten_minute_deadline_at": "2026-07-11T08:10:07Z",
|
||||
"agent99_startup_recovery_terminal": "terminated_by_pt10m_task_limit_before_terminal_breach_receipt",
|
||||
"vmware_autostart_task_ran_at": "2026-07-11T16:26:48+08:00",
|
||||
"host188_postgresql_ready_at": "2026-07-11T16:40:05+08:00",
|
||||
"host120_vix_soft_reboot_at": "2026-07-11T16:49:32+08:00",
|
||||
"host120_recovered_at": "2026-07-11T16:52:14+08:00",
|
||||
"host121_vix_soft_reboot_at": "2026-07-11T16:56:00+08:00",
|
||||
"host121_recovered_at": "2026-07-11T16:58:46+08:00",
|
||||
"core_routes_green_at": "2026-07-11T16:59:01+08:00",
|
||||
"signoz_bridge_green_at": "2026-07-11T17:02:00+08:00",
|
||||
"final_agent99_service_recovery_at": "2026-07-11T17:35:54+08:00",
|
||||
"all_public_surface_matrix_verified_at": "2026-07-11T17:37:00+08:00"
|
||||
},
|
||||
"observer_evidence": {
|
||||
"initial_events": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559/events.tsv",
|
||||
"continued_summary": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-1559-part2/summary.txt",
|
||||
"continued_host_down_samples": 567,
|
||||
"continued_host_down_samples_authoritative": false,
|
||||
"continued_host_probe_false_negative_cause": "curl_telnet_waited_for_ssh_payload_and_returned_nonzero_after_tcp_connect",
|
||||
"corrected_observer": "/tmp/awoooi-full-host-reboot-drill/agent99-20260711-observer-fixed",
|
||||
"continued_route_down_samples": 486,
|
||||
"continued_recovery_transitions": 0,
|
||||
"secret_value_read": false,
|
||||
"remote_write_performed": false
|
||||
},
|
||||
"recovery_evidence": {
|
||||
"latest_110_full_sop_artifact": "/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623",
|
||||
"post_start_result": "GREEN_WITH_EVIDENCE_WARNINGS",
|
||||
"service_green": true,
|
||||
"product_data_green": true,
|
||||
"stock_freshness_status": "ok",
|
||||
"backup_core_green": true,
|
||||
"host188_service_green": true,
|
||||
"host188_hygiene_green": true,
|
||||
"required_hosts_reachable": "7/7",
|
||||
"windows99_vmware_verify_ready": true,
|
||||
"windows99_vmware_running_guests": "5/5",
|
||||
"windows99_update_no_auto_reboot_ready": true,
|
||||
"scorecard_active_blockers": [
|
||||
"all_required_hosts_not_in_10_minute_reboot_window",
|
||||
"fresh_all_host_reboot_event_missing",
|
||||
"host_boot_observation_older_than_target_window"
|
||||
],
|
||||
"vmware_controlled_apply_evidence": "C:\\Wooo\\Agent99\\evidence\\windows99-vmware-autostart-apply-20260711-173505.json",
|
||||
"vmware_controlled_apply_backup": "C:\\Wooo\\Agent99\\deploy\\vmware-autostart-backup-20260711-173505",
|
||||
"agent99_service_recovery_elapsed_seconds": 10.3,
|
||||
"agent99_service_recovery_completed": true,
|
||||
"agent99_coordinator_verified": true,
|
||||
"agent99_reboot_slo_claimed": false,
|
||||
"public_routes_green": "23/23",
|
||||
"public_surface_results": [
|
||||
{"id": "awoooi-web", "url": "https://awoooi.wooo.work/", "http_status": 307},
|
||||
{"id": "awoooi-api", "url": "https://awoooi.wooo.work/api/v1/health", "http_status": 200},
|
||||
{"id": "iwooos", "url": "https://awoooi.wooo.work/zh-TW/iwooos", "http_status": 200},
|
||||
{"id": "vibework", "url": "https://vibework.wooo.work/", "http_status": 200},
|
||||
{"id": "awooogo", "url": "https://awooogo.wooo.work/", "http_status": 200},
|
||||
{"id": "2026fifa", "url": "https://2026fifa.wooo.work/", "http_status": 200},
|
||||
{"id": "agent", "url": "https://agent.wooo.work/", "http_status": 200},
|
||||
{"id": "momo", "url": "https://mo.wooo.work/", "http_status": 302},
|
||||
{"id": "momo-health", "url": "https://mo.wooo.work/health", "http_status": 200},
|
||||
{"id": "stock", "url": "https://stock.wooo.work/", "http_status": 200},
|
||||
{"id": "stock-health", "url": "https://stock.wooo.work/healthz", "http_status": 200},
|
||||
{"id": "stock-api", "url": "https://stock.wooo.work/api/healthz", "http_status": 200},
|
||||
{"id": "bitan", "url": "https://bitan.wooo.work/", "http_status": 200},
|
||||
{"id": "tsenyang", "url": "https://tsenyang.com/", "http_status": 200},
|
||||
{"id": "tsenyang-www", "url": "https://www.tsenyang.com/", "http_status": 200},
|
||||
{"id": "vtuber", "url": "https://vtuber.wooo.work/", "http_status": 200},
|
||||
{"id": "gitea", "url": "https://gitea.wooo.work/", "http_status": 200},
|
||||
{"id": "harbor", "url": "https://harbor.wooo.work/", "http_status": 200},
|
||||
{"id": "registry", "url": "https://registry.wooo.work/", "http_status": 200},
|
||||
{"id": "sentry", "url": "https://sentry.wooo.work/", "http_status": 302},
|
||||
{"id": "signoz", "url": "https://signoz.wooo.work/", "http_status": 200},
|
||||
{"id": "langfuse", "url": "https://langfuse.wooo.work/", "http_status": 200},
|
||||
{"id": "aiops", "url": "https://aiops.wooo.work/", "http_status": 307}
|
||||
]
|
||||
},
|
||||
"latest_readback": {
|
||||
"host_99": "up",
|
||||
"host_110": "up",
|
||||
"host_111": "up",
|
||||
"host_112": "up",
|
||||
"host_120": "up_k3s_active",
|
||||
"host_121": "up_k3s_active",
|
||||
"host_188": "up_startup_active",
|
||||
"awoooi_api_http_status": 200,
|
||||
"stock_freshness_http_status": 200,
|
||||
"momo_health_http_status": 200,
|
||||
"gitea_http_status": 200,
|
||||
"harbor_http_status": 200,
|
||||
"signoz_http_status": 200,
|
||||
"all_public_routes": "23/23_http_2xx_or_expected_3xx"
|
||||
},
|
||||
"bounded_recovery_attempts": {
|
||||
"windows99_wol_packets_sent": 36,
|
||||
"windows99_wol_result": "no_immediate_management_readback",
|
||||
"windows99_intel_amt_probe": "timeout",
|
||||
"macos111_wol_packets_sent": 12,
|
||||
"macos111_wol_result": "no_immediate_network_readback",
|
||||
"cloud_management_result": "no_authorized_external_executor_available"
|
||||
},
|
||||
"root_cause_classification": {
|
||||
"slo_result": "ten_minute_recovery_failed_even_though_windows99_booted_before_the_deadline",
|
||||
"primary_causes": [
|
||||
"windows99_vmware_autostart_task_did_not_run_until_18_minutes_45_seconds_after_windows_boot",
|
||||
"agent99_startup_recovery_was_terminated_by_its_pt10m_scheduled_task_limit",
|
||||
"agent99_boot_event_was_consumed_before_terminal_recovery_and_later_runs_lost_the_fresh_reboot_scope",
|
||||
"host188_postgresql_was_not_ready_until_16_40_05_and_k3s_120_121_waited_on_192_168_0_188_5432"
|
||||
],
|
||||
"observer_defect": "host TCP probe produced false negatives after switching from nc to curl telnet; route samples remain valid",
|
||||
"host120_121_secondary_reboots": "VMware VIX SoftReboot requests were issued from the Windows99 VMware control plane; no Agent99 or repo reset path exists and Windows process auditing was unavailable to attribute the actor",
|
||||
"public_experience_limit": "local edge fallback eventually worked for AWOOOI but Stock Gitea and Harbor initially exposed raw 502 because no external maintenance origin is deployed",
|
||||
"external_power_limit": "no verified off-LAN L0 supervisor or out-of-band power callback is deployed"
|
||||
},
|
||||
"required_closure": [
|
||||
"deploy_external_l0_supervisor_outside_the_local_failure_domain",
|
||||
"provision_verified_out_of_band_power_for_host99",
|
||||
"provision_external_maintenance_origin_or_cdn_failover",
|
||||
"audit_every_vmware_reset_stop_or_suspend_with_trace_id_and_verifier",
|
||||
"repeat_the_full_drill_and_close_within_600_seconds"
|
||||
],
|
||||
"claims_verified": [
|
||||
"all_required_hosts_recovered_late",
|
||||
"all_23_public_routes_are_currently_reachable",
|
||||
"agent99_runtime_is_integrated_with_the_full_sop_service_recovery_verifier",
|
||||
"windows99_vmware_autostart_and_update_policy_are_currently_green",
|
||||
"the_original_ten_minute_slo_breach_was_not_rewritten_as_success"
|
||||
],
|
||||
"claims_forbidden": [
|
||||
"all_services_recovered_within_ten_minutes_for_this_drill",
|
||||
"future_full_host_reboots_are_proven_to_recover_within_ten_minutes",
|
||||
"external_maintenance_failover_is_complete",
|
||||
"external_l0_power_recovery_is_deployed"
|
||||
]
|
||||
}
|
||||
@@ -1,9 +1,9 @@
|
||||
# AWOOOI 重開機恢復 SOP
|
||||
|
||||
> **版本**: v5.28
|
||||
> **版本**: v5.30
|
||||
> **最後更新**: 2026-07-11 (台北時間)
|
||||
> **更新者**: Codex
|
||||
> **觸發事件**: 2026-07-11 Agent99 Recover false-completion 與 full-SOP coordinator 整合
|
||||
> **觸發事件**: 2026-07-11 真實全主機 reboot drill、late recovery 與 Agent99/VMware/runtime 閉環
|
||||
|
||||
---
|
||||
|
||||
@@ -21,9 +21,27 @@
|
||||
|
||||
## 架構概覽與依賴圖
|
||||
|
||||
### 2026-07-11 真實 reboot drill：SLA breach 與 late recovery 真相
|
||||
|
||||
本輪不是模擬。重啟前 Agent99 runtime `7cf83f7f59a5d72ce6c303f59c144bd5bf8566df` 已 `14/14` matched、核心 task `9/9`，production-principal `Recover` baseline 已通過 full-SOP coordinator，且正確標示 `scope=service_recovery`、`rebootSloClaimed=false`。七主機與六核心路由在 observer 開始時均可達。
|
||||
|
||||
真實重啟後，`2026-07-11T08:00:07Z` 五台 VMware guests 與六核心路由離線，T+10 `08:10:07Z` 時服務仍未恢復，因此本輪 SLA 明確失敗。後續交叉比對 Windows boot time 證明 99 實際於 `16:08:03+08:00` 開機；舊 observer 將 host probe 從 `nc` 改成 `curl telnet://` 後，TCP 已連線仍等待 SSH payload 到 timeout，導致大量 host-down false negative。這些 host samples 不再作為「99 沒有回來」的證據，route-down samples 與 T+10 breach 仍有效；修正版 observer 已改用 Python `socket.create_connection`。
|
||||
|
||||
真正延遲鏈是：Agent99 Startup-Recovery 於 `16:08:57` 啟動，卻被原本 `PT10M` task limit 終止；boot event 又在 terminal recovery 前被消耗。VMware autostart task 直到 `16:26:48` 才執行，188 PostgreSQL 到 `16:40:05` 才 ready，120/121 K3s 在此之前持續等待 `192.168.0.188:5432`。120 與 121 又分別在 `16:49:32`、`16:56:00` 收到 Windows 99 VMware VIX `SoftReboot`；這不是 guest crash，也不是 unattended-upgrades。VMware process audit 未啟用，不能歸屬 actor；任何後續 reset/stop/suspend 都必須有同一 trace、before evidence、verifier 與 Telegram/KM receipt。
|
||||
|
||||
服務於 `16:59` 後恢復，SignOz upstream bridge 受控修復後 public route 也回 200。`17:35` production-principal Agent99 `Recover` 用 `10.3` 秒完成 service-recovery，七主機、VMware、Harbor、K3s、service/data/freshness/backup/188 hygiene/full-SOP coordinator 全通過，且 `rebootSloClaimed=false`；`17:37` 全部 23 個 public surfaces 為預期 2xx/3xx。機器可讀完整 receipt 在 `docs/operations/agent99-full-host-reboot-drill-20260711.snapshot.json`。
|
||||
|
||||
因此正式 SOP 固定為三層，不能再把 99 本機排程當成完整自動化：
|
||||
|
||||
1. **L0 外部層（99/LAN 之外）**：每 15 秒 public synthetic，連續兩輪 outage quorum 後建立 durable incident；執行繁中 Telegram down、外部 maintenance origin、具 verifier 的 99 out-of-band power。WOL 只可作第一個低風險嘗試，不是唯一電源路徑。
|
||||
2. **L1 Agent99/VMware 層**：99 回線後由 L0 投遞同一 incident；Agent99 驗證 Windows boot、runtime manifest、tasks、VMware service、VMX、lock，依序恢復並驗證 110/188/112/120/121。111 是獨立實體 Mac。
|
||||
3. **L2 110 full-SOP 層**：110 回線後建立 fresh scorecard，驗證七主機、service/data/freshness/backup、188 hygiene、edge/maintenance、VMware/update policy、版本與 callback。只有 blocker=0 且 `can_claim_slo=true` 才可撤維護頁與結案。
|
||||
|
||||
L0 source contract 是 `scripts/reboot-recovery/external-l0-recovery-supervisor.py`、`awoooi-external-l0-recovery-supervisor.service/.timer` 與 `ops/reboot-recovery/external-l0-recovery-supervisor.example.json`。Callback 只能位於 `/usr/local/libexec/awoooi-l0` allowlist，使用 direct exec 與 `--event-file`，禁止 arbitrary shell；Telegram/token、maintenance provider 與電源憑證留在外部 host-local wrapper，不寫入 repo/evidence。未部署外部 runtime、OOB power 與 maintenance callbacks前，正式狀態是 `slo_breached_recovered_late`：目前服務正常，不代表下一次全斷可在 10 分鐘內自動恢復。
|
||||
|
||||
### 2026-07-11 P0 Cold-start 實機收旂
|
||||
|
||||
最新完成 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-145616` 回報 `GREEN_WITH_EVIDENCE_WARNINGS`、readiness `87%`、service/data/backup/post-start/188 hygiene 皆綠。唯三 blockers 是 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`；它們只能在下一次真實全主機 reboot 後同一 10 分鐘 window 驗收，未取得執行當下的 break-glass 指令前不得為了清分數重啟。
|
||||
最新完成 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623` 回報 `GREEN_WITH_EVIDENCE_WARNINGS`、readiness `87%`、七主機 reachable、service/data/backup/post-start/188 hygiene 與 Windows99 VMware `5/5` 皆綠。唯三 blockers 是 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`；它們保留本輪逾時真相，只能由下一次真實全主機 reboot 的同一 10 分鐘 window 驗收，不能靠事後健康讀回清除。
|
||||
|
||||
- 99 VMware required guest set 是 `110/188/120/121/112`。111 是 M1 Pro MacBook Local Ollama，已用 110 public key + forced command 回傳 Darwin boot/uptime/Ollama status，不授權 shell。舊版把 111 當 Windows99 VMX 的敘述依 live inventory 標記為 superseded。
|
||||
- 110 runner fail-closed authority 只能使用 `flock` + `timeout` + `--ensure`，不得每分鐘無條件 mask + daemon-reload。完成條件是 apply/reload count 為 0 且 PID 1 CPU 收旂。
|
||||
@@ -47,7 +65,9 @@
|
||||
|
||||
部署前 gate 是：PowerShell parse、Agent99 contract、synthetic fresh/blocked fixture、runtime staging hash；部署後 gate 是：`scripts/reboot-recovery/agent99-live-preflight.ps1` 回 `9/9` tasks、relay listener、queue zero、required evidence fresh、runtime manifest matched，再由 production S4U principal 觸發 `Recover`。Administrator 直接跑 launcher 不能取代 production-principal verifier。
|
||||
|
||||
2026-07-11 15:35 source receipt：Python coordinator contract `6 passed`、Agent99 API contract `45 passed`、reboot-recovery suite `131 passed`；Windows staged validation 與 PowerShell synthetic `12/12` green。99 現行 live runtime 仍是 `34e32f6e...`，新 coordinator 尚待原子 promote，完成前不得說 Agent99 已整合 full SOP。
|
||||
2026-07-11 17:35 final runtime receipt：Agent99 已原子 promote 為 `442b319bb80e009291ced7afa4f704cbe3b64886`，deploy evidence `C:\Wooo\Agent99\evidence\agent99-deploy-20260711-173023.json`；runtime manifest `14/14`、mismatch `0`、九個 task 存在、SRE relay 監聽、Startup-Recovery limit 為 `PT20M`。`0a4ddb133` 讓 reboot event 保持 `pendingRecovery` 直到 terminal、支援 `slo_breached_recovery_pending/recovered_late/recovered_within_slo`，並修正 observer TCP probe；`442b319bb` 讓 scorecard 接受 principal-aware VM power provenance。production-principal evidence `C:\Wooo\Agent99\evidence\agent99-Recover-20260711-173543.json` 顯示 `scope=service_recovery`、completed/verified true、elapsed `10.3s`、`rebootSloClaimed=false`。
|
||||
|
||||
Windows 99 VMware task 已經由 `scripts/reboot-recovery/windows99-vmware-autostart-controlled-apply.ps1` 正式受控套用：DryRun、Apply、Verify 同一 receipt；task XML、舊 startup script、Windows Update policy 與 service start modes 都有 rollback。Evidence `C:\Wooo\Agent99\evidence\windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process 都是 `5`、running guest rows `5`、process-aware duplicate-start guard true、沒有 host reboot、沒有 VM power change、沒有 secret read。這才是 Agent99 與重啟 SOP 的 runtime 整合證據；仍不取代下一次 600 秒 E2E drill。
|
||||
|
||||
### 2026-07-10 P0 Agent99 Transport / Verified Remediation 覆蓋
|
||||
|
||||
@@ -861,6 +881,8 @@ echo "=== 結果: ${PASS} 通過, ${FAIL} 失敗 ==="
|
||||
|
||||
| 版本 | 日期 | 說明 |
|
||||
|------|------|------|
|
||||
| v5.30 | 2026-07-11 | 交叉驗證 99 實際 boot time，修正 observer false negative；部署 durable reboot state、PT20M task、principal-aware VM verifier/parser、VMware controlled apply，並以 production-principal Recover 與 23/23 public matrix 證明 late recovery |
|
||||
| v5.29 | 2026-07-11 | 初始 observer 將 99 判為未回電；此判讀已由 v5.30 的 Windows boot/readback 與 socket probe evidence supersede，T+10 breach 本身仍有效 |
|
||||
| v1.0 | 2026-04-04 | 初版：containerd + Docker + PostgreSQL WAL 修復流程 |
|
||||
| v2.0 | 2026-04-05 上午 | 188 啟動腳本 + 110 啟動腳本 + Harbor race condition 修復 |
|
||||
| v3.0 | 2026-04-05 下午 | 完整架構重盤 + Gitea Runner 自動化 + 告警鏈路根因修復 + NetworkPolicy 修正 |
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Agent99 全域 AI 自動化主計畫
|
||||
|
||||
> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 15:35 CST
|
||||
> 建立時間：2026-07-10 15:05 CST；最近更新：2026-07-11 17:40 CST
|
||||
> 狀態：P0 執行中，尚未達成全域 AI 自動化
|
||||
> 機器可讀總帳：`docs/operations/agent99-enterprise-ai-automation-work-items.snapshot.json`
|
||||
> 歷史需求來源：`docs/workplans/2026-07-02-commander-inserted-requirements-priority-ledger.md`
|
||||
@@ -68,6 +68,16 @@
|
||||
| KM/RAG/MCP/PlayBook | 可重播 evidence、知識、工具介面、已驗證修復與 trust/recurrence 統計 |
|
||||
| Telegram SRE 戰情室 | 人類命令與事件摘要介面，不是資料庫、任意 shell 或唯一事件來源 |
|
||||
|
||||
### 3.1 重啟恢復三層控制契約
|
||||
|
||||
全主機重啟不能只靠 99 上的 Agent99。正式恢復鏈固定拆成三個失效域，並以同一 `incident_id/run_id` 串接：
|
||||
|
||||
1. **L0 外部存活與電源層**：部署在 LAN、99、110、188 之外，持續探測公開路由；連續兩輪達 outage quorum 後建立 incident，依 allowlist 執行繁中 Telegram down 卡、外部維護 origin、99 out-of-band power。它必須在本地系統全斷時仍可運作。
|
||||
2. **L1 Agent99 與 VMware 層**：99 回線後，L0 將 durable incident 投遞至 Agent99；Agent99 先驗證 Windows boot、Task Scheduler、VMware service/VMX/lock，再依序啟動並驗證 110/188/112/120/121。111 是獨立實體 Mac，不走 VMware。
|
||||
3. **L2 110 full-SOP 層**：110 回線後產生 fresh scorecard，驗證七主機、K3s/Docker/Harbor/Gitea、產品路由、資料 freshness、備份、188 hygiene、maintenance、版本與 callback。只有同一 reboot window 的 blocker=0 且 `can_claim_slo=true` 才能結案。
|
||||
|
||||
`Wake-on-LAN` 只能是 L0 的第一個低風險嘗試，不能當唯一遠端電源方案。正式 closure 需要可讀回的 BMC/IPMI/AMT、受控 PDU/智慧插座，或其他具 power-state verifier 的獨立電源控制；同時要在韌體啟用 AC power restore 並實測。
|
||||
|
||||
## 4. 標準 AI 閉環
|
||||
|
||||
所有告警與指令必須走同一條鏈：
|
||||
@@ -104,14 +114,14 @@
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 5 | AG99-P0-005 | 模式專屬 verifier、真 rollback、cooldown/circuit-breaker | 待辦 | Recover/Perf/LoadShed/Harbor/AWOOOI/backup/provider 各有 post-condition；失敗不寫成功 cooldown |
|
||||
| 6 | AG99-P0-006 | Agent99 → AWOOOI/AwoooP/IWOOOS completion callback | 待辦 | incident/Run/Work Item/Verifier/recurrence 同一 trace 收斂；Telegram 可讀回報只從結果物件產生 |
|
||||
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中，current P0；Agent99 coordinator source/test 綠、99 runtime 待 promote** | boot_id/uptime/VM power 建立 reboot event；Agent99 必須關聯 110 同輪 full-SOP scorecard；首可用/全綠時間與 blocker/ETA 可查 |
|
||||
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **baseline live green；真實 reboot 驗收併入 P0-007** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green；111 由實體 Mac readback 驗證 |
|
||||
| 7 | AG99-P0-007 | 全主機 10 分鐘 reboot state machine | **進行中，current P0；本輪 breach 後已 late recovery，L1/L2 runtime 已整合，L0/600 秒重測待完成** | L0 off-LAN detection/OOB power、L1 Agent99/VMware、L2 110 full-SOP 必須以同一 incident 閉環；首可用/全綠時間與 blocker/ETA 可查 |
|
||||
| 8 | AG99-P0-008 | 99 VMware/VMX/autostart/lock 與 Windows Update no-auto-reboot | **本次 boot runtime green；controlled apply/rollback/verifier 完成，待下次 reboot 驗證 startup 時序** | 110/188/112/120/121 VMX、autostart task、guest running 與 no-auto-reboot verifier green；111 由實體 Mac readback 驗證 |
|
||||
|
||||
### Wave 2：擴至所有產品與運維面
|
||||
|
||||
| 順序 | ID | 工作 | 狀態 | 驗收 |
|
||||
| --- | --- | --- | --- | --- |
|
||||
| 9 | AG99-P0-009 | 外部 down/up watcher、L0/L1 維護頁與 502 failover | 進行中 | 99/內網全斷仍能告警；edge 失聯自動顯示維護頁；恢復後撤回並驗證，不改正式 DNS 作為日常修復 |
|
||||
| 9 | AG99-P0-009 | 外部 down/up watcher、L0/L1 維護頁與 502 failover | **進行中；真實 drill 已證明 runtime 缺口，supervisor source 已建立** | 99/內網全斷仍能告警及受控 power restore；edge 失聯自動顯示外部維護頁；恢復後投遞 Agent99、撤回並驗證，不用臨時 DNS mutation 當日常修復 |
|
||||
| 10 | AG99-P0-010 | 12 產品、全部網站/API 的版本與資料 freshness 矩陣 | 進行中 | 每列具 source SHA、CD、deploy marker、runtime、HTTP、功能 smoke、data timestamp；任何缺欄不宣稱最新 |
|
||||
| 11 | AG99-P0-011 | 全域 log source registry、AI 異常判讀與 correlation | 待辦 | host/product/site/service/package/tool logs 有 freshness、redaction、tag、cluster、primary/secondary/noise 判定 |
|
||||
| 12 | AG99-P0-012 | 備份 observability、integrity、offsite 與 restore drill | 進行中；Gitea/Velero/110-188 slice live green | 所有 backup domain 有 schedule/last success/freshness/restore receipt；失敗可定位且 TG 只報一次狀態變化 |
|
||||
@@ -173,6 +183,16 @@
|
||||
- 110 最新 direct artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-151841` 顯示 7 host reachable、`GREEN_WITH_EVIDENCE_WARNINGS`、service/data/backup/188 hygiene/edge/VMware green；目前唯三 scorecard blockers 是尚未發生新一輪全主機 reboot window。Production API 仍混入舊 111 snapshot，因此 Agent99 verifier 固定以 110 direct artifact 優先，API 只作次級投影。
|
||||
- current P0 改為 `AG99-P0-007`。下一步固定為：建立 local commit、原子 promote 到 99、跑 production-principal `Recover` baseline 與 live preflight、確認舊五階段不能再單獨 resolved，然後執行已獲批准的真實全主機 reboot drill。main push 仍服從單一 writer slot。
|
||||
|
||||
### 2026-07-11 17:40 真實全主機 reboot drill 與 Agent99 整合結果
|
||||
|
||||
- 本輪 T+10 `08:10:07Z` 仍是明確 breach，但後續 live truth 證明 99 已在 `16:08:03+08:00` 開機。舊 observer 使用 `curl telnet://` 等待 SSH payload，造成 host false negative；修正版改用 socket connect。不得再寫「99 沒有回來」，也不得因此抹掉服務逾時。
|
||||
- Agent99 Startup-Recovery 在 `16:08:57` 執行，卻被 `PT10M` task limit 終止且 boot event 被提早消耗；VMware task 到 `16:26:48` 才執行。188 PostgreSQL `16:40:05` ready 後 K3s 才能收斂。120/121 的第二次 reboot 由 VMware log 證實是外部 VIX `SoftReboot`，不是 crash 或 update；因 Windows process audit 不可用，actor 不可歸屬，後續 reset/stop/suspend 必須有 trace/receipt。
|
||||
- `0a4ddb133` 已補 durable `pendingRecovery`、late-recovery terminal state、Startup-Recovery `PT20M` 與 socket observer；`442b319bb` 修正 scorecard 解析 principal-aware VM power provenance；`801a50b01` 新增正式 VMware controlled-apply wrapper，含 DryRun、task/script/policy/service backup、Apply、independent Verify 與 rollback。
|
||||
- Windows 99 runtime 已部署 `442b319bb80e009291ced7afa4f704cbe3b64886`：manifest `14/14`、mismatch `0`、task `9/9`、SRE relay listening、Startup-Recovery `PT20M`。VMware controlled receipt `windows99-vmware-autostart-apply-20260711-173505.json` 回報 pre/post VMX process `5/5`、五 guests running、process-aware duplicate-start guard true、沒有 reboot/VM power change/secret read。
|
||||
- 110 artifact `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-172623` 顯示七主機 reachable、service/data/freshness/backup/188 hygiene/edge/VMware 全綠；只保留本輪時間窗三個 blocker。production-principal `agent99-Recover-20260711-173543.json` 在 `10.3s` 完成 service recovery、coordinator verified，且 `rebootSloClaimed=false`。
|
||||
- 全部 23 個 public surfaces 已從 110 驗證為預期 2xx/3xx，SignOz bridge 也已受控恢復。這證明「目前已恢復」與「Agent99 已整合 full SOP」，不證明下一次全主機重啟會在 600 秒內完成。
|
||||
- L0 external supervisor、可驗證 OOB power 與外部 maintenance origin 仍只到 source contract，未有 production runtime receipt。current P0 因此仍是 `AG99-P0-007`；下一次 drill 只有在 L0/L1/L2 使用同一 incident 且 600 秒內 blocker=0 才能 terminal complete。
|
||||
|
||||
## 7. 工作項強制欄位與治理
|
||||
|
||||
每個工作項必須先有：`id`、priority/order/status、scope、owner lane、dependencies、acceptance 與 next action。任何工作項升到 `candidate_ready`、`in_progress` 或 `executing` 前，還必須補齊 problem、trigger、evidence sources、AI role、action mode、controlled actions、prohibited actions、rollback、mode-specific verifier、callback、evidence refs 與 last update；缺一不可執行。
|
||||
@@ -188,4 +208,4 @@
|
||||
|
||||
## 8. 立即下一步
|
||||
|
||||
目前唯一 P0 是 `AG99-P0-007`。先把已測試的 `recoveryCoordinator` 以 local commit SHA 原子 promote 到 99，驗證 runtime manifest、9/9 tasks、production-principal Recover evidence 與 full-SOP coordinator outcome；再執行一次真實全主機 reboot drill，要求 T+0 偵測、T+10 前 VM/host/service/route/data/backup/version/maintenance/verifier 同輪收斂。P0-007 terminal 後依序回到 `AG99-P0-003 secure ingress → P0-004 EvidenceCollect/RepairCandidate → P0-005 mode verifier/rollback → P0-006 callback`，再推外部 watcher、全產品版本矩陣、logs、backup、performance 與 Gitea/Harbor DR。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不得用推送協調打斷 runtime 主線。
|
||||
目前唯一 P0 仍是 `AG99-P0-007`，不另拆支線。L1 Agent99/VMware 與 L2 110 full-SOP 已有 runtime receipts；下一步只收斂剩餘 L0：把 external supervisor 部署到 99/110/188/LAN 之外，接上可驗證 OOB power、外部 maintenance、durable incident 與 Agent99 resume callback，再用同一 incident 做 600 秒 E2E 重測。之後依序前進：`P0-008 → P0-009 → P0-010 → P0-003 → P0-004 → P0-005 → P0-006 → P0-011 → P0-012 → P0-013 → P0-014`。main push 只在唯一 writer slot 進行；沒有 slot 時保留 local commit，不讓推送協調打斷 runtime 主線。
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[3]
|
||||
RECEIPT = ROOT / "docs" / "operations" / "agent99-full-host-reboot-drill-20260711.snapshot.json"
|
||||
|
||||
|
||||
def test_real_drill_receipt_preserves_breach_and_late_recovery_truth() -> None:
|
||||
payload = json.loads(RECEIPT.read_text(encoding="utf-8"))
|
||||
|
||||
assert payload["status"] == "slo_breached_recovered_late"
|
||||
assert payload["observer_evidence"]["continued_recovery_transitions"] == 0
|
||||
assert payload["observer_evidence"]["continued_host_down_samples_authoritative"] is False
|
||||
assert payload["latest_readback"]["host_99"] == "up"
|
||||
assert payload["latest_readback"]["all_public_routes"].startswith("23/23")
|
||||
assert payload["recovery_evidence"]["windows99_vmware_running_guests"] == "5/5"
|
||||
public_results = payload["recovery_evidence"]["public_surface_results"]
|
||||
assert len(public_results) == 23
|
||||
assert all(200 <= row["http_status"] < 400 for row in public_results)
|
||||
assert payload["recovery_evidence"]["agent99_reboot_slo_claimed"] is False
|
||||
assert "deploy_external_l0_supervisor_outside_the_local_failure_domain" in payload["required_closure"]
|
||||
assert "provision_verified_out_of_band_power_for_host99" in payload["required_closure"]
|
||||
assert "all_services_recovered_within_ten_minutes_for_this_drill" in payload["claims_forbidden"]
|
||||
assert payload["observer_evidence"]["secret_value_read"] is False
|
||||
|
||||
|
||||
def test_real_drill_receipt_records_the_repaired_agent99_and_vmware_contracts() -> None:
|
||||
payload = json.loads(RECEIPT.read_text(encoding="utf-8"))
|
||||
|
||||
assert payload["source_runtime"]["runtime_mismatch_count"] == 0
|
||||
assert payload["source_runtime"]["startup_recovery_execution_limit"] == "PT20M"
|
||||
assert payload["recovery_evidence"]["agent99_coordinator_verified"] is True
|
||||
assert payload["root_cause_classification"]["host120_121_secondary_reboots"].startswith(
|
||||
"VMware VIX SoftReboot"
|
||||
)
|
||||
Reference in New Issue
Block a user