From 43a3a06a3af456a5ae6d73aae3c486a2d1e010d3 Mon Sep 17 00:00:00 2001 From: ogt Date: Sat, 11 Jul 2026 20:00:16 +0800 Subject: [PATCH] feat(recovery): close host112 guest readiness loop --- agent99-control-plane.ps1 | 122 +++++++++++++++++- agent99-sre-alert-inbox.ps1 | 23 +++- agent99-telegram-inbox.ps1 | 15 ++- apps/api/src/services/agent99_sre_bridge.py | 14 ++ ...t_agent99_alert_dispatch_order_contract.py | 3 + ..._agent99_control_plane_outcome_contract.py | 2 + apps/api/tests/test_agent99_sre_bridge.py | 26 ++++ ...nt99_transport_recovery_deploy_contract.py | 18 +++ 8 files changed, 217 insertions(+), 6 deletions(-) diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 2223b433b..002ff49ce 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2975,12 +2975,14 @@ function Get-AgentOutcomeContract { $publicRows = @($data.public) $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0) + $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk $checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false - $modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk) + $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) + $modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk) } "Recover" { $expectedVms = @($Config.vms).Count @@ -2994,13 +2996,15 @@ function Get-AgentOutcomeContract { $publicRows = @($data.public) $publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0) $coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified) + $host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified) $checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)" $checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)" $checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk $checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk $checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk + $checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ',')) $checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ',')) - $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) + $modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk) } "StartVMs" { $expectedVms = @($Config.vms).Count @@ -4087,6 +4091,107 @@ function Invoke-AgentProviderFreshnessTriage { $candidate } +function Convert-AgentHost112GuestReadback { + param([object]$Transport) + + $values = @{} + $text = if ($Transport -and $Transport.PSObject.Properties["output"]) { [string]$Transport.output } else { "" } + foreach ($match in [regex]::Matches($text, '(?:^|\s)([A-Za-z0-9_]+)=([^\s]+)')) { + $values[$match.Groups[1].Value] = $match.Groups[2].Value + } + $get = { + param([string]$Name, [string]$Default = "unknown") + if ($values.ContainsKey($Name)) { return [string]$values[$Name] } + return $Default + } + $asBool = { + param([string]$Name) + [bool]((& $get $Name "0") -eq "1") + } + $checks = @( + [pscustomobject]@{ name = "readback_present"; passed = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) }, + [pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") }, + [pscustomobject]@{ name = "console_ready"; passed = (& $asBool "console_ready") }, + [pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") }, + [pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") }, + [pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") } + ) + $failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name }) + [pscustomobject]@{ + schemaVersion = "agent99_host112_guest_readback_v1" + readbackPresent = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) + ready = [bool]($failedChecks.Count -eq 0) + bootId = & $get "boot_id" + uptimeSeconds = & $get "uptime_seconds" "-1" + systemdState = & $get "systemd_state" + defaultTarget = & $get "default_target" + graphicalTarget = & $get "graphical_target" + displayManager = & $get "display_manager" + lightdm = & $get "lightdm" + vmwareTools = & $get "vmware_tools" + xorg = & $get "xorg" + wazuhIndexer = & $get "wazuh_indexer" + wazuhManager = & $get "wazuh_manager" + wazuhDashboard = & $get "wazuh_dashboard" + filebeat = & $get "filebeat" + consoleReady = (& $asBool "console_ready") + servicesReady = (& $asBool "services_ready") + recoveryTimerReady = (& $asBool "recovery_timer_ready") + guestReady = (& $asBool "guest_ready") + actionCount = & $get "action_count" "0" + actions = & $get "actions" "none" + transportOk = [bool]($Transport -and $Transport.ok) + transportExitCode = if ($Transport) { $Transport.exitCode } else { -1 } + transportRoute = if ($Transport -and $Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "unknown" } + transportSerialized = [bool]($Transport -and $Transport.PSObject.Properties["transportSerialized"] -and $Transport.transportSerialized) + transportLockWaitMs = if ($Transport -and $Transport.PSObject.Properties["transportLockWaitMs"]) { $Transport.transportLockWaitMs } else { 0 } + checks = $checks + failedChecks = $failedChecks + } +} + +function Invoke-AgentHost112GuestRecovery { + $targetHost = "192.168.0.112" + $checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check" + $applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply" + $beforeTransport = Invoke-HostSshText $targetHost $checkCommand 20 1 + $before = Convert-AgentHost112GuestReadback $beforeTransport + $applyTransport = $null + $after = $before + $applyAttempted = $false + + if (-not $before.ready -and $ControlledApply) { + $applyAttempted = $true + Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery failedChecks=$($before.failedChecks -join ',')" + $applyTransport = Invoke-HostSshText $targetHost $applyCommand 300 1 + $afterTransport = Invoke-HostSshText $targetHost $checkCommand 20 1 + $after = Convert-AgentHost112GuestReadback $afterTransport + } + + $verified = [bool]$after.ready + $changed = [bool]($applyAttempted -and $verified -and -not $before.ready) + $result = [pscustomobject]@{ + schemaVersion = "agent99_host112_guest_recovery_v1" + targetHost = $targetHost + controlledApply = [bool]$ControlledApply + applyAttempted = $applyAttempted + applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null } + changed = $changed + verified = $verified + before = $before + after = $after + rollback = "systemd package backup under /var/lib/awoooi-host112-recovery/backups" + verifier = "forced_command_host112_guest_readiness_check" + prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write") + } + if ($changed) { + Record-AgentEvent "host112_guest_recovered" "info" "112 圖形登入、VMware Tools、Wazuh 與 recovery timer 已由 Agent99 修復並驗證。" $result -Alert + } elseif ($applyAttempted -and -not $verified) { + Record-AgentEvent "host112_guest_recovery_failed" "critical" "112 受控修復後仍未通過 verifier；保留 failed checks 與 rollback evidence。" $result -Alert + } + $result +} + function Test-HostReachability { $results = @() foreach ($hostIp in $Config.hosts) { @@ -4366,6 +4471,7 @@ Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledAp $vmResults = @() $hostResults = @() +$host112Recovery = $null $harbor = $null $awooo = $null $public = @() @@ -4419,6 +4525,15 @@ if ($Mode -in @("Status", "Recover")) { } } +if ($Mode -in @("Status", "Recover")) { + $host112Recovery = Invoke-AgentHost112GuestRecovery + if ($Mode -eq "Recover") { + $recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds) + } elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) { + $recoveryTrigger = Start-AgentRecoveryFromObservation "host112_guest_not_ready" $host112Recovery.after + } +} + if ($Mode -in @("Status", "Recover", "HarborRepair")) { $harbor = Get-HarborState } @@ -4537,7 +4652,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit if ($Mode -eq "Recover") { $elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1) $unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count - $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) + $builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0) $coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified) $requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow) $rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed) @@ -4587,6 +4702,7 @@ $result = [pscustomobject]@{ recoveryCoordinator = $recoveryCoordinator vmResults = $vmResults hosts = $hostResults + host112Recovery = $host112Recovery harbor = $harbor awoooi = $awooo public = $public diff --git a/agent99-sre-alert-inbox.ps1 b/agent99-sre-alert-inbox.ps1 index eeef17a9b..2fba0304d 100644 --- a/agent99-sre-alert-inbox.ps1 +++ b/agent99-sre-alert-inbox.ps1 @@ -84,7 +84,7 @@ function Test-AgentAlertActionable { if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) { return $true } - if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") { + if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") { return $true } return $false @@ -113,6 +113,9 @@ function Resolve-AgentAlertRoute { "host_down" = "Recover" "host_reboot" = "Recover" "vm_not_running" = "Recover" + "host112_guest_recovery" = "Recover" + "host112_guest_not_ready" = "Recover" + "host112_guest_readback_missing" = "Recover" "performance_pressure" = "Perf" "performance_cpu" = "Perf" "performance_memory" = "Perf" @@ -132,7 +135,7 @@ function Resolve-AgentAlertRoute { if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" } } - if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") { + if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup|host112_guest|host 112.*(?:guest|console|lightdm)") { return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" } } if ($text -match "backup|snapshot|restore drill|cron") { @@ -301,6 +304,22 @@ function Invoke-AgentAlertRoutingSelfTest { message = "host_reboot detected vm_not_running" } }, + [pscustomobject]@{ + name = "host 112 guest recovery" + expectedActionable = $true + expectedMode = "Recover" + expectedRouteSource = "kind" + expectedControlledApply = $true + alert = [pscustomobject]@{ + id = "selftest-host112-guest" + source = "agent99-routing-selftest" + severity = "critical" + kind = "host112_guest_recovery" + service = "host112-guest-recovery" + host = "192.168.0.112" + message = "Host 112 Console LightDM VMware Tools and Wazuh not ready" + } + }, [pscustomobject]@{ name = "performance pressure" expectedActionable = $true diff --git a/agent99-telegram-inbox.ps1 b/agent99-telegram-inbox.ps1 index dbdbedd69..d023a1238 100644 --- a/agent99-telegram-inbox.ps1 +++ b/agent99-telegram-inbox.ps1 @@ -292,6 +292,7 @@ function Get-AgentTelegramAlertSeverity { function Get-AgentTelegramAlertKind { param([string]$Text) $lower = $Text.ToLowerInvariant() + if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112_guest_recovery" } if ($lower -match "\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return "cicd_failure" } if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" } if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" } @@ -313,6 +314,7 @@ function Get-AgentTelegramAlertKind { function Get-AgentTelegramAlertService { param([string]$Text) $lower = $Text.ToLowerInvariant() + if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112-guest-recovery" } if ($lower -match "stock") { return "stockplatform" } if ($lower -match "2026fifa|fifa") { return "2026fifa" } if ($lower -match "bitan" -or (Test-AgentTextContainsCode $Text @(0x78A7, 0x6F6D))) { return "bitan-pharmacy" } @@ -348,7 +350,7 @@ function Get-AgentTelegramAlertTarget { return $targetMatch.Groups[1].Value.Trim() } if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" } - if ($Kind -eq "host_recovery" -and $HostName) { return $HostName } + if ($Kind -in @("host_recovery", "host112_guest_recovery") -and $HostName) { return $HostName } if ($Service -and $Service -ne "unknown") { return $Service } return $Kind } @@ -361,6 +363,7 @@ function Get-AgentTelegramSuggestedMode { "wazuh_security_alert" = "SecurityTriage" "public_route_502" = "Recover" "host_recovery" = "Recover" + "host112_guest_recovery" = "Recover" "performance_cpu" = "Perf" "performance_memory" = "Perf" "performance_disk" = "Perf" @@ -383,6 +386,7 @@ function Get-AgentTelegramSourceEventResolutionPolicy { "backup_health", "public_route_502", "host_recovery", + "host112_guest_recovery", "performance_cpu", "performance_memory", "performance_disk", @@ -566,6 +570,15 @@ function Invoke-AgentTelegramInboxSelfTest { expectedService = "bitan-pharmacy" expectedHost = "" }, + [pscustomobject]@{ + name = "host 112 guest readiness" + text = "critical Host112GuestNotReady host 192.168.0.112 Console LightDM VMware Tools Wazuh not ready" + expectedMonitoring = $true + expectedSeverity = "critical" + expectedKind = "host112_guest_recovery" + expectedService = "host112-guest-recovery" + expectedHost = "192.168.0.112" + }, [pscustomobject]@{ name = "security triage" text = "Wazuh SIEM critical security alert auth_failed brute force suspicious login host 192.168.0.112" diff --git a/apps/api/src/services/agent99_sre_bridge.py b/apps/api/src/services/agent99_sre_bridge.py index 8484817e9..1c733dd79 100644 --- a/apps/api/src/services/agent99_sre_bridge.py +++ b/apps/api/src/services/agent99_sre_bridge.py @@ -95,6 +95,12 @@ def resolve_agent99_alert_kind( annotations=annotations or {}, ) + if re.search( + r"host112guest(?:readbackmissing|notready)|host112_guest|" + r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)", + text, + ): + return "host112_guest_recovery" if re.search( r"provider_freshness_signal|source_provider_freshness_triage|" r"raw_signal_normalized|controlled_runtime_candidate|freshness|" @@ -168,6 +174,7 @@ def _suggested_mode_for_kind(kind: str) -> str | None: "harbor_registry": "HarborRepair", "awoooi_k3s": "AwoooRepair", "host_recovery": "Recover", + "host112_guest_recovery": "Recover", "application_ai_config_error": "Status", "cicd_failure": "Status", "monitoring_alert": "Status", @@ -183,6 +190,7 @@ def _resolution_policy_for_kind(kind: str) -> str: "harbor_registry", "awoooi_k3s", "host_recovery", + "host112_guest_recovery", }: return "mode_verifier_can_resolve" return "external_source_receipt_required" @@ -335,6 +343,12 @@ def build_agent99_sre_alert( "directReference,verifierReadback; mark missing,expired,timeout," "no_false_green; do not switch provider, call paid model, edit env, or reload." ) + elif kind == "host112_guest_recovery": + instruction = ( + "Run the allowlisted host 112 guest recovery; verify graphical.target, " + "LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot " + "the host, change VM power, reset the VM, read secrets or change firewall." + ) payload: dict[str, Any] = { "id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}", diff --git a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py index b3b292687..59881bfe8 100644 --- a/apps/api/tests/test_agent99_alert_dispatch_order_contract.py +++ b/apps/api/tests/test_agent99_alert_dispatch_order_contract.py @@ -41,6 +41,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None: assert '"database_health" = "Status"' in source assert '"alert_chain_health" = "Status"' in source assert '"host_recovery" = "Recover"' in source + assert '"host112_guest_recovery" = "Recover"' in source assert '"cicd_failure" = "Status"' in source assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source @@ -48,6 +49,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None: telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8") assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source assert '"cicd_failure" = "Status"' in telegram_source + assert '"host112_guest_recovery" = "Recover"' in telegram_source assert 'policy = $sourceEventResolutionPolicy' in telegram_source @@ -69,3 +71,4 @@ def test_telegram_ingress_writes_same_structured_route_contract() -> None: assert 'routeSource = "telegram_structured_kind"' in source assert '"database_health" = "Status"' in source assert '"alert_chain_health" = "Status"' in source + assert '"host112_guest_recovery" = "Recover"' in source diff --git a/apps/api/tests/test_agent99_control_plane_outcome_contract.py b/apps/api/tests/test_agent99_control_plane_outcome_contract.py index 0dc45d3a2..01d07e138 100644 --- a/apps/api/tests/test_agent99_control_plane_outcome_contract.py +++ b/apps/api/tests/test_agent99_control_plane_outcome_contract.py @@ -19,6 +19,8 @@ def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None: assert '"verifying"' in source assert '"candidate_ready"' in source assert "ok = [bool]($exitCode -eq 0)" not in source + assert 'New-AgentOutcomeCheck "host112_guest_ready" $host112Ok' in source + assert "$data.host112Recovery -and $data.host112Recovery.verified" in source def test_agent99_problem_counts_separate_verified_resolution() -> None: diff --git a/apps/api/tests/test_agent99_sre_bridge.py b/apps/api/tests/test_agent99_sre_bridge.py index f1e0580f4..8cce78a4d 100644 --- a/apps/api/tests/test_agent99_sre_bridge.py +++ b/apps/api/tests/test_agent99_sre_bridge.py @@ -105,6 +105,14 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None: "host_recovery", "Recover", ), + ( + "Host112GuestNotReady", + "host112-guest-recovery", + "Host 112 console LightDM VMware Tools and Wazuh are not ready", + {"host": "112", "service": "host112-guest-recovery"}, + "host112_guest_recovery", + "Recover", + ), ( "KubePodCrashLooping", "awoooi-api", @@ -176,6 +184,24 @@ def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts() assert cicd["resolution"]["policy"] == "external_source_receipt_required" +def test_host112_guest_recovery_precedes_wazuh_security_classification() -> None: + payload = build_agent99_sre_alert( + alert_id="host112-guest-not-ready", + alertname="Host112GuestNotReady", + severity="critical", + namespace="host112", + target_resource="host112-guest-recovery", + message="Host 112 Console VMware Tools Wazuh services are not ready", + labels={"host": "112", "service": "host112-guest-recovery"}, + ) + + assert payload["kind"] == "host112_guest_recovery" + assert payload["suggestedMode"] == "Recover" + assert payload["controlledApply"] is True + assert payload["resolution"]["policy"] == "mode_verifier_can_resolve" + assert "do not reboot" in payload["instruction"] + + def test_alert_kind_mapping_for_route_and_performance() -> None: assert ( resolve_agent99_alert_kind( diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 8f3f17666..0e4763a7f 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -37,6 +37,24 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None: assert "Start-Process" not in trigger assert 'Record-AgentEvent "recovery_slo_result"' in source assert "withinSlo = $withinSlo" in source + assert "function Invoke-AgentHost112GuestRecovery" in source + assert 'Start-AgentRecoveryFromObservation "host112_guest_not_ready"' in source + assert 'New-AgentRecoveryPhase "host112_guest_readiness"' in source + assert '$host112Recovery -and $host112Recovery.verified' in source + + +def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None: + source = CONTROL.read_text(encoding="utf-8") + function = source[source.index("function Convert-AgentHost112GuestReadback") :] + function = function[: function.index("function Test-HostReachability")] + + assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function + assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function + assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2 + assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function + assert 'verifier = "forced_command_host112_guest_readiness_check"' in function + for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"): + assert f'"{forbidden}"' in function def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None: