feat(recovery): close host112 guest readiness loop
This commit is contained in:
@@ -2975,12 +2975,14 @@ function Get-AgentOutcomeContract {
|
||||
$publicRows = @($data.public)
|
||||
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||||
$aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0)
|
||||
$host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified)
|
||||
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
|
||||
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
|
||||
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
|
||||
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
|
||||
$checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false
|
||||
$modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk)
|
||||
$checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ','))
|
||||
$modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk)
|
||||
}
|
||||
"Recover" {
|
||||
$expectedVms = @($Config.vms).Count
|
||||
@@ -2994,13 +2996,15 @@ function Get-AgentOutcomeContract {
|
||||
$publicRows = @($data.public)
|
||||
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
|
||||
$coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified)
|
||||
$host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified)
|
||||
$checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)"
|
||||
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
|
||||
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
|
||||
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
|
||||
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
|
||||
$checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ','))
|
||||
$checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ','))
|
||||
$modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk)
|
||||
$modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk)
|
||||
}
|
||||
"StartVMs" {
|
||||
$expectedVms = @($Config.vms).Count
|
||||
@@ -4087,6 +4091,107 @@ function Invoke-AgentProviderFreshnessTriage {
|
||||
$candidate
|
||||
}
|
||||
|
||||
function Convert-AgentHost112GuestReadback {
|
||||
param([object]$Transport)
|
||||
|
||||
$values = @{}
|
||||
$text = if ($Transport -and $Transport.PSObject.Properties["output"]) { [string]$Transport.output } else { "" }
|
||||
foreach ($match in [regex]::Matches($text, '(?:^|\s)([A-Za-z0-9_]+)=([^\s]+)')) {
|
||||
$values[$match.Groups[1].Value] = $match.Groups[2].Value
|
||||
}
|
||||
$get = {
|
||||
param([string]$Name, [string]$Default = "unknown")
|
||||
if ($values.ContainsKey($Name)) { return [string]$values[$Name] }
|
||||
return $Default
|
||||
}
|
||||
$asBool = {
|
||||
param([string]$Name)
|
||||
[bool]((& $get $Name "0") -eq "1")
|
||||
}
|
||||
$checks = @(
|
||||
[pscustomobject]@{ name = "readback_present"; passed = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) },
|
||||
[pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") },
|
||||
[pscustomobject]@{ name = "console_ready"; passed = (& $asBool "console_ready") },
|
||||
[pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") },
|
||||
[pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") },
|
||||
[pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") }
|
||||
)
|
||||
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
|
||||
[pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_readback_v1"
|
||||
readbackPresent = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready"))
|
||||
ready = [bool]($failedChecks.Count -eq 0)
|
||||
bootId = & $get "boot_id"
|
||||
uptimeSeconds = & $get "uptime_seconds" "-1"
|
||||
systemdState = & $get "systemd_state"
|
||||
defaultTarget = & $get "default_target"
|
||||
graphicalTarget = & $get "graphical_target"
|
||||
displayManager = & $get "display_manager"
|
||||
lightdm = & $get "lightdm"
|
||||
vmwareTools = & $get "vmware_tools"
|
||||
xorg = & $get "xorg"
|
||||
wazuhIndexer = & $get "wazuh_indexer"
|
||||
wazuhManager = & $get "wazuh_manager"
|
||||
wazuhDashboard = & $get "wazuh_dashboard"
|
||||
filebeat = & $get "filebeat"
|
||||
consoleReady = (& $asBool "console_ready")
|
||||
servicesReady = (& $asBool "services_ready")
|
||||
recoveryTimerReady = (& $asBool "recovery_timer_ready")
|
||||
guestReady = (& $asBool "guest_ready")
|
||||
actionCount = & $get "action_count" "0"
|
||||
actions = & $get "actions" "none"
|
||||
transportOk = [bool]($Transport -and $Transport.ok)
|
||||
transportExitCode = if ($Transport) { $Transport.exitCode } else { -1 }
|
||||
transportRoute = if ($Transport -and $Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "unknown" }
|
||||
transportSerialized = [bool]($Transport -and $Transport.PSObject.Properties["transportSerialized"] -and $Transport.transportSerialized)
|
||||
transportLockWaitMs = if ($Transport -and $Transport.PSObject.Properties["transportLockWaitMs"]) { $Transport.transportLockWaitMs } else { 0 }
|
||||
checks = $checks
|
||||
failedChecks = $failedChecks
|
||||
}
|
||||
}
|
||||
|
||||
function Invoke-AgentHost112GuestRecovery {
|
||||
$targetHost = "192.168.0.112"
|
||||
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check"
|
||||
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"
|
||||
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
|
||||
$before = Convert-AgentHost112GuestReadback $beforeTransport
|
||||
$applyTransport = $null
|
||||
$after = $before
|
||||
$applyAttempted = $false
|
||||
|
||||
if (-not $before.ready -and $ControlledApply) {
|
||||
$applyAttempted = $true
|
||||
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery failedChecks=$($before.failedChecks -join ',')"
|
||||
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 300 1
|
||||
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
|
||||
$after = Convert-AgentHost112GuestReadback $afterTransport
|
||||
}
|
||||
|
||||
$verified = [bool]$after.ready
|
||||
$changed = [bool]($applyAttempted -and $verified -and -not $before.ready)
|
||||
$result = [pscustomobject]@{
|
||||
schemaVersion = "agent99_host112_guest_recovery_v1"
|
||||
targetHost = $targetHost
|
||||
controlledApply = [bool]$ControlledApply
|
||||
applyAttempted = $applyAttempted
|
||||
applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null }
|
||||
changed = $changed
|
||||
verified = $verified
|
||||
before = $before
|
||||
after = $after
|
||||
rollback = "systemd package backup under /var/lib/awoooi-host112-recovery/backups"
|
||||
verifier = "forced_command_host112_guest_readiness_check"
|
||||
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
|
||||
}
|
||||
if ($changed) {
|
||||
Record-AgentEvent "host112_guest_recovered" "info" "112 圖形登入、VMware Tools、Wazuh 與 recovery timer 已由 Agent99 修復並驗證。" $result -Alert
|
||||
} elseif ($applyAttempted -and -not $verified) {
|
||||
Record-AgentEvent "host112_guest_recovery_failed" "critical" "112 受控修復後仍未通過 verifier;保留 failed checks 與 rollback evidence。" $result -Alert
|
||||
}
|
||||
$result
|
||||
}
|
||||
|
||||
function Test-HostReachability {
|
||||
$results = @()
|
||||
foreach ($hostIp in $Config.hosts) {
|
||||
@@ -4366,6 +4471,7 @@ Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledAp
|
||||
|
||||
$vmResults = @()
|
||||
$hostResults = @()
|
||||
$host112Recovery = $null
|
||||
$harbor = $null
|
||||
$awooo = $null
|
||||
$public = @()
|
||||
@@ -4419,6 +4525,15 @@ if ($Mode -in @("Status", "Recover")) {
|
||||
}
|
||||
}
|
||||
|
||||
if ($Mode -in @("Status", "Recover")) {
|
||||
$host112Recovery = Invoke-AgentHost112GuestRecovery
|
||||
if ($Mode -eq "Recover") {
|
||||
$recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
|
||||
} elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) {
|
||||
$recoveryTrigger = Start-AgentRecoveryFromObservation "host112_guest_not_ready" $host112Recovery.after
|
||||
}
|
||||
}
|
||||
|
||||
if ($Mode -in @("Status", "Recover", "HarborRepair")) {
|
||||
$harbor = Get-HarborState
|
||||
}
|
||||
@@ -4537,7 +4652,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit
|
||||
if ($Mode -eq "Recover") {
|
||||
$elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1)
|
||||
$unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count
|
||||
$builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0)
|
||||
$builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0)
|
||||
$coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified)
|
||||
$requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow)
|
||||
$rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed)
|
||||
@@ -4587,6 +4702,7 @@ $result = [pscustomobject]@{
|
||||
recoveryCoordinator = $recoveryCoordinator
|
||||
vmResults = $vmResults
|
||||
hosts = $hostResults
|
||||
host112Recovery = $host112Recovery
|
||||
harbor = $harbor
|
||||
awoooi = $awooo
|
||||
public = $public
|
||||
|
||||
@@ -84,7 +84,7 @@ function Test-AgentAlertActionable {
|
||||
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
|
||||
return $true
|
||||
}
|
||||
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
|
||||
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
|
||||
return $true
|
||||
}
|
||||
return $false
|
||||
@@ -113,6 +113,9 @@ function Resolve-AgentAlertRoute {
|
||||
"host_down" = "Recover"
|
||||
"host_reboot" = "Recover"
|
||||
"vm_not_running" = "Recover"
|
||||
"host112_guest_recovery" = "Recover"
|
||||
"host112_guest_not_ready" = "Recover"
|
||||
"host112_guest_readback_missing" = "Recover"
|
||||
"performance_pressure" = "Perf"
|
||||
"performance_cpu" = "Perf"
|
||||
"performance_memory" = "Perf"
|
||||
@@ -132,7 +135,7 @@ function Resolve-AgentAlertRoute {
|
||||
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
|
||||
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" }
|
||||
}
|
||||
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
|
||||
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup|host112_guest|host 112.*(?:guest|console|lightdm)") {
|
||||
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" }
|
||||
}
|
||||
if ($text -match "backup|snapshot|restore drill|cron") {
|
||||
@@ -301,6 +304,22 @@ function Invoke-AgentAlertRoutingSelfTest {
|
||||
message = "host_reboot detected vm_not_running"
|
||||
}
|
||||
},
|
||||
[pscustomobject]@{
|
||||
name = "host 112 guest recovery"
|
||||
expectedActionable = $true
|
||||
expectedMode = "Recover"
|
||||
expectedRouteSource = "kind"
|
||||
expectedControlledApply = $true
|
||||
alert = [pscustomobject]@{
|
||||
id = "selftest-host112-guest"
|
||||
source = "agent99-routing-selftest"
|
||||
severity = "critical"
|
||||
kind = "host112_guest_recovery"
|
||||
service = "host112-guest-recovery"
|
||||
host = "192.168.0.112"
|
||||
message = "Host 112 Console LightDM VMware Tools and Wazuh not ready"
|
||||
}
|
||||
},
|
||||
[pscustomobject]@{
|
||||
name = "performance pressure"
|
||||
expectedActionable = $true
|
||||
|
||||
@@ -292,6 +292,7 @@ function Get-AgentTelegramAlertSeverity {
|
||||
function Get-AgentTelegramAlertKind {
|
||||
param([string]$Text)
|
||||
$lower = $Text.ToLowerInvariant()
|
||||
if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112_guest_recovery" }
|
||||
if ($lower -match "\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return "cicd_failure" }
|
||||
if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" }
|
||||
if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" }
|
||||
@@ -313,6 +314,7 @@ function Get-AgentTelegramAlertKind {
|
||||
function Get-AgentTelegramAlertService {
|
||||
param([string]$Text)
|
||||
$lower = $Text.ToLowerInvariant()
|
||||
if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112-guest-recovery" }
|
||||
if ($lower -match "stock") { return "stockplatform" }
|
||||
if ($lower -match "2026fifa|fifa") { return "2026fifa" }
|
||||
if ($lower -match "bitan" -or (Test-AgentTextContainsCode $Text @(0x78A7, 0x6F6D))) { return "bitan-pharmacy" }
|
||||
@@ -348,7 +350,7 @@ function Get-AgentTelegramAlertTarget {
|
||||
return $targetMatch.Groups[1].Value.Trim()
|
||||
}
|
||||
if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" }
|
||||
if ($Kind -eq "host_recovery" -and $HostName) { return $HostName }
|
||||
if ($Kind -in @("host_recovery", "host112_guest_recovery") -and $HostName) { return $HostName }
|
||||
if ($Service -and $Service -ne "unknown") { return $Service }
|
||||
return $Kind
|
||||
}
|
||||
@@ -361,6 +363,7 @@ function Get-AgentTelegramSuggestedMode {
|
||||
"wazuh_security_alert" = "SecurityTriage"
|
||||
"public_route_502" = "Recover"
|
||||
"host_recovery" = "Recover"
|
||||
"host112_guest_recovery" = "Recover"
|
||||
"performance_cpu" = "Perf"
|
||||
"performance_memory" = "Perf"
|
||||
"performance_disk" = "Perf"
|
||||
@@ -383,6 +386,7 @@ function Get-AgentTelegramSourceEventResolutionPolicy {
|
||||
"backup_health",
|
||||
"public_route_502",
|
||||
"host_recovery",
|
||||
"host112_guest_recovery",
|
||||
"performance_cpu",
|
||||
"performance_memory",
|
||||
"performance_disk",
|
||||
@@ -566,6 +570,15 @@ function Invoke-AgentTelegramInboxSelfTest {
|
||||
expectedService = "bitan-pharmacy"
|
||||
expectedHost = ""
|
||||
},
|
||||
[pscustomobject]@{
|
||||
name = "host 112 guest readiness"
|
||||
text = "critical Host112GuestNotReady host 192.168.0.112 Console LightDM VMware Tools Wazuh not ready"
|
||||
expectedMonitoring = $true
|
||||
expectedSeverity = "critical"
|
||||
expectedKind = "host112_guest_recovery"
|
||||
expectedService = "host112-guest-recovery"
|
||||
expectedHost = "192.168.0.112"
|
||||
},
|
||||
[pscustomobject]@{
|
||||
name = "security triage"
|
||||
text = "Wazuh SIEM critical security alert auth_failed brute force suspicious login host 192.168.0.112"
|
||||
|
||||
@@ -95,6 +95,12 @@ def resolve_agent99_alert_kind(
|
||||
annotations=annotations or {},
|
||||
)
|
||||
|
||||
if re.search(
|
||||
r"host112guest(?:readbackmissing|notready)|host112_guest|"
|
||||
r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)",
|
||||
text,
|
||||
):
|
||||
return "host112_guest_recovery"
|
||||
if re.search(
|
||||
r"provider_freshness_signal|source_provider_freshness_triage|"
|
||||
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
|
||||
@@ -168,6 +174,7 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
|
||||
"harbor_registry": "HarborRepair",
|
||||
"awoooi_k3s": "AwoooRepair",
|
||||
"host_recovery": "Recover",
|
||||
"host112_guest_recovery": "Recover",
|
||||
"application_ai_config_error": "Status",
|
||||
"cicd_failure": "Status",
|
||||
"monitoring_alert": "Status",
|
||||
@@ -183,6 +190,7 @@ def _resolution_policy_for_kind(kind: str) -> str:
|
||||
"harbor_registry",
|
||||
"awoooi_k3s",
|
||||
"host_recovery",
|
||||
"host112_guest_recovery",
|
||||
}:
|
||||
return "mode_verifier_can_resolve"
|
||||
return "external_source_receipt_required"
|
||||
@@ -335,6 +343,12 @@ def build_agent99_sre_alert(
|
||||
"directReference,verifierReadback; mark missing,expired,timeout,"
|
||||
"no_false_green; do not switch provider, call paid model, edit env, or reload."
|
||||
)
|
||||
elif kind == "host112_guest_recovery":
|
||||
instruction = (
|
||||
"Run the allowlisted host 112 guest recovery; verify graphical.target, "
|
||||
"LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot "
|
||||
"the host, change VM power, reset the VM, read secrets or change firewall."
|
||||
)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
|
||||
|
||||
@@ -41,6 +41,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
|
||||
assert '"database_health" = "Status"' in source
|
||||
assert '"alert_chain_health" = "Status"' in source
|
||||
assert '"host_recovery" = "Recover"' in source
|
||||
assert '"host112_guest_recovery" = "Recover"' in source
|
||||
assert '"cicd_failure" = "Status"' in source
|
||||
assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source
|
||||
assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source
|
||||
@@ -48,6 +49,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
|
||||
telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8")
|
||||
assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source
|
||||
assert '"cicd_failure" = "Status"' in telegram_source
|
||||
assert '"host112_guest_recovery" = "Recover"' in telegram_source
|
||||
assert 'policy = $sourceEventResolutionPolicy' in telegram_source
|
||||
|
||||
|
||||
@@ -69,3 +71,4 @@ def test_telegram_ingress_writes_same_structured_route_contract() -> None:
|
||||
assert 'routeSource = "telegram_structured_kind"' in source
|
||||
assert '"database_health" = "Status"' in source
|
||||
assert '"alert_chain_health" = "Status"' in source
|
||||
assert '"host112_guest_recovery" = "Recover"' in source
|
||||
|
||||
@@ -19,6 +19,8 @@ def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None:
|
||||
assert '"verifying"' in source
|
||||
assert '"candidate_ready"' in source
|
||||
assert "ok = [bool]($exitCode -eq 0)" not in source
|
||||
assert 'New-AgentOutcomeCheck "host112_guest_ready" $host112Ok' in source
|
||||
assert "$data.host112Recovery -and $data.host112Recovery.verified" in source
|
||||
|
||||
|
||||
def test_agent99_problem_counts_separate_verified_resolution() -> None:
|
||||
|
||||
@@ -105,6 +105,14 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
|
||||
"host_recovery",
|
||||
"Recover",
|
||||
),
|
||||
(
|
||||
"Host112GuestNotReady",
|
||||
"host112-guest-recovery",
|
||||
"Host 112 console LightDM VMware Tools and Wazuh are not ready",
|
||||
{"host": "112", "service": "host112-guest-recovery"},
|
||||
"host112_guest_recovery",
|
||||
"Recover",
|
||||
),
|
||||
(
|
||||
"KubePodCrashLooping",
|
||||
"awoooi-api",
|
||||
@@ -176,6 +184,24 @@ def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts()
|
||||
assert cicd["resolution"]["policy"] == "external_source_receipt_required"
|
||||
|
||||
|
||||
def test_host112_guest_recovery_precedes_wazuh_security_classification() -> None:
|
||||
payload = build_agent99_sre_alert(
|
||||
alert_id="host112-guest-not-ready",
|
||||
alertname="Host112GuestNotReady",
|
||||
severity="critical",
|
||||
namespace="host112",
|
||||
target_resource="host112-guest-recovery",
|
||||
message="Host 112 Console VMware Tools Wazuh services are not ready",
|
||||
labels={"host": "112", "service": "host112-guest-recovery"},
|
||||
)
|
||||
|
||||
assert payload["kind"] == "host112_guest_recovery"
|
||||
assert payload["suggestedMode"] == "Recover"
|
||||
assert payload["controlledApply"] is True
|
||||
assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
|
||||
assert "do not reboot" in payload["instruction"]
|
||||
|
||||
|
||||
def test_alert_kind_mapping_for_route_and_performance() -> None:
|
||||
assert (
|
||||
resolve_agent99_alert_kind(
|
||||
|
||||
@@ -37,6 +37,24 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None:
|
||||
assert "Start-Process" not in trigger
|
||||
assert 'Record-AgentEvent "recovery_slo_result"' in source
|
||||
assert "withinSlo = $withinSlo" in source
|
||||
assert "function Invoke-AgentHost112GuestRecovery" in source
|
||||
assert 'Start-AgentRecoveryFromObservation "host112_guest_not_ready"' in source
|
||||
assert 'New-AgentRecoveryPhase "host112_guest_readiness"' in source
|
||||
assert '$host112Recovery -and $host112Recovery.verified' in source
|
||||
|
||||
|
||||
def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
function = source[source.index("function Convert-AgentHost112GuestReadback") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function
|
||||
assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function
|
||||
assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2
|
||||
assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function
|
||||
assert 'verifier = "forced_command_host112_guest_readiness_check"' in function
|
||||
for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"):
|
||||
assert f'"{forbidden}"' in function
|
||||
|
||||
|
||||
def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None:
|
||||
|
||||
Reference in New Issue
Block a user