feat(recovery): close host112 guest readiness loop

This commit is contained in:
ogt
2026-07-11 20:00:16 +08:00
parent 23751a7fc3
commit 43a3a06a3a
8 changed files with 217 additions and 6 deletions

View File

@@ -2975,12 +2975,14 @@ function Get-AgentOutcomeContract {
$publicRows = @($data.public)
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
$aiOk = [bool](@($data.aiServices | Where-Object { -not $_.ok }).Count -eq 0)
$host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified)
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
$checks += New-AgentOutcomeCheck "ai_services_healthy" $aiOk $false
$modeVerifierPassed = [bool]($hostsOk -and $harborOk -and $awoooOk -and $publicOk)
$checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ','))
$modeVerifierPassed = [bool]($hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk)
}
"Recover" {
$expectedVms = @($Config.vms).Count
@@ -2994,13 +2996,15 @@ function Get-AgentOutcomeContract {
$publicRows = @($data.public)
$publicOk = [bool]($publicRows.Count -gt 0 -and @($publicRows | Where-Object { -not $_.ok }).Count -eq 0)
$coordinatorOk = [bool]($data.recoveryCoordinator -and $data.recoveryCoordinator.verified)
$host112Ok = [bool]($data.host112Recovery -and $data.host112Recovery.verified)
$checks += New-AgentOutcomeCheck "configured_vms_ready" $vmsOk $true "expected=$expectedVms actual=$($vmRows.Count)"
$checks += New-AgentOutcomeCheck "required_hosts_reachable" $hostsOk $true "expected=$expectedHosts actual=$($hostRows.Count)"
$checks += New-AgentOutcomeCheck "harbor_healthy" $harborOk
$checks += New-AgentOutcomeCheck "awoooi_deployments_ready" $awoooOk
$checks += New-AgentOutcomeCheck "public_routes_healthy" $publicOk
$checks += New-AgentOutcomeCheck "host112_guest_ready" $host112Ok $true $([string](@($data.host112Recovery.after.failedChecks) -join ','))
$checks += New-AgentOutcomeCheck "full_sop_coordinator_verified" $coordinatorOk $true $([string](@($data.recoveryCoordinator.failedChecks) -join ','))
$modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk)
$modeVerifierPassed = [bool]($vmsOk -and $hostsOk -and $host112Ok -and $harborOk -and $awoooOk -and $publicOk -and $coordinatorOk)
}
"StartVMs" {
$expectedVms = @($Config.vms).Count
@@ -4087,6 +4091,107 @@ function Invoke-AgentProviderFreshnessTriage {
$candidate
}
function Convert-AgentHost112GuestReadback {
param([object]$Transport)
$values = @{}
$text = if ($Transport -and $Transport.PSObject.Properties["output"]) { [string]$Transport.output } else { "" }
foreach ($match in [regex]::Matches($text, '(?:^|\s)([A-Za-z0-9_]+)=([^\s]+)')) {
$values[$match.Groups[1].Value] = $match.Groups[2].Value
}
$get = {
param([string]$Name, [string]$Default = "unknown")
if ($values.ContainsKey($Name)) { return [string]$values[$Name] }
return $Default
}
$asBool = {
param([string]$Name)
[bool]((& $get $Name "0") -eq "1")
}
$checks = @(
[pscustomobject]@{ name = "readback_present"; passed = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready")) },
[pscustomobject]@{ name = "systemd_running"; passed = [bool]((& $get "systemd_state") -eq "running") },
[pscustomobject]@{ name = "console_ready"; passed = (& $asBool "console_ready") },
[pscustomobject]@{ name = "services_ready"; passed = (& $asBool "services_ready") },
[pscustomobject]@{ name = "recovery_timer_ready"; passed = (& $asBool "recovery_timer_ready") },
[pscustomobject]@{ name = "guest_ready"; passed = (& $asBool "guest_ready") }
)
$failedChecks = @($checks | Where-Object { -not $_.passed } | ForEach-Object { $_.name })
[pscustomobject]@{
schemaVersion = "agent99_host112_guest_readback_v1"
readbackPresent = [bool]($values.ContainsKey("boot_id") -and $values.ContainsKey("guest_ready"))
ready = [bool]($failedChecks.Count -eq 0)
bootId = & $get "boot_id"
uptimeSeconds = & $get "uptime_seconds" "-1"
systemdState = & $get "systemd_state"
defaultTarget = & $get "default_target"
graphicalTarget = & $get "graphical_target"
displayManager = & $get "display_manager"
lightdm = & $get "lightdm"
vmwareTools = & $get "vmware_tools"
xorg = & $get "xorg"
wazuhIndexer = & $get "wazuh_indexer"
wazuhManager = & $get "wazuh_manager"
wazuhDashboard = & $get "wazuh_dashboard"
filebeat = & $get "filebeat"
consoleReady = (& $asBool "console_ready")
servicesReady = (& $asBool "services_ready")
recoveryTimerReady = (& $asBool "recovery_timer_ready")
guestReady = (& $asBool "guest_ready")
actionCount = & $get "action_count" "0"
actions = & $get "actions" "none"
transportOk = [bool]($Transport -and $Transport.ok)
transportExitCode = if ($Transport) { $Transport.exitCode } else { -1 }
transportRoute = if ($Transport -and $Transport.PSObject.Properties["route"]) { [string]$Transport.route } else { "unknown" }
transportSerialized = [bool]($Transport -and $Transport.PSObject.Properties["transportSerialized"] -and $Transport.transportSerialized)
transportLockWaitMs = if ($Transport -and $Transport.PSObject.Properties["transportLockWaitMs"]) { $Transport.transportLockWaitMs } else { 0 }
checks = $checks
failedChecks = $failedChecks
}
}
function Invoke-AgentHost112GuestRecovery {
$targetHost = "192.168.0.112"
$checkCommand = "/usr/local/sbin/awoooi-host112-guest-readiness --check"
$applyCommand = "sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"
$beforeTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
$before = Convert-AgentHost112GuestReadback $beforeTransport
$applyTransport = $null
$after = $before
$applyAttempted = $false
if (-not $before.ready -and $ControlledApply) {
$applyAttempted = $true
Write-AgentLog "CONTROLLED_APPLY host112_guest_recovery failedChecks=$($before.failedChecks -join ',')"
$applyTransport = Invoke-HostSshText $targetHost $applyCommand 300 1
$afterTransport = Invoke-HostSshText $targetHost $checkCommand 20 1
$after = Convert-AgentHost112GuestReadback $afterTransport
}
$verified = [bool]$after.ready
$changed = [bool]($applyAttempted -and $verified -and -not $before.ready)
$result = [pscustomobject]@{
schemaVersion = "agent99_host112_guest_recovery_v1"
targetHost = $targetHost
controlledApply = [bool]$ControlledApply
applyAttempted = $applyAttempted
applyExitCode = if ($applyTransport) { $applyTransport.exitCode } else { $null }
changed = $changed
verified = $verified
before = $before
after = $after
rollback = "systemd package backup under /var/lib/awoooi-host112-recovery/backups"
verifier = "forced_command_host112_guest_readiness_check"
prohibitedActions = @("host_reboot", "vm_power_change", "vm_reset", "secret_read", "firewall_change", "database_write")
}
if ($changed) {
Record-AgentEvent "host112_guest_recovered" "info" "112 圖形登入、VMware Tools、Wazuh 與 recovery timer 已由 Agent99 修復並驗證。" $result -Alert
} elseif ($applyAttempted -and -not $verified) {
Record-AgentEvent "host112_guest_recovery_failed" "critical" "112 受控修復後仍未通過 verifier保留 failed checks 與 rollback evidence。" $result -Alert
}
$result
}
function Test-HostReachability {
$results = @()
foreach ($hostIp in $Config.hosts) {
@@ -4366,6 +4471,7 @@ Record-AgentEvent "agent_start" "info" "mode=$Mode controlledApply=$ControlledAp
$vmResults = @()
$hostResults = @()
$host112Recovery = $null
$harbor = $null
$awooo = $null
$public = @()
@@ -4419,6 +4525,15 @@ if ($Mode -in @("Status", "Recover")) {
}
}
if ($Mode -in @("Status", "Recover")) {
$host112Recovery = Invoke-AgentHost112GuestRecovery
if ($Mode -eq "Recover") {
$recoveryPhases += New-AgentRecoveryPhase "host112_guest_readiness" $(if ($host112Recovery.verified) { "ok" } else { "failed" }) (((Get-Date) - $recoveryStartedAt).TotalSeconds)
} elseif (-not $host112Recovery.verified -and -not $recoveryTrigger) {
$recoveryTrigger = Start-AgentRecoveryFromObservation "host112_guest_not_ready" $host112Recovery.after
}
}
if ($Mode -in @("Status", "Recover", "HarborRepair")) {
$harbor = Get-HarborState
}
@@ -4537,7 +4652,7 @@ $summarySeverity = if ($publicFailures -gt 0 -or $aiFailures -gt 0 -or $perfCrit
if ($Mode -eq "Recover") {
$elapsedSeconds = [math]::Round(((Get-Date) - $recoveryStartedAt).TotalSeconds, 1)
$unreachableCount = @($hostResults | Where-Object { -not $_.ping }).Count
$builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0)
$builtInRecoveryCompleted = [bool]($unreachableCount -eq 0 -and $host112Recovery -and $host112Recovery.verified -and $harbor -and $harbor.coreHealthy -and $harbor.redisUp -and $harbor.registryHealthy -and $awooo -and $awooo.deployReady -and -not $awooo.imagePullFailure -and -not $awooo.crashFailure -and $publicFailures -eq 0)
$coordinatorVerified = [bool]($recoveryCoordinator -and $recoveryCoordinator.verified)
$requiresFreshWindow = [bool]($recoveryCoordinator -and $recoveryCoordinator.requireFreshRebootWindow)
$rebootSloClaimed = [bool]($requiresFreshWindow -and $recoveryCoordinator.rebootSloClaimed)
@@ -4587,6 +4702,7 @@ $result = [pscustomobject]@{
recoveryCoordinator = $recoveryCoordinator
vmResults = $vmResults
hosts = $hostResults
host112Recovery = $host112Recovery
harbor = $harbor
awoooi = $awooo
public = $public

View File

@@ -84,7 +84,7 @@ function Test-AgentAlertActionable {
if ($severity -in @("critical", "warning", "error", "degraded", "down", "failed")) {
return $true
}
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
if ($text -match "502|provider_freshness_signal|source_provider_freshness_triage|raw_signal_normalized|controlled_runtime_candidate|freshness|last_seen|last seen|ingestion|imagepullbackoff|errimagepull|crashloopbackoff|host_down|vm_not_running|vm_stuck|host112_guest|backup_failed|backup_missing|disk_used|load_per_core|cpu|memory|harbor|registry|k3s|wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc|gitea|repo_missing|repository_missing|missing_key|e_ai_missing_key|application_ai_config_error|locale guard|ai support") {
return $true
}
return $false
@@ -113,6 +113,9 @@ function Resolve-AgentAlertRoute {
"host_down" = "Recover"
"host_reboot" = "Recover"
"vm_not_running" = "Recover"
"host112_guest_recovery" = "Recover"
"host112_guest_not_ready" = "Recover"
"host112_guest_readback_missing" = "Recover"
"performance_pressure" = "Perf"
"performance_cpu" = "Perf"
"performance_memory" = "Perf"
@@ -132,7 +135,7 @@ function Resolve-AgentAlertRoute {
if ($text -match "502|site_502|public_route_502|route_down|website_down|nginx upstream") {
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "public_route" }
}
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup") {
if ($text -match "reboot-auto-recovery|reboot_auto_recovery|cold[-_ ]start|all_required_hosts_not_in_10_minute_reboot_window|host_down|host_reboot|reboot_detected|vm_not_running|powered_off|vmware|startup|host112_guest|host 112.*(?:guest|console|lightdm)") {
return [pscustomobject]@{ mode = "Recover"; source = "text_fallback"; matched = "host_recovery" }
}
if ($text -match "backup|snapshot|restore drill|cron") {
@@ -301,6 +304,22 @@ function Invoke-AgentAlertRoutingSelfTest {
message = "host_reboot detected vm_not_running"
}
},
[pscustomobject]@{
name = "host 112 guest recovery"
expectedActionable = $true
expectedMode = "Recover"
expectedRouteSource = "kind"
expectedControlledApply = $true
alert = [pscustomobject]@{
id = "selftest-host112-guest"
source = "agent99-routing-selftest"
severity = "critical"
kind = "host112_guest_recovery"
service = "host112-guest-recovery"
host = "192.168.0.112"
message = "Host 112 Console LightDM VMware Tools and Wazuh not ready"
}
},
[pscustomobject]@{
name = "performance pressure"
expectedActionable = $true

View File

@@ -292,6 +292,7 @@ function Get-AgentTelegramAlertSeverity {
function Get-AgentTelegramAlertKind {
param([string]$Text)
$lower = $Text.ToLowerInvariant()
if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112_guest_recovery" }
if ($lower -match "\bci/?cd\b|\bcicd\b|build-and-deploy|deploy failed|deployment failed|build failed|部署失敗|構建失敗") { return "cicd_failure" }
if ($lower -match "backup|snapshot|cron" -or (Test-AgentTextContainsCode $Text @(0x5099, 0x4EFD))) { return "backup_health" }
if ($lower -match "wazuh|siem|security|intrusion|malware|auth_failed|bruteforce|cve|ioc") { return "wazuh_security_alert" }
@@ -313,6 +314,7 @@ function Get-AgentTelegramAlertKind {
function Get-AgentTelegramAlertService {
param([string]$Text)
$lower = $Text.ToLowerInvariant()
if ($lower -match "host112guest|host112_guest|awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)") { return "host112-guest-recovery" }
if ($lower -match "stock") { return "stockplatform" }
if ($lower -match "2026fifa|fifa") { return "2026fifa" }
if ($lower -match "bitan" -or (Test-AgentTextContainsCode $Text @(0x78A7, 0x6F6D))) { return "bitan-pharmacy" }
@@ -348,7 +350,7 @@ function Get-AgentTelegramAlertTarget {
return $targetMatch.Groups[1].Value.Trim()
}
if ($Kind -eq "provider_freshness_signal") { return "provider_freshness_signal" }
if ($Kind -eq "host_recovery" -and $HostName) { return $HostName }
if ($Kind -in @("host_recovery", "host112_guest_recovery") -and $HostName) { return $HostName }
if ($Service -and $Service -ne "unknown") { return $Service }
return $Kind
}
@@ -361,6 +363,7 @@ function Get-AgentTelegramSuggestedMode {
"wazuh_security_alert" = "SecurityTriage"
"public_route_502" = "Recover"
"host_recovery" = "Recover"
"host112_guest_recovery" = "Recover"
"performance_cpu" = "Perf"
"performance_memory" = "Perf"
"performance_disk" = "Perf"
@@ -383,6 +386,7 @@ function Get-AgentTelegramSourceEventResolutionPolicy {
"backup_health",
"public_route_502",
"host_recovery",
"host112_guest_recovery",
"performance_cpu",
"performance_memory",
"performance_disk",
@@ -566,6 +570,15 @@ function Invoke-AgentTelegramInboxSelfTest {
expectedService = "bitan-pharmacy"
expectedHost = ""
},
[pscustomobject]@{
name = "host 112 guest readiness"
text = "critical Host112GuestNotReady host 192.168.0.112 Console LightDM VMware Tools Wazuh not ready"
expectedMonitoring = $true
expectedSeverity = "critical"
expectedKind = "host112_guest_recovery"
expectedService = "host112-guest-recovery"
expectedHost = "192.168.0.112"
},
[pscustomobject]@{
name = "security triage"
text = "Wazuh SIEM critical security alert auth_failed brute force suspicious login host 192.168.0.112"

View File

@@ -95,6 +95,12 @@ def resolve_agent99_alert_kind(
annotations=annotations or {},
)
if re.search(
r"host112guest(?:readbackmissing|notready)|host112_guest|"
r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)",
text,
):
return "host112_guest_recovery"
if re.search(
r"provider_freshness_signal|source_provider_freshness_triage|"
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
@@ -168,6 +174,7 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
"harbor_registry": "HarborRepair",
"awoooi_k3s": "AwoooRepair",
"host_recovery": "Recover",
"host112_guest_recovery": "Recover",
"application_ai_config_error": "Status",
"cicd_failure": "Status",
"monitoring_alert": "Status",
@@ -183,6 +190,7 @@ def _resolution_policy_for_kind(kind: str) -> str:
"harbor_registry",
"awoooi_k3s",
"host_recovery",
"host112_guest_recovery",
}:
return "mode_verifier_can_resolve"
return "external_source_receipt_required"
@@ -335,6 +343,12 @@ def build_agent99_sre_alert(
"directReference,verifierReadback; mark missing,expired,timeout,"
"no_false_green; do not switch provider, call paid model, edit env, or reload."
)
elif kind == "host112_guest_recovery":
instruction = (
"Run the allowlisted host 112 guest recovery; verify graphical.target, "
"LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot "
"the host, change VM power, reset the VM, read secrets or change firewall."
)
payload: dict[str, Any] = {
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",

View File

@@ -41,6 +41,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source
assert '"host_recovery" = "Recover"' in source
assert '"host112_guest_recovery" = "Recover"' in source
assert '"cicd_failure" = "Status"' in source
assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source
assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source
@@ -48,6 +49,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8")
assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source
assert '"cicd_failure" = "Status"' in telegram_source
assert '"host112_guest_recovery" = "Recover"' in telegram_source
assert 'policy = $sourceEventResolutionPolicy' in telegram_source
@@ -69,3 +71,4 @@ def test_telegram_ingress_writes_same_structured_route_contract() -> None:
assert 'routeSource = "telegram_structured_kind"' in source
assert '"database_health" = "Status"' in source
assert '"alert_chain_health" = "Status"' in source
assert '"host112_guest_recovery" = "Recover"' in source

View File

@@ -19,6 +19,8 @@ def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None:
assert '"verifying"' in source
assert '"candidate_ready"' in source
assert "ok = [bool]($exitCode -eq 0)" not in source
assert 'New-AgentOutcomeCheck "host112_guest_ready" $host112Ok' in source
assert "$data.host112Recovery -and $data.host112Recovery.verified" in source
def test_agent99_problem_counts_separate_verified_resolution() -> None:

View File

@@ -105,6 +105,14 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
"host_recovery",
"Recover",
),
(
"Host112GuestNotReady",
"host112-guest-recovery",
"Host 112 console LightDM VMware Tools and Wazuh are not ready",
{"host": "112", "service": "host112-guest-recovery"},
"host112_guest_recovery",
"Recover",
),
(
"KubePodCrashLooping",
"awoooi-api",
@@ -176,6 +184,24 @@ def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts()
assert cicd["resolution"]["policy"] == "external_source_receipt_required"
def test_host112_guest_recovery_precedes_wazuh_security_classification() -> None:
payload = build_agent99_sre_alert(
alert_id="host112-guest-not-ready",
alertname="Host112GuestNotReady",
severity="critical",
namespace="host112",
target_resource="host112-guest-recovery",
message="Host 112 Console VMware Tools Wazuh services are not ready",
labels={"host": "112", "service": "host112-guest-recovery"},
)
assert payload["kind"] == "host112_guest_recovery"
assert payload["suggestedMode"] == "Recover"
assert payload["controlledApply"] is True
assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
assert "do not reboot" in payload["instruction"]
def test_alert_kind_mapping_for_route_and_performance() -> None:
assert (
resolve_agent99_alert_kind(

View File

@@ -37,6 +37,24 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None:
assert "Start-Process" not in trigger
assert 'Record-AgentEvent "recovery_slo_result"' in source
assert "withinSlo = $withinSlo" in source
assert "function Invoke-AgentHost112GuestRecovery" in source
assert 'Start-AgentRecoveryFromObservation "host112_guest_not_ready"' in source
assert 'New-AgentRecoveryPhase "host112_guest_readiness"' in source
assert '$host112Recovery -and $host112Recovery.verified' in source
def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:
source = CONTROL.read_text(encoding="utf-8")
function = source[source.index("function Convert-AgentHost112GuestReadback") :]
function = function[: function.index("function Test-HostReachability")]
assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function
assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function
assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2
assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function
assert 'verifier = "forced_command_host112_guest_readiness_check"' in function
for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"):
assert f'"{forbidden}"' in function
def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None: