feat(recovery): close host112 guest readiness loop
This commit is contained in:
@@ -95,6 +95,12 @@ def resolve_agent99_alert_kind(
|
||||
annotations=annotations or {},
|
||||
)
|
||||
|
||||
if re.search(
|
||||
r"host112guest(?:readbackmissing|notready)|host112_guest|"
|
||||
r"awoooi_host112_|host 112.*(?:guest|console|lightdm|vmware tools)",
|
||||
text,
|
||||
):
|
||||
return "host112_guest_recovery"
|
||||
if re.search(
|
||||
r"provider_freshness_signal|source_provider_freshness_triage|"
|
||||
r"raw_signal_normalized|controlled_runtime_candidate|freshness|"
|
||||
@@ -168,6 +174,7 @@ def _suggested_mode_for_kind(kind: str) -> str | None:
|
||||
"harbor_registry": "HarborRepair",
|
||||
"awoooi_k3s": "AwoooRepair",
|
||||
"host_recovery": "Recover",
|
||||
"host112_guest_recovery": "Recover",
|
||||
"application_ai_config_error": "Status",
|
||||
"cicd_failure": "Status",
|
||||
"monitoring_alert": "Status",
|
||||
@@ -183,6 +190,7 @@ def _resolution_policy_for_kind(kind: str) -> str:
|
||||
"harbor_registry",
|
||||
"awoooi_k3s",
|
||||
"host_recovery",
|
||||
"host112_guest_recovery",
|
||||
}:
|
||||
return "mode_verifier_can_resolve"
|
||||
return "external_source_receipt_required"
|
||||
@@ -335,6 +343,12 @@ def build_agent99_sre_alert(
|
||||
"directReference,verifierReadback; mark missing,expired,timeout,"
|
||||
"no_false_green; do not switch provider, call paid model, edit env, or reload."
|
||||
)
|
||||
elif kind == "host112_guest_recovery":
|
||||
instruction = (
|
||||
"Run the allowlisted host 112 guest recovery; verify graphical.target, "
|
||||
"LightDM, VMware Tools, Wazuh services and recovery timer; do not reboot "
|
||||
"the host, change VM power, reset the VM, read secrets or change firewall."
|
||||
)
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"id": f"awoooi-alertmanager-{_safe_file_token(alert_id)}",
|
||||
|
||||
@@ -41,6 +41,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
|
||||
assert '"database_health" = "Status"' in source
|
||||
assert '"alert_chain_health" = "Status"' in source
|
||||
assert '"host_recovery" = "Recover"' in source
|
||||
assert '"host112_guest_recovery" = "Recover"' in source
|
||||
assert '"cicd_failure" = "Status"' in source
|
||||
assert "sourceEventResolutionPolicy = $sourceEventResolutionPolicy" in source
|
||||
assert "sourceEventResolutionRequired = $sourceEventResolutionRequired" in source
|
||||
@@ -48,6 +49,7 @@ def test_agent99_runtime_prioritizes_structured_route_over_text() -> None:
|
||||
telegram_source = TELEGRAM_INBOX.read_text(encoding="utf-8")
|
||||
assert "function Get-AgentTelegramSourceEventResolutionPolicy" in telegram_source
|
||||
assert '"cicd_failure" = "Status"' in telegram_source
|
||||
assert '"host112_guest_recovery" = "Recover"' in telegram_source
|
||||
assert 'policy = $sourceEventResolutionPolicy' in telegram_source
|
||||
|
||||
|
||||
@@ -69,3 +71,4 @@ def test_telegram_ingress_writes_same_structured_route_contract() -> None:
|
||||
assert 'routeSource = "telegram_structured_kind"' in source
|
||||
assert '"database_health" = "Status"' in source
|
||||
assert '"alert_chain_health" = "Status"' in source
|
||||
assert '"host112_guest_recovery" = "Recover"' in source
|
||||
|
||||
@@ -19,6 +19,8 @@ def test_agent99_queue_uses_verified_outcome_not_exit_code() -> None:
|
||||
assert '"verifying"' in source
|
||||
assert '"candidate_ready"' in source
|
||||
assert "ok = [bool]($exitCode -eq 0)" not in source
|
||||
assert 'New-AgentOutcomeCheck "host112_guest_ready" $host112Ok' in source
|
||||
assert "$data.host112Recovery -and $data.host112Recovery.verified" in source
|
||||
|
||||
|
||||
def test_agent99_problem_counts_separate_verified_resolution() -> None:
|
||||
|
||||
@@ -105,6 +105,14 @@ def test_provider_freshness_alert_builds_agent99_contract() -> None:
|
||||
"host_recovery",
|
||||
"Recover",
|
||||
),
|
||||
(
|
||||
"Host112GuestNotReady",
|
||||
"host112-guest-recovery",
|
||||
"Host 112 console LightDM VMware Tools and Wazuh are not ready",
|
||||
{"host": "112", "service": "host112-guest-recovery"},
|
||||
"host112_guest_recovery",
|
||||
"Recover",
|
||||
),
|
||||
(
|
||||
"KubePodCrashLooping",
|
||||
"awoooi-api",
|
||||
@@ -176,6 +184,24 @@ def test_agent99_resolution_policy_uses_mode_verifier_only_for_stateful_alerts()
|
||||
assert cicd["resolution"]["policy"] == "external_source_receipt_required"
|
||||
|
||||
|
||||
def test_host112_guest_recovery_precedes_wazuh_security_classification() -> None:
|
||||
payload = build_agent99_sre_alert(
|
||||
alert_id="host112-guest-not-ready",
|
||||
alertname="Host112GuestNotReady",
|
||||
severity="critical",
|
||||
namespace="host112",
|
||||
target_resource="host112-guest-recovery",
|
||||
message="Host 112 Console VMware Tools Wazuh services are not ready",
|
||||
labels={"host": "112", "service": "host112-guest-recovery"},
|
||||
)
|
||||
|
||||
assert payload["kind"] == "host112_guest_recovery"
|
||||
assert payload["suggestedMode"] == "Recover"
|
||||
assert payload["controlledApply"] is True
|
||||
assert payload["resolution"]["policy"] == "mode_verifier_can_resolve"
|
||||
assert "do not reboot" in payload["instruction"]
|
||||
|
||||
|
||||
def test_alert_kind_mapping_for_route_and_performance() -> None:
|
||||
assert (
|
||||
resolve_agent99_alert_kind(
|
||||
|
||||
@@ -37,6 +37,24 @@ def test_agent99_auto_recovery_is_single_flight_and_slo_measured() -> None:
|
||||
assert "Start-Process" not in trigger
|
||||
assert 'Record-AgentEvent "recovery_slo_result"' in source
|
||||
assert "withinSlo = $withinSlo" in source
|
||||
assert "function Invoke-AgentHost112GuestRecovery" in source
|
||||
assert 'Start-AgentRecoveryFromObservation "host112_guest_not_ready"' in source
|
||||
assert 'New-AgentRecoveryPhase "host112_guest_readiness"' in source
|
||||
assert '$host112Recovery -and $host112Recovery.verified' in source
|
||||
|
||||
|
||||
def test_agent99_host112_recovery_is_allowlisted_and_independently_verified() -> None:
|
||||
source = CONTROL.read_text(encoding="utf-8")
|
||||
function = source[source.index("function Convert-AgentHost112GuestReadback") :]
|
||||
function = function[: function.index("function Test-HostReachability")]
|
||||
|
||||
assert '"/usr/local/sbin/awoooi-host112-guest-readiness --check"' in function
|
||||
assert '"sudo -n /usr/local/sbin/awoooi-host112-guest-readiness --apply"' in function
|
||||
assert function.count("Invoke-HostSshText $targetHost $checkCommand") == 2
|
||||
assert 'schemaVersion = "agent99_host112_guest_recovery_v1"' in function
|
||||
assert 'verifier = "forced_command_host112_guest_readiness_check"' in function
|
||||
for forbidden in ("host_reboot", "vm_power_change", "vm_reset", "secret_read"):
|
||||
assert f'"{forbidden}"' in function
|
||||
|
||||
|
||||
def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None:
|
||||
|
||||
Reference in New Issue
Block a user