From b9ee3c5cbb2881a0075c852cb29be8f7c66c8975 Mon Sep 17 00:00:00 2001 From: ogt Date: Wed, 15 Jul 2026 15:21:24 +0800 Subject: [PATCH] fix(agent99): stop Host112 verifier churn --- agent99-bootstrap.ps1 | 2 + agent99-control-plane.ps1 | 95 +++- agent99-deploy.ps1 | 7 + agent99.config.99.example.json | 3 + agent99.config.example.json | 3 + .../services/awooop_ansible_audit_service.py | 197 ++++++++ .../awooop_ansible_check_mode_service.py | 227 ++++++++- ...nt99_transport_recovery_deploy_contract.py | 8 + ...est_ansible_incident_ledger_churn_guard.py | 317 ++++++++++++ .../api/tests/test_sre_typed_domain_router.py | 7 + ...99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md | 32 ++ ...recovery-readback-2026-07-15.snapshot.json | 61 +++ docs/runbooks/FULL-STACK-COLD-START-SOP.md | 34 ++ .../install-macos111-host-boot-readback.sh | 4 +- .../macos-host-boot-readback.sh | 55 +++ ...nt99_node_exporter_performance_contract.py | 9 + .../test_reboot_p0_operational_contract.py | 6 + ...ows99_host112_kernel_memory_soft_reboot.py | 80 +++ ...ws99-host112-kernel-memory-soft-reboot.ps1 | 464 ++++++++++++++++++ 19 files changed, 1592 insertions(+), 19 deletions(-) create mode 100644 apps/api/tests/test_ansible_incident_ledger_churn_guard.py create mode 100644 docs/operations/agent99-host112-host111-recovery-readback-2026-07-15.snapshot.json create mode 100644 scripts/reboot-recovery/tests/test_windows99_host112_kernel_memory_soft_reboot.py create mode 100644 scripts/reboot-recovery/windows99-host112-kernel-memory-soft-reboot.ps1 diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index be020aeec..40cce581a 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -361,6 +361,8 @@ if (-not (Test-Path $configPath)) { }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, + "processCpuHostPercentWarning": 25, + "processCpuHostPercentCritical": 50, "memoryAvailablePercentCritical": 10, "kernelPercpuMemoryPercentWarning": 10, "kernelPercpuMemoryPercentCritical": 20, diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index 68bca2b88..318dc3879 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -633,7 +633,9 @@ function Format-AgentTelegramLegacyText { $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $load = Get-AgentObjectValue $Data "loadPerCore" $null + $cpuHost = Get-AgentObjectValue $Data "cpuHostPercent" $null $mem = Get-AgentObjectValue $Data "memAvailablePercent" $null + $swap = Get-AgentObjectValue $Data "swapUsedPercent" $null $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null $metricSource = Get-AgentObjectValue $Data "metricSource" "unknown" @@ -649,6 +651,9 @@ function Format-AgentTelegramLegacyText { } elseif ($reasons -contains "disk_used_high") { $lines += "原因 Reason: 磁碟使用率 $disk percent,已達 critical threshold。" $lines += "Agent99 動作 Action: 先執行 allowlisted cleanup;reboot 不是第一反應。" + } elseif ($reasons -contains "cpu_host_percent_warning" -or $reasons -contains "cpu_host_percent_critical") { + $lines += "原因 Reason: macOS 即時 CPU 使用率=$cpuHost percent;load/core=$load 只保留為趨勢,不單獨觸發 critical。" + $lines += "Agent99 動作 Action: 已收集安全程序摘要;只在即時 CPU 持續達門檻時執行 allowlisted 降載。" } elseif ($reasons -contains "load_per_core_warning" -or $reasons -contains "load_per_core_critical") { $lines += "原因 Reason: CPU load/core=$load。" $lines += "Agent99 動作 Action: 僅使用 allowlisted load reduction;不得任意停止服務。" @@ -667,7 +672,9 @@ function Format-AgentTelegramLegacyText { } $metricParts = @() if ($null -ne $load -and [string]$load -ne "") { $metricParts += "load/core=$load" } + if ($null -ne $cpuHost -and [string]$cpuHost -ne "") { $metricParts += "CPU current=$cpuHost percent" } if ($null -ne $mem -and [string]$mem -ne "") { $metricParts += "memAvail=$mem percent" } + if ($null -ne $swap -and [string]$swap -ne "") { $metricParts += "swap=$swap percent" } if ($null -ne $disk -and [string]$disk -ne "") { $metricParts += "disk=$disk percent" } if ($null -ne $kernelPercpu -and [string]$kernelPercpu -ne "") { $metricParts += "kernel Percpu=$kernelPercpu percent" } if ($null -ne $kernelSUnreclaim -and [string]$kernelSUnreclaim -ne "") { $metricParts += "kernel SUnreclaim=$kernelSUnreclaim percent" } @@ -974,7 +981,9 @@ function Get-AgentIncidentCardModel { } elseif ($EventType -match "^performance_") { $reasons = if ($Data -and $Data.PSObject.Properties["reasons"]) { @($Data.reasons) } else { @() } $load = Get-AgentObjectValue $Data "loadPerCore" $null + $cpuHost = Get-AgentObjectValue $Data "cpuHostPercent" $null $memory = Get-AgentObjectValue $Data "memAvailablePercent" $null + $swap = Get-AgentObjectValue $Data "swapUsedPercent" $null $disk = Get-AgentObjectValue $Data "diskUsedPercent" $null $kernelPercpu = Get-AgentObjectValue $Data "kernelPercpuMemoryPercent" $null $kernelSUnreclaim = Get-AgentObjectValue $Data "kernelSUnreclaimMemoryPercent" $null @@ -1007,6 +1016,12 @@ function Get-AgentIncidentCardModel { [pscustomobject]@{ name = "Kernel SUnreclaim"; value = $kernelSUnreclaim; scale = 100.0; suffix = "%" }, [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" } ) + } elseif ($null -ne $cpuHost) { + @( + [pscustomobject]@{ name = "CPU current"; value = $cpuHost; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "可用記憶體"; value = $memory; scale = 100.0; suffix = "%" }, + [pscustomobject]@{ name = "Swap 使用"; value = $swap; scale = 100.0; suffix = "%" } + ) } else { @( [pscustomobject]@{ name = "CPU load/core"; value = $load; scale = 2.0; suffix = "" }, @@ -2968,6 +2983,8 @@ function Get-PerfThresholds { [pscustomobject]@{ loadPerCoreWarning = if ($perf.loadPerCoreWarning) { [double]$perf.loadPerCoreWarning } else { 0.9 } loadPerCoreCritical = if ($perf.loadPerCoreCritical) { [double]$perf.loadPerCoreCritical } else { 1.5 } + processCpuHostPercentWarning = if ($perf.processCpuHostPercentWarning) { [double]$perf.processCpuHostPercentWarning } else { 25 } + processCpuHostPercentCritical = if ($perf.processCpuHostPercentCritical) { [double]$perf.processCpuHostPercentCritical } else { 50 } memoryAvailablePercentCritical = if ($perf.memoryAvailablePercentCritical) { [double]$perf.memoryAvailablePercentCritical } else { 10 } kernelPercpuMemoryPercentWarning = if ($perf.kernelPercpuMemoryPercentWarning) { [double]$perf.kernelPercpuMemoryPercentWarning } else { 10 } kernelPercpuMemoryPercentCritical = if ($perf.kernelPercpuMemoryPercentCritical) { [double]$perf.kernelPercpuMemoryPercentCritical } else { 20 } @@ -3049,6 +3066,10 @@ df -P / 2>/dev/null $text = $readback.output $cores = 1 $load1 = $null + $metricOs = "unknown" + $cpuHostPercent = $null + $swapUsedPercent = $null + $inlineProcessSamples = @() $memAvailablePercent = $null $diskUsedPercent = $null $kernelPercpuMemoryPercent = $null @@ -3058,6 +3079,7 @@ df -P / 2>/dev/null "(?m)^perf_schema=agent99_perf_readback_v1\s+os=(?\S+)\s+cores=(?\d+)\s+load1=(?[0-9]+(?:[\.,][0-9]+)?)\s+mem_available_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+disk_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" ) if ($canonicalPerfMatch.Success) { + $metricOs = [string]$canonicalPerfMatch.Groups["os"].Value $cores = [int]$canonicalPerfMatch.Groups["cores"].Value $load1 = Convert-AgentDouble $canonicalPerfMatch.Groups["load"].Value $memAvailablePercent = Convert-AgentDouble $canonicalPerfMatch.Groups["memory"].Value @@ -3092,6 +3114,18 @@ df -P / 2>/dev/null if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } } } + $darwinPressureMatch = [regex]::Match( + $text, + "(?m)^darwin_pressure_schema=agent99_darwin_pressure_v1\s+cpu_host_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+swap_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" + ) + if ($metricOs -eq "darwin" -and $darwinPressureMatch.Success) { + $cpuHostPercent = Convert-AgentDouble $darwinPressureMatch.Groups["cpu"].Value + $swapUsedPercent = Convert-AgentDouble $darwinPressureMatch.Groups["swap"].Value + $inlineProcessSamples = @([regex]::Matches( + $text, + "(?m)^darwin_process_sample_v1\s+pid=\d+\s+ppid=\d+\s+cpu_percent=[0-9]+(?:[\.,][0-9]+)?\s+memory_percent=[0-9]+(?:[\.,][0-9]+)?\s+rss_kib=\d+\s+name=[A-Za-z0-9._-]+\s*$" + ) | ForEach-Object { $_.Value.Trim() } | Select-Object -First 8) + } if ($nodeExporterReadback.ok) { if ($nodeExporterReadback.PSObject.Properties["kernelPercpuMemoryPercent"]) { $kernelPercpuMemoryPercent = $nodeExporterReadback.kernelPercpuMemoryPercent @@ -3121,7 +3155,15 @@ df -P / 2>/dev/null $severity = "warning" $reasons += "perf_readback_failed" } - if ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreCritical) { + if ($metricOs -eq "darwin" -and $null -ne $cpuHostPercent) { + if ($cpuHostPercent -ge $thresholds.processCpuHostPercentCritical) { + $severity = "critical" + $reasons += "cpu_host_percent_critical" + } elseif ($cpuHostPercent -ge $thresholds.processCpuHostPercentWarning -and $severity -ne "critical") { + $severity = "warning" + $reasons += "cpu_host_percent_warning" + } + } elseif ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreCritical) { $severity = "critical" $reasons += "load_per_core_critical" } elseif ($null -ne $loadPerCore -and $loadPerCore -ge $thresholds.loadPerCoreWarning -and $severity -ne "critical") { @@ -3159,7 +3201,7 @@ df -P / 2>/dev/null if ($Config.performance -and $Config.performance.PSObject.Properties["topCpuTimeoutSeconds"]) { $topCpuTimeoutSeconds = [int]$Config.performance.topCpuTimeoutSeconds } - $processDiagnosticReason = [bool](@($reasons | Where-Object { $_ -match '^(load_per_core|memory_available|kernel_)' }).Count -gt 0) + $processDiagnosticReason = [bool](@($reasons | Where-Object { $_ -match '^(load_per_core|cpu_host_percent|memory_available|kernel_)' }).Count -gt 0) $shouldCaptureTopCpu = $processDiagnosticReason -and (($severity -eq "critical") -or ($severity -eq "warning" -and $captureTopCpuOnWarning)) if ($shouldCaptureTopCpu -and $readback.ok) { $diagnosticDue = $true @@ -3171,7 +3213,14 @@ df -P / 2>/dev/null $diagnosticDue = $true } } - if ($diagnosticDue) { + if ($inlineProcessSamples.Count -gt 0) { + $diagnosticNow = [DateTimeOffset]::Now.ToString("o") + $diagnosticLastAttemptAt = $diagnosticNow + $diagnosticCapturedAt = $diagnosticNow + $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } + $topCpu = @($inlineProcessSamples) + $diagnosticStatus = "captured_inline" + } elseif ($diagnosticDue) { $diagnosticNow = [DateTimeOffset]::Now.ToString("o") $diagnosticLastAttemptAt = $diagnosticNow $diagnosticKind = if (@($reasons | Where-Object { $_ -match '^(memory_available|kernel_)' }).Count -gt 0) { "top_rss" } else { "top_cpu" } @@ -3203,6 +3252,9 @@ df -P / 2>/dev/null cores = $cores load1 = $load1 loadPerCore = $loadPerCore + metricOs = $metricOs + cpuHostPercent = $cpuHostPercent + swapUsedPercent = $swapUsedPercent memAvailablePercent = $memAvailablePercent diskUsedPercent = $diskUsedPercent kernelPercpuMemoryPercent = $kernelPercpuMemoryPercent @@ -7105,13 +7157,27 @@ function Invoke-AgentExternalHostRecovery { $targetHost = [string](Get-AgentObjectValue $externalHost "host" "") $required = [bool](-not $externalHost.PSObject.Properties["required"] -or (Convert-AgentBool $externalHost.required)) $recoveryAction = [string](Get-AgentObjectValue $externalHost "recoveryAction" "wake_on_lan") + $failureBackoffMinutes = [math]::Max(5, [math]::Min(240, [int](Get-AgentObjectValue $externalHost "failureBackoffMinutes" 30))) + $safeName = [regex]::Replace($name, "[^A-Za-z0-9_.-]", "_") + $failureMarkerPath = Join-Path $EvidenceDir "external-host-recovery-$safeName.failure.marker" + $priorFailureMarkerPresent = [bool](Test-Path -LiteralPath $failureMarkerPath -PathType Leaf) $before = Test-AgentExternalHostReadiness $externalHost $after = $before $actionAttempted = $false $actionReceipt = $null - $terminal = if ($before.ready) { "idempotent_already_verified_healthy" } else { "observed_unreachable" } + $failureBackoffActive = $false + $failureBackoffAgeMinutes = $null + if (-not $before.ready -and $priorFailureMarkerPresent) { + $failureBackoffAgeMinutes = [math]::Round(((Get-Date) - (Get-Item -LiteralPath $failureMarkerPath).LastWriteTime).TotalMinutes, 1) + $failureBackoffActive = [bool]($failureBackoffAgeMinutes -lt $failureBackoffMinutes) + } + $terminal = if ($before.ready -and $priorFailureMarkerPresent) { "recovered_during_backoff_verified" } elseif ($before.ready) { "idempotent_already_verified_healthy" } else { "observed_unreachable" } - if (-not $before.ready -and -not $ControlledApply) { + if ($before.ready) { + Remove-Item -LiteralPath $failureMarkerPath -Force -ErrorAction SilentlyContinue | Out-Null + } elseif ($failureBackoffActive) { + $terminal = "failure_backoff_active" + } elseif (-not $ControlledApply) { $terminal = "controlled_apply_required" } elseif (-not $before.ready -and $recoveryAction -ne "wake_on_lan") { $terminal = "unsupported_recovery_action" @@ -7128,6 +7194,11 @@ function Invoke-AgentExternalHostRecovery { $after = Test-AgentExternalHostReadiness $externalHost } $terminal = if ($after.ready) { "verified_ready" } elseif (-not $actionReceipt.ok) { "wol_transport_failed" } else { "wake_sent_not_verified" } + if ($after.ready) { + Remove-Item -LiteralPath $failureMarkerPath -Force -ErrorAction SilentlyContinue | Out-Null + } else { + Set-Content -LiteralPath $failureMarkerPath -Value (Get-Date -Format o) -Encoding UTF8 + } } $result = [pscustomobject]@{ @@ -7140,17 +7211,27 @@ function Invoke-AgentExternalHostRecovery { actionAttempted = $actionAttempted runtimeWritePerformed = [bool]($actionAttempted -and $actionReceipt -and $actionReceipt.sentPackets -gt 0) changed = [bool]($actionAttempted -and -not $before.ready -and $after.ready) + recoveryObserved = [bool]($before.ready -and $priorFailureMarkerPresent) verified = [bool]$after.ready before = $before actionReceipt = $actionReceipt after = $after terminal = $terminal + failureBackoffMinutes = $failureBackoffMinutes + failureBackoffActive = $failureBackoffActive + failureBackoffAgeMinutes = $failureBackoffAgeMinutes + failureMarkerRef = Convert-AgentPublicReceiptLeaf $failureMarkerPath rollback = "not_applicable_magic_packet_is_stateless" verifier = "icmp_and_required_tcp_ports" prohibitedActions = @("host_reboot", "power_cut", "service_restart", "secret_read", "firewall_change", "database_write") } - if ($result.changed) { - Record-AgentEvent "external_host_recovered" "info" "$name 已由 Agent99 WOL 喚醒並通過 ping/TCP verifier。" $result -Alert + if ($result.changed -or $result.recoveryObserved) { + $recoveryMessage = if ($result.changed) { + "$name 已由 Agent99 WOL 喚醒並通過 ping/TCP verifier。" + } else { + "$name 已在 WOL backoff 期間恢復,並通過 ping/TCP verifier。" + } + Record-AgentEvent "external_host_recovered" "info" $recoveryMessage $result -Alert } elseif ($actionAttempted -and -not $result.verified) { Record-AgentEvent "external_host_recovery_failed" "critical" "$name 已執行 WOL,但仍未通過 ping/TCP verifier。" $result -Alert } diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index 5402d50b8..200fbdbd9 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -371,6 +371,7 @@ try { packetBursts = 5 verifyTimeoutSeconds = 180 verifyIntervalSeconds = 5 + failureBackoffMinutes = 30 verifyTcpPorts = @(22) enabled = $true required = $true @@ -390,6 +391,7 @@ try { Add-DefaultProperty $host111External "packetBursts" 5 Add-DefaultProperty $host111External "verifyTimeoutSeconds" 180 Add-DefaultProperty $host111External "verifyIntervalSeconds" 5 + Add-DefaultProperty $host111External "failureBackoffMinutes" 30 Add-DefaultProperty $host111External "verifyTcpPorts" @(22) Add-DefaultProperty $host111External "enabled" $true Add-DefaultProperty $host111External "required" $true @@ -538,6 +540,8 @@ try { foreach ($performancePolicyName in @( "nodeExporterTimeoutSeconds", "topCpuDiagnosticCooldownMinutes", + "processCpuHostPercentWarning", + "processCpuHostPercentCritical", "kernelPercpuMemoryPercentWarning", "kernelPercpuMemoryPercentCritical", "kernelSUnreclaimMemoryPercentCritical" @@ -598,6 +602,8 @@ try { $nodeExporterRoutesReady -and [int]$persistedConfig.performance.nodeExporterTimeoutSeconds -eq 8 -and [int]$persistedConfig.performance.topCpuDiagnosticCooldownMinutes -eq 15 -and + [int]$persistedConfig.performance.processCpuHostPercentWarning -eq 25 -and + [int]$persistedConfig.performance.processCpuHostPercentCritical -eq 50 -and [int]$persistedConfig.performance.kernelPercpuMemoryPercentWarning -eq 10 -and [int]$persistedConfig.performance.kernelPercpuMemoryPercentCritical -eq 20 -and [int]$persistedConfig.performance.kernelSUnreclaimMemoryPercentCritical -eq 25 @@ -618,6 +624,7 @@ try { [bool]$persistedHost111External[0].enabled -and [bool]$persistedHost111External[0].required -and @($persistedHost111External[0].macAddresses).Count -ge 1 -and + [int]$persistedHost111External[0].failureBackoffMinutes -eq 30 -and 22 -in @($persistedHost111External[0].verifyTcpPorts | ForEach-Object { [int]$_ }) ) if ( diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 4533279a3..2f9001430 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -79,6 +79,7 @@ "packetBursts": 5, "verifyTimeoutSeconds": 180, "verifyIntervalSeconds": 5, + "failureBackoffMinutes": 30, "verifyTcpPorts": [ 22 ], @@ -241,6 +242,8 @@ }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, + "processCpuHostPercentWarning": 25, + "processCpuHostPercentCritical": 50, "memoryAvailablePercentCritical": 10, "kernelPercpuMemoryPercentWarning": 10, "kernelPercpuMemoryPercentCritical": 20, diff --git a/agent99.config.example.json b/agent99.config.example.json index e8e664216..52eb3558a 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -73,6 +73,7 @@ "packetBursts": 5, "verifyTimeoutSeconds": 180, "verifyIntervalSeconds": 5, + "failureBackoffMinutes": 30, "verifyTcpPorts": [ 22 ], @@ -230,6 +231,8 @@ }, "loadPerCoreWarning": 0.9, "loadPerCoreCritical": 1.5, + "processCpuHostPercentWarning": 25, + "processCpuHostPercentCritical": 50, "memoryAvailablePercentCritical": 10, "kernelPercpuMemoryPercentWarning": 10, "kernelPercpuMemoryPercentCritical": 20, diff --git a/apps/api/src/services/awooop_ansible_audit_service.py b/apps/api/src/services/awooop_ansible_audit_service.py index 356d9ceea..e7c6f7d85 100644 --- a/apps/api/src/services/awooop_ansible_audit_service.py +++ b/apps/api/src/services/awooop_ansible_audit_service.py @@ -1383,6 +1383,7 @@ def _incident_public_dict(incident: Any) -> dict[str, Any]: return { "incident_id": getattr(incident, "incident_id", None), "project_id": getattr(incident, "project_id", None), + "status": getattr(getattr(incident, "status", None), "value", None), "alertname": getattr(incident, "alertname", None), "alert_category": getattr(incident, "alert_category", None), "notification_type": getattr(incident, "notification_type", None), @@ -1392,6 +1393,187 @@ def _incident_public_dict(incident: Any) -> dict[str, Any]: } +_CANONICAL_INCIDENT_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,30}$") +_CANONICAL_PROJECT_ID_RE = re.compile(r"^[A-Za-z0-9_.:-]{1,64}$") + + +def _canonical_incident_status(value: Any) -> str: + normalized = str(getattr(value, "value", value) or "").strip().upper() + return ( + normalized + if normalized + in {"INVESTIGATING", "MITIGATING", "RESOLVED", "CLOSED", "ESCALATED"} + else "INVESTIGATING" + ) + + +def _canonical_incident_severity(value: Any) -> str: + normalized = str(getattr(value, "value", value) or "").strip().upper() + return { + "CRITICAL": "P0", + "HIGH": "P1", + "WARNING": "P2", + "MEDIUM": "P2", + "LOW": "P3", + "INFO": "P3", + }.get( + normalized, + normalized if normalized in {"P0", "P1", "P2", "P3"} else "P2", + ) + + +def _canonical_incident_ledger_params( + incident_payload: dict[str, Any], + *, + automation_run_id: str, +) -> dict[str, Any] | None: + incident_id = str(incident_payload.get("incident_id") or "").strip() + project_id = str(incident_payload.get("project_id") or "awoooi").strip() + if ( + _CANONICAL_INCIDENT_ID_RE.fullmatch(incident_id) is None + or _CANONICAL_PROJECT_ID_RE.fullmatch(project_id) is None + ): + return None + + signals = [ + signal + for signal in incident_payload.get("signals") or [] + if isinstance(signal, dict) + ][:20] + affected_services = [ + str(value)[:100] + for value in incident_payload.get("affected_services") or [] + if str(value).strip() + ][:50] + notification_type = str( + incident_payload.get("notification_type") or "" + ).strip() + if len(notification_type) > 10: + notification_type = "" + decision_chain = { + "schema_version": "ansible_candidate_incident_ingress_v1", + "trace_id": str( + incident_payload.get("trace_id") or automation_run_id + ), + "run_id": str(incident_payload.get("run_id") or automation_run_id), + "work_item_id": str(incident_payload.get("work_item_id") or ""), + "source_receipt_ref": str( + incident_payload.get("source_receipt_ref") or "" + ), + "asset_scope_aliases": [ + str(value)[:100] + for value in incident_payload.get("asset_scope_aliases") or [] + if str(value).strip() + ][:20], + "normalized_asset_identity": True, + "candidate_runtime_apply_executed": False, + } + return { + "incident_id": incident_id, + "project_id": project_id, + "status": _canonical_incident_status(incident_payload.get("status")), + "severity": _canonical_incident_severity( + incident_payload.get("severity") + ), + "signals": json.dumps(signals, ensure_ascii=False), + "affected_services": json.dumps( + affected_services, + ensure_ascii=False, + ), + "decision_chain": json.dumps(decision_chain, ensure_ascii=False), + "alertname": str(incident_payload.get("alertname") or "")[:100] + or None, + "notification_type": notification_type or None, + "alert_category": str( + incident_payload.get("alert_category") or "" + )[:50] + or None, + } + + +async def _ensure_ansible_incident_ledger_with_db( + db: Any, + incident_payload: dict[str, Any], + *, + automation_run_id: str, +) -> dict[str, bool]: + params = _canonical_incident_ledger_params( + incident_payload, + automation_run_id=automation_run_id, + ) + if params is None: + return {"ready": False, "inserted": False} + result = await db.execute( + text(""" + WITH inserted AS ( + INSERT INTO incidents ( + incident_id, + project_id, + status, + severity, + signals, + affected_services, + decision_chain, + proposal_ids, + alertname, + notification_type, + alert_category + ) VALUES ( + :incident_id, + :project_id, + CAST(:status AS incidentstatus), + CAST(:severity AS severity), + CAST(:signals AS json), + CAST(:affected_services AS json), + CAST(:decision_chain AS json), + CAST('[]' AS json), + :alertname, + :notification_type, + :alert_category + ) + ON CONFLICT (incident_id) DO NOTHING + RETURNING incident_id + ) + SELECT + EXISTS (SELECT 1 FROM inserted) AS inserted, + EXISTS ( + SELECT 1 + FROM incidents incident + WHERE incident.incident_id = :incident_id + AND incident.project_id = :project_id + ) AS ready + """), + params, + ) + row = result.mappings().one_or_none() + return { + "ready": bool(row and row.get("ready") is True), + "inserted": bool(row and row.get("inserted") is True), + } + + +async def ensure_ansible_incident_ledger( + *, + incident: Any, + automation_run_id: str, + project_id: str | None = None, +) -> dict[str, bool]: + """Ensure a controlled candidate has a durable canonical incident row.""" + + incident_payload = dict(_incident_public_dict(incident)) + if project_id: + incident_payload["project_id"] = project_id + effective_project_id = str( + incident_payload.get("project_id") or "awoooi" + ) + async with get_db_context(effective_project_id) as db: + return await _ensure_ansible_incident_ledger_with_db( + db, + incident_payload, + automation_run_id=automation_run_id, + ) + + def build_ansible_decision_audit_payload( *, incident: Any, @@ -1734,6 +1916,8 @@ async def record_ansible_decision_audit( except (TypeError, ValueError): candidate_op_id = str(uuid4()) payload["input"]["automation_run_id"] = candidate_op_id + incident_payload = dict(_incident_public_dict(incident)) + incident_payload["project_id"] = project_id terminal_candidate_sql = verified_ansible_candidate_terminal_sql("candidate") target_route_generation = str( payload["input"].get("target_route_generation") or "" @@ -1764,6 +1948,19 @@ async def record_ansible_decision_audit( ] }, ) + incident_ledger = await _ensure_ansible_incident_ledger_with_db( + db, + incident_payload, + automation_run_id=candidate_op_id, + ) + if incident_ledger["ready"] is not True: + logger.warning( + "ansible_candidate_incident_ledger_missing", + incident_id=incident_id, + project_id=project_id, + automation_run_id=candidate_op_id, + ) + return False latest_result = await db.execute( text(f""" SELECT diff --git a/apps/api/src/services/awooop_ansible_check_mode_service.py b/apps/api/src/services/awooop_ansible_check_mode_service.py index 7e5ffa387..5e555f5d4 100644 --- a/apps/api/src/services/awooop_ansible_check_mode_service.py +++ b/apps/api/src/services/awooop_ansible_check_mode_service.py @@ -32,6 +32,7 @@ from src.services.ai_automation_runtime_contract import ( AI_AUTOMATION_STAGE_RECEIPT_SCHEMA_VERSION, ) from src.services.awooop_ansible_audit_service import ( + ensure_ansible_incident_ledger, get_ansible_catalog_candidates, get_ansible_catalog_item, ) @@ -40,6 +41,9 @@ from src.services.awooop_ansible_learning_writeback import ( ensure_ansible_rag_writeback, record_ansible_playbook_trust_writeback, ) +from src.services.awooop_ansible_post_verifier import ( + SCHEMA_VERSION as ANSIBLE_POST_VERIFIER_SCHEMA_VERSION, +) from src.services.awooop_ansible_post_verifier import ( run_ansible_asset_post_verifier, ) @@ -1252,6 +1256,134 @@ def _claim_from_apply_operation_row(row: dict[str, Any]) -> tuple[AnsibleCheckMo return claim, result +def _historical_incident_payload_for_claim( + claim: AnsibleCheckModeClaim, + *, + project_id: str, +) -> dict[str, Any]: + target_selector = claim.input_payload.get("target_selector") + if not isinstance(target_selector, Mapping): + target_selector = {} + typed_route = claim.input_payload.get("typed_target_route") + if not isinstance(typed_route, Mapping): + typed_route = {} + canonical_asset_id = str( + claim.input_payload.get("canonical_asset_id") + or target_selector.get("canonical_asset_id") + or typed_route.get("canonical_asset_id") + or "" + ) + alertname = str(claim.input_payload.get("alertname") or "").strip() + if ( + not alertname + and claim.catalog_id == "ansible:wazuh-manager-posture-readback" + ): + alertname = "WazuhManagerPostureScheduled" + if not alertname: + alertname = f"AnsibleControlledApply:{claim.catalog_id}" + affected_services = [ + str(value) + for value in target_selector.get("affected_services") or [] + if str(value).strip() + ] + if not affected_services and canonical_asset_id: + affected_services = [canonical_asset_id] + automation_run_id = _automation_run_id_for_claim(claim) + return { + "incident_id": claim.incident_id, + "project_id": project_id, + "status": "INVESTIGATING", + "severity": claim.risk_level, + "alertname": alertname[:100], + "alert_category": str( + claim.input_payload.get("typed_domain") + or typed_route.get("target_kind") + or "automation" + )[:50], + "notification_type": None, + "affected_services": affected_services, + "trace_id": str( + claim.input_payload.get("trace_id") or automation_run_id + ), + "run_id": str( + claim.input_payload.get("run_id") or automation_run_id + ), + "work_item_id": str( + claim.input_payload.get("work_item_id") or "" + ), + "source_receipt_ref": str( + claim.input_payload.get("source_receipt_ref") or "" + ), + "asset_scope_aliases": ( + claim.input_payload.get("asset_scope_aliases") or [] + ), + "signals": [ + { + "alert_name": alertname[:100], + "severity": _canonical_risk_severity(claim.risk_level), + "source": "ansible_executor_ledger_reconciliation", + "labels": { + "catalog_id": claim.catalog_id, + "canonical_asset_id": canonical_asset_id, + }, + "annotations": { + "summary": ( + "canonical incident ledger reconciled from durable " + "controlled apply" + ) + }, + } + ], + } + + +def _canonical_risk_severity(risk_level: str) -> str: + return { + "critical": "P0", + "high": "P1", + "medium": "P2", + "warning": "P2", + "low": "P3", + "info": "P3", + }.get(str(risk_level or "").lower(), "P2") + + +def _durable_post_verifier_receipt_for_backfill( + row: Mapping[str, Any], + claim: AnsibleCheckModeClaim, + *, + apply_op_id: str, +) -> dict[str, Any] | None: + receipt = _json_loads(row.get("durable_verifier_receipt")) + required_count = _int_from_value( + receipt.get("required_postcondition_count"), + default=0, + ) + passed_count = _int_from_value( + receipt.get("passed_postcondition_count"), + default=0, + ) + active_blockers = receipt.get("active_blockers") + if not isinstance(active_blockers, list): + active_blockers = ["invalid_active_blockers"] + checks = ( + receipt.get("schema_version") + == ANSIBLE_POST_VERIFIER_SCHEMA_VERSION, + receipt.get("apply_op_id") == apply_op_id, + receipt.get("automation_run_id") + == _automation_run_id_for_claim(claim), + receipt.get("catalog_id") == claim.catalog_id, + receipt.get("verification_result") == "success", + receipt.get("all_postconditions_passed") is True, + required_count > 0, + passed_count == required_count, + not active_blockers, + receipt.get("writes_on_verify") is False, + receipt.get("executor_returncode") == 0, + ) + return receipt if all(checks) else None + + def _claim_from_stale_check_mode_row( row: dict[str, Any], ) -> AnsibleCheckModeClaim | None: @@ -5556,25 +5688,25 @@ async def _record_apply_post_verifier_terminal( return False -async def _record_post_apply_verifier_and_learning( +async def _resolve_ansible_post_verifier_receipt( claim: AnsibleCheckModeClaim, result: AnsibleRunResult, *, apply_op_id: str, - project_id: str, -) -> dict[str, Any]: - """Persist post-apply verifier evidence and KM learning for Ansible apply.""" - - action_label = _post_apply_action_label(claim, apply_op_id=apply_op_id) + durable_verifier_receipt: Mapping[str, Any] | None = None, +) -> tuple[dict[str, Any], bool]: automation_run_id = _automation_run_id_for_claim(claim) + if durable_verifier_receipt is not None: + return dict(durable_verifier_receipt), True try: - verifier_receipt = await run_ansible_asset_post_verifier( + receipt = await run_ansible_asset_post_verifier( catalog_id=claim.catalog_id, automation_run_id=automation_run_id, apply_op_id=apply_op_id, inventory_hosts=claim.inventory_hosts, executor_returncode=result.returncode, ) + return receipt, False except Exception as exc: logger.warning( "ansible_independent_post_verifier_failed", @@ -5583,8 +5715,8 @@ async def _record_post_apply_verifier_and_learning( apply_op_id=apply_op_id, error_type=type(exc).__name__, ) - verifier_receipt = { - "schema_version": "awoooi_ansible_asset_post_verifier_v1", + return { + "schema_version": ANSIBLE_POST_VERIFIER_SCHEMA_VERSION, "verifier": "asset_specific_read_only_host_postconditions", "independent_source": "broker_ssh_host_runtime_readback", "automation_run_id": automation_run_id, @@ -5602,7 +5734,28 @@ async def _record_post_apply_verifier_and_learning( "executor_returncode_trusted": False, "raw_output_stored": False, "writes_on_verify": False, - } + }, False + + +async def _record_post_apply_verifier_and_learning( + claim: AnsibleCheckModeClaim, + result: AnsibleRunResult, + *, + apply_op_id: str, + project_id: str, + durable_verifier_receipt: Mapping[str, Any] | None = None, +) -> dict[str, Any]: + """Persist post-apply verifier evidence and KM learning for Ansible apply.""" + + action_label = _post_apply_action_label(claim, apply_op_id=apply_op_id) + verifier_receipt, verifier_reused = ( + await _resolve_ansible_post_verifier_receipt( + claim, + result, + apply_op_id=apply_op_id, + durable_verifier_receipt=durable_verifier_receipt, + ) + ) verification_result = _post_apply_verification_result( result, verifier_receipt, @@ -5640,6 +5793,7 @@ async def _record_post_apply_verifier_and_learning( "trust_learning": False, "rag_writeback": False, "runtime_stage_receipts": [], + "durable_verifier_receipt_reused": verifier_reused, } try: @@ -6136,6 +6290,9 @@ async def backfill_missing_auto_repair_execution_receipts_once( "runtime_stage_receipts_written": 0, "incident_closure_written": 0, "telegram_receipt_acknowledged": 0, + "incident_ledger_reconciled": 0, + "incident_ledger_reconcile_failed": 0, + "durable_verifier_receipt_reused": 0, "retry_terminal_projection_scanned": 0, "retry_terminal_projection_written": 0, "retry_terminal_projection_retry_receipt_written": 0, @@ -6207,6 +6364,28 @@ async def backfill_missing_auto_repair_execution_receipts_once( AND verifier.post_execution_state ->> 'apply_op_id' = apply.op_id::text AND verifier.verification_result = 'success' ) AS verifier_ready, + ( + SELECT verifier.post_execution_state + FROM incident_evidence verifier + WHERE verifier.incident_id = coalesce( + apply.incident_id::text, + apply.input ->> 'incident_id' + ) + AND verifier.post_execution_state ->> 'apply_op_id' + = apply.op_id::text + AND verifier.verification_result = 'success' + ORDER BY verifier.collected_at DESC + LIMIT 1 + ) AS durable_verifier_receipt, + EXISTS ( + SELECT 1 + FROM incidents incident + WHERE incident.incident_id = coalesce( + apply.incident_id::text, + apply.input ->> 'incident_id' + ) + AND incident.project_id = :project_id + ) AS incident_ledger_ready, apply.created_at FROM automation_operation_log apply WHERE apply.operation_type = 'ansible_apply_executed' @@ -6343,12 +6522,38 @@ async def backfill_missing_auto_repair_execution_receipts_once( stats["skipped"] += 1 continue claim, result = reconstructed + if row.get("incident_ledger_ready") is False: + incident_ledger = await ensure_ansible_incident_ledger( + incident=_historical_incident_payload_for_claim( + claim, + project_id=project_id, + ), + automation_run_id=_automation_run_id_for_claim(claim), + project_id=project_id, + ) + if incident_ledger.get("ready") is not True: + stats["incident_ledger_reconcile_failed"] += 1 + stats["skipped"] += 1 + continue + if incident_ledger.get("inserted") is True: + stats["incident_ledger_reconciled"] += 1 + apply_op_id = str(row.get("op_id") or "") + durable_verifier_receipt = ( + _durable_post_verifier_receipt_for_backfill( + row, + claim, + apply_op_id=apply_op_id, + ) + ) writeback = await _record_post_apply_verifier_and_learning( claim, result, - apply_op_id=str(row.get("op_id") or ""), + apply_op_id=apply_op_id, project_id=project_id, + durable_verifier_receipt=durable_verifier_receipt, ) + if writeback.get("durable_verifier_receipt_reused") is True: + stats["durable_verifier_receipt_reused"] += 1 verified_result = replace( result, post_verifier_passed=( diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index db4cf9002..7a7be12c5 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -205,6 +205,7 @@ def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None assert host111["enabled"] is True assert host111["verifyTcpPorts"] == [22] assert host111["verifyTimeoutSeconds"] <= 300 + assert host111["failureBackoffMinutes"] == 30 host112 = next(item for item in config["vms"] if item["host"] == "192.168.0.112") assert host112["vmx"] == r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" @@ -220,6 +221,13 @@ def test_agent99_host111_recovery_is_allowlisted_and_independently_verified() -> assert "Invoke-AgentExternalHostRecovery" in function assert "Test-AgentExternalHostReadiness" in function assert 'terminal = if ($after.ready) { "verified_ready" }' in function + assert 'terminal = "failure_backoff_active"' in function + assert '"recovered_during_backoff_verified"' in function + assert "recoveryObserved" in function + assert "if ($result.changed -or $result.recoveryObserved)" in function + assert "external-host-recovery-$safeName.failure.marker" in function + assert "failureBackoffMinutes" in function + assert "failureBackoffActive" in function assert 'verifier = "icmp_and_required_tcp_ports"' in function for prohibited in ( "host_reboot", diff --git a/apps/api/tests/test_ansible_incident_ledger_churn_guard.py b/apps/api/tests/test_ansible_incident_ledger_churn_guard.py new file mode 100644 index 000000000..fbbdc8e11 --- /dev/null +++ b/apps/api/tests/test_ansible_incident_ledger_churn_guard.py @@ -0,0 +1,317 @@ +from __future__ import annotations + +import json +import os +from contextlib import asynccontextmanager +from unittest.mock import AsyncMock + +import pytest + +os.environ.setdefault( + "DATABASE_URL", + "postgresql+asyncpg://test:test@localhost/test", +) + +from src.jobs.awooop_ansible_candidate_backfill_job import ( # noqa: E402 + _wazuh_posture_incident, +) +from src.services import awooop_ansible_audit_service as audit # noqa: E402 +from src.services import awooop_ansible_check_mode_service as service # noqa: E402 + + +class _MappingResult: + def __init__(self, row: dict | None = None) -> None: + self._row = row + + def mappings(self) -> _MappingResult: + return self + + def one_or_none(self) -> dict | None: + return self._row + + +class _RowsResult: + def __init__(self, rows: list[dict]) -> None: + self._rows = rows + + def mappings(self) -> _RowsResult: + return self + + def all(self) -> list[dict]: + return self._rows + + +class _SequenceDB: + def __init__(self, *results) -> None: + self._results = list(results) + self.statements: list[str] = [] + self.parameters: list[dict] = [] + + async def execute(self, statement, parameters=None): + self.statements.append(str(statement)) + self.parameters.append(dict(parameters or {})) + return self._results.pop(0) + + +def _wazuh_claim() -> service.AnsibleCheckModeClaim: + run_id = "00000000-0000-0000-0000-000000000202" + return service.AnsibleCheckModeClaim( + op_id="00000000-0000-0000-0000-000000000201", + source_candidate_op_id="00000000-0000-0000-0000-000000000200", + incident_id="IWZ-POSTURE-2026071506", + catalog_id="ansible:wazuh-manager-posture-readback", + playbook_path="infra/ansible/playbooks/wazuh-manager-posture-readback.yml", + apply_playbook_path=( + "infra/ansible/playbooks/wazuh-manager-posture-readback.yml" + ), + inventory_hosts=("host_112",), + risk_level="low", + input_payload={ + "automation_run_id": run_id, + "trace_id": run_id, + "run_id": run_id, + "work_item_id": "P0-03-WAZUH-MANAGER-POSTURE", + "source_receipt_ref": "scheduled-wazuh-manager-posture:2026071506", + "asset_scope_aliases": ["security_manager_primary"], + "canonical_asset_id": "service:wazuh-manager", + "typed_domain": "windows_vmware_control_plane", + "target_selector": { + "affected_services": ["wazuh-manager", "siem"], + "canonical_asset_id": "service:wazuh-manager", + }, + }, + ) + + +def _durable_verifier_receipt( + claim: service.AnsibleCheckModeClaim, + *, + apply_op_id: str, +) -> dict: + return { + "schema_version": service.ANSIBLE_POST_VERIFIER_SCHEMA_VERSION, + "verifier": "asset_specific_read_only_host_postconditions", + "independent_source": "broker_ssh_host_runtime_readback", + "automation_run_id": claim.input_payload["automation_run_id"], + "apply_op_id": apply_op_id, + "catalog_id": claim.catalog_id, + "verification_result": "success", + "all_postconditions_passed": True, + "required_postcondition_count": 2, + "passed_postcondition_count": 2, + "postconditions": [ + {"condition_id": "unit_loaded", "passed": True}, + {"condition_id": "runtime", "passed": True}, + ], + "active_blockers": [], + "executor_returncode": 0, + "executor_returncode_trusted": False, + "raw_output_stored": False, + "writes_on_verify": False, + } + + +def test_wazuh_candidate_builds_bounded_canonical_incident_ledger() -> None: + run_id = "00000000-0000-0000-0000-000000000202" + incident = _wazuh_posture_incident( + automation_run_id=run_id, + bucket_ref="2026071506", + ) + + params = audit._canonical_incident_ledger_params( + incident, + automation_run_id=run_id, + ) + + assert params is not None + assert params["incident_id"] == "IWZ-POSTURE-2026071506" + assert params["status"] == "INVESTIGATING" + assert params["severity"] == "P3" + assert params["notification_type"] is None + decision_chain = json.loads(params["decision_chain"]) + assert decision_chain["trace_id"] == run_id + assert decision_chain["work_item_id"] == "P0-03-WAZUH-MANAGER-POSTURE" + assert decision_chain["normalized_asset_identity"] is True + assert decision_chain["candidate_runtime_apply_executed"] is False + + +@pytest.mark.asyncio +async def test_incident_ledger_insert_is_idempotent_and_read_back() -> None: + db = _SequenceDB(_MappingResult({"inserted": True, "ready": True})) + run_id = "00000000-0000-0000-0000-000000000202" + incident = _wazuh_posture_incident( + automation_run_id=run_id, + bucket_ref="2026071506", + ) + + result = await audit._ensure_ansible_incident_ledger_with_db( + db, + incident, + automation_run_id=run_id, + ) + + assert result == {"ready": True, "inserted": True} + assert "ON CONFLICT (incident_id) DO NOTHING" in db.statements[0] + assert "UPDATE incidents" not in db.statements[0] + assert "EXISTS" in db.statements[0] + + +@pytest.mark.asyncio +async def test_durable_verifier_receipt_skips_a_second_ssh_probe( + monkeypatch: pytest.MonkeyPatch, +) -> None: + claim = _wazuh_claim() + apply_op_id = "00000000-0000-0000-0000-000000000203" + receipt = _durable_verifier_receipt(claim, apply_op_id=apply_op_id) + ssh_probe = AsyncMock(side_effect=AssertionError("SSH verifier repeated")) + monkeypatch.setattr(service, "run_ansible_asset_post_verifier", ssh_probe) + + resolved, reused = await service._resolve_ansible_post_verifier_receipt( + claim, + service.AnsibleRunResult( + returncode=0, + stdout="", + stderr="", + duration_ms=5, + ), + apply_op_id=apply_op_id, + durable_verifier_receipt=receipt, + ) + + assert reused is True + assert resolved == receipt + ssh_probe.assert_not_awaited() + + +def test_backfill_accepts_only_same_run_successful_durable_verifier() -> None: + claim = _wazuh_claim() + apply_op_id = "00000000-0000-0000-0000-000000000203" + receipt = _durable_verifier_receipt(claim, apply_op_id=apply_op_id) + + assert ( + service._durable_post_verifier_receipt_for_backfill( + {"durable_verifier_receipt": receipt}, + claim, + apply_op_id=apply_op_id, + ) + == receipt + ) + + wrong_run = {**receipt, "automation_run_id": "wrong-run"} + assert ( + service._durable_post_verifier_receipt_for_backfill( + {"durable_verifier_receipt": wrong_run}, + claim, + apply_op_id=apply_op_id, + ) + is None + ) + + +@pytest.mark.asyncio +async def test_backfill_repairs_orphan_incident_and_reuses_verifier( + monkeypatch: pytest.MonkeyPatch, +) -> None: + claim = _wazuh_claim() + apply_op_id = "00000000-0000-0000-0000-000000000203" + receipt = _durable_verifier_receipt(claim, apply_op_id=apply_op_id) + row = { + "op_id": apply_op_id, + "parent_op_id": claim.op_id, + "incident_id": claim.incident_id, + "input": { + **claim.input_payload, + "catalog_id": claim.catalog_id, + "source_candidate_op_id": claim.source_candidate_op_id, + "check_mode_op_id": claim.op_id, + "playbook_path": claim.playbook_path, + "apply_playbook_path": claim.apply_playbook_path, + "inventory_hosts": list(claim.inventory_hosts), + "risk_level": claim.risk_level, + }, + "output": {"returncode": 0}, + "dry_run_result": {}, + "error": None, + "duration_ms": 5, + "status": "success", + "incident_ledger_ready": False, + "verifier_ready": True, + "durable_verifier_receipt": receipt, + } + db = _SequenceDB(_MappingResult(), _RowsResult([row])) + + @asynccontextmanager + async def fake_db_context(_project_id: str): + yield db + + monkeypatch.setattr(service, "get_db_context", fake_db_context) + monkeypatch.setattr( + service, + "backfill_missing_retry_terminal_projections_once", + AsyncMock( + return_value={ + "scanned": 0, + "written": 0, + "retry_receipt_written": 0, + "telegram_receipt_acknowledged": 0, + "incident_receipt_written": 0, + "lifecycle_written": 0, + "verified": 0, + "runtime_apply_executed": False, + "error": None, + } + ), + ) + incident_ledger = AsyncMock(return_value={"ready": True, "inserted": True}) + monkeypatch.setattr(service, "ensure_ansible_incident_ledger", incident_ledger) + writeback = AsyncMock( + return_value={ + "verification": True, + "verification_passed": True, + "verification_result": "success", + "learning": True, + "trust_learning": True, + "rag_writeback": True, + "runtime_stage_receipts": [], + "durable_verifier_receipt_reused": True, + } + ) + monkeypatch.setattr( + service, + "_record_post_apply_verifier_and_learning", + writeback, + ) + monkeypatch.setattr( + service, + "_record_auto_repair_execution_receipt", + AsyncMock(return_value=True), + ) + monkeypatch.setattr( + service, + "_record_timeline_projection_receipt", + AsyncMock(return_value=None), + ) + monkeypatch.setattr( + service, + "_record_runtime_stage_receipts", + AsyncMock(return_value=True), + ) + monkeypatch.setattr( + service, + "_reconcile_verified_apply_closure_projections", + AsyncMock( + return_value={ + "closed": True, + "telegram_receipt_acknowledged": True, + } + ), + ) + + result = await service.backfill_missing_auto_repair_execution_receipts_once(limit=1) + + assert result["incident_ledger_reconciled"] == 1 + assert result["incident_ledger_reconcile_failed"] == 0 + assert result["durable_verifier_receipt_reused"] == 1 + assert result["incident_closure_written"] == 1 + incident_ledger.assert_awaited_once() + assert writeback.await_args.kwargs["durable_verifier_receipt"] == receipt diff --git a/apps/api/tests/test_sre_typed_domain_router.py b/apps/api/tests/test_sre_typed_domain_router.py index c926ce5b5..83884d81a 100644 --- a/apps/api/tests/test_sre_typed_domain_router.py +++ b/apps/api/tests/test_sre_typed_domain_router.py @@ -127,6 +127,9 @@ class _GenerationResult: def first(self): return self._row + def one_or_none(self): + return self._row + def scalar(self): return self._scalar @@ -143,6 +146,10 @@ class _GenerationDb: self.parameters.append(parameters or {}) if "pg_advisory_xact_lock" in sql: return _GenerationResult() + if "WITH inserted AS" in sql and "INSERT INTO incidents" in sql: + return _GenerationResult( + row={"inserted": False, "ready": True} + ) if "SELECT" in sql and "candidate.op_id::text AS op_id" in sql: return _GenerationResult(row=self.latest) if "typed_route_reconciler" in sql: diff --git a/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md index 757e1b5b8..ce2b1c079 100644 --- a/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md +++ b/docs/operations/AGENT99-HOST112-CONTROLLED-RECOVERY-CONTRACT.md @@ -71,3 +71,35 @@ not part of this rolling timer ledger. child deadline. The control plane validates these reserves before attempting Host112 recovery. + +## 2026-07-15 kernel-memory recovery receipt + +Run `AG99-H112-KMEM-20260715-1531` established the production recovery pattern +for Host112 kernel-memory exhaustion: + +1. The check gate required the exact Host112 VMX to be owned by one SYSTEM + `vmware-vmx` process, all required guest services to be active, + `MemAvailable <= 5%`, either `Percpu >= 20%` or `SUnreclaim >= 25%`, and no + source-121 SSH session during the preceding 15 minutes. +2. The only apply was `vmrun reset soft` from a temporary + SYSTEM scheduled task. The task was removed after terminal readback. No hard + reset, VM stop, host reboot, secret read, firewall change, or other VM action + occurred. +3. The independent verifier required a changed guest boot ID, all eight + required services active, `MemAvailable >= 25%`, `Percpu < 5%`, + `SUnreclaim < 10%`, no source-121 sessions, and unchanged SYSTEM ownership of + the exact VMX. + +Production before/after evidence: + +- Boot ID changed from `eec29776-d98b-4e2c-9972-a73eb2bb6298` to + `8423d3d2-ed94-4bec-9f3e-f80b17e718a4`. +- `MemAvailable` recovered from `3.39%` to `67.02%`. +- `Percpu` recovered from `32.61%` to `0.30%`. +- `SUnreclaim` recovered from `36.13%` to `0.96%`. +- The durable verifier terminal was `verified_recovered`; source-121 sessions + remained zero. + +The first check at 15:00 failed closed because three source-121 sessions were +still inside the 15-minute window. A later clean window was required; deleting +locks, killing `vmware-vmx`, or forcing a reset was never an acceptable bypass. diff --git a/docs/operations/agent99-host112-host111-recovery-readback-2026-07-15.snapshot.json b/docs/operations/agent99-host112-host111-recovery-readback-2026-07-15.snapshot.json new file mode 100644 index 000000000..2dbfd0bc8 --- /dev/null +++ b/docs/operations/agent99-host112-host111-recovery-readback-2026-07-15.snapshot.json @@ -0,0 +1,61 @@ +{ + "schema_version": "agent99_host_recovery_incident_readback_v1", + "generated_at": "2026-07-15T16:49:24+08:00", + "trace_id": "agent99-host112-kernel-memory-20260715-1531", + "run_id": "AG99-H112-KMEM-20260715-1531", + "work_item_id": "AG99-P0-HOST112-KERNEL-MEMORY", + "host112": { + "terminal": "verified_recovered", + "apply": "vmware_tools_soft_guest_reset", + "before_boot_id": "eec29776-d98b-4e2c-9972-a73eb2bb6298", + "after_boot_id": "8423d3d2-ed94-4bec-9f3e-f80b17e718a4", + "before": { + "mem_available_percent": 3.39, + "kernel_percpu_percent": 32.61, + "kernel_sunreclaim_percent": 36.13 + }, + "after": { + "mem_available_percent": 67.02, + "kernel_percpu_percent": 0.30, + "kernel_sunreclaim_percent": 0.96, + "source121_sessions_15m": 0, + "required_services_active": 8 + }, + "prohibited_actions_performed": [] + }, + "service_recovery": { + "evidence_ref": "agent99-Recover-20260715-160810.json", + "scope": "service_recovery", + "elapsed_seconds": 14.4, + "completed": true, + "within_slo": true, + "runtime_write_performed": false, + "k3s_deployments": { + "awoooi-api": "2/2", + "awoooi-web": "2/2", + "awoooi-worker": "1/1" + }, + "public_failures": 0, + "fresh_all_host_reboot_slo_claimed": false + }, + "host111": { + "asset_type": "external_macos_host", + "terminal": "wake_sent_not_verified", + "reachable": false, + "wol_attempts_observed": 3, + "packets_per_attempt": 40, + "failure_backoff_minutes": 30, + "latest_runtime_evidence_ref": "agent99-Recover-20260715-164157.json", + "telegram_message_id": "39483", + "safe_next_action": "retain incident, suppress duplicate WOL, verify power/network and exact host identity before recovery close" + }, + "source_change": { + "gitea_main_deployed": false, + "feature_branch_only": true, + "windows_ast_error_count": 0, + "focused_tests_passed": 82, + "github_used": false + }, + "secret_value_read": false, + "raw_log_stored": false +} diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 162aca170..f5e788b9e 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -2711,3 +2711,37 @@ Prometheus 必須載入: - 不得為了產生 receipt 而送測試 Telegram、呼叫 Bot API、改 Alertmanager receiver route、改 silence 或 workflow_dispatch。 - 不得讀取 token / chat raw payload / secret volume。 - 不得用 receipt marker 取代真正 backup freshness、restore drill、Gitea bundle completeness 或 offsite verifier。 + +### 16.7 2026-07-15 Host112 kernel recovery 與 Host111 外部主機失聯 + +本輪先修正 `121 -> 112` Wazuh verifier ledger 缺口。五筆已有成功 apply / verifier +但缺 canonical incident row 的歷史工作,會被 broker backfill 反覆重跑,每輪最多建立 +10 個 Host112 SSH session。必須先補 incident ledger 並重用同一 durable verifier;不得以 +重啟 112 掩蓋 session churn。 + +Host112 的 controlled recovery 只在 source-121 連續 15 分鐘零 session 後放行。production +run `AG99-H112-KMEM-20260715-1531` 執行單一 VMware Tools soft guest reset,boot ID +確實改變,八個必要服務均 active,`MemAvailable / Percpu / SUnreclaim` 從 +`3.39 / 32.61 / 36.13%` 恢復到 `67.02 / 0.30 / 0.96%`。其後 Agent99 +service-recovery verifier 在 14.4 秒內完成,Harbor、K3s、188 edge、public routes 均通過, +`awoooi-api/web/worker` 為 `2/2, 2/2, 1/1`。 + +這一輪不是 fresh all-host reboot event,因此只能宣稱 service recovery within 10 minutes, +不能宣稱新的全主機重啟 SLA。scorecard 的 `fresh_all_host_reboot_event_missing` 是 drill +證據缺口,不是服務故障,也不得為了清除它而重啟主機。 + +同輪 Host111 macOS 先發生短暫 Contacts/system diagnostic CPU burst,接著 `.111` 與 +Bonjour `.80` 位址均失聯。Agent99 已自動 Detect 並執行 allowlisted WOL;每次送出 40 +個 magic packets,但 ping/22 verifier 仍失敗,terminal 必須維持 +`wake_sent_not_verified`。後續規則: + +- Darwin 效能使用 `cpu_host_percent` 即時樣本,不以殘留 load average 單獨判定 critical。 +- forced-command readback 同輪輸出安全 top-process 摘要,只含 PID、CPU、memory、RSS、 + executable basename;禁止 command arguments、env、prompt、token 或 secret。 +- WOL 失敗後進入 30 分鐘 failure backoff;相同 lifecycle 只更新 incident,不重送 WOL + 或 Telegram。 +- Host111 是外部 Mac,不是 Windows99 VMware guest;沒有 exact VMX 時禁止 VM power + action、刪 `.lck`、殺 `vmware-vmx` 或把它標成已恢復。 + +機器可讀 incident evidence: +`docs/operations/agent99-host112-host111-recovery-readback-2026-07-15.snapshot.json`。 diff --git a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh index 3bc93502c..f00f3b21a 100755 --- a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh +++ b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh @@ -116,7 +116,9 @@ control_path_check() { validate_readback() { local output="$1" grep -Eq '^boot_id=darwin_[0-9]+ uptime_seconds=[0-9]+ systemd_state=darwin_ssh startup_enabled=(enabled|unknown) startup_active=(active|inactive)$' <<<"$output" \ - && grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" + && grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" \ + && grep -Eq '^darwin_pressure_schema=agent99_darwin_pressure_v1 cpu_host_percent=[0-9]+([.][0-9]+)? swap_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" \ + && grep -Eq '^darwin_process_sample_v1 pid=[0-9]+ ppid=[0-9]+ cpu_percent=[0-9]+([.][0-9]+)? memory_percent=[0-9]+([.][0-9]+)? rss_kib=[0-9]+ name=[A-Za-z0-9._-]+$' <<<"$output" } check() { diff --git a/scripts/reboot-recovery/macos-host-boot-readback.sh b/scripts/reboot-recovery/macos-host-boot-readback.sh index 149e09fae..cf9f4cf57 100755 --- a/scripts/reboot-recovery/macos-host-boot-readback.sh +++ b/scripts/reboot-recovery/macos-host-boot-readback.sh @@ -40,6 +40,37 @@ mem_available_percent="$(memory_pressure -Q 2>/dev/null | awk -F': ' ' exit } ')" +process_cpu_sum="$(ps -A -o %cpu= 2>/dev/null | awk '{ sum += $1 } END { printf "%.2f", sum + 0 }')" +cpu_host_percent="$(awk -v total="$process_cpu_sum" -v cores="$logical_cores" 'BEGIN { + if (cores <= 0) { exit 1 } + value = total / cores + if (value > 100) { value = 100 } + printf "%.2f", value +}')" +swap_total_mib="$(sysctl vm.swapusage 2>/dev/null | awk '{ + for (i = 1; i <= NF; i += 1) { + if ($i == "total") { + value = $(i + 2) + gsub(/M/, "", value) + print value + exit + } + } +}')" +swap_used_mib="$(sysctl vm.swapusage 2>/dev/null | awk '{ + for (i = 1; i <= NF; i += 1) { + if ($i == "used") { + value = $(i + 2) + gsub(/M/, "", value) + print value + exit + } + } +}')" +swap_used_percent="$(awk -v used="$swap_used_mib" -v total="$swap_total_mib" 'BEGIN { + if (total <= 0) { printf "0.00"; exit } + printf "%.2f", (used / total) * 100 +}')" disk_used_percent="$(df -Pk / 2>/dev/null | awk ' NR == 2 { value = $5 @@ -52,6 +83,8 @@ disk_used_percent="$(df -Pk / 2>/dev/null | awk ' if [[ ! "$logical_cores" =~ ^[0-9]+$ ]] || (( logical_cores < 1 )) || [[ ! "$load1" =~ ^[0-9]+([.][0-9]+)?$ ]] || [[ ! "$mem_available_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] || + [[ ! "$cpu_host_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] || + [[ ! "$swap_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] || [[ ! "$disk_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]]; then echo "BLOCKER darwin_perf_readback_unavailable" exit 65 @@ -61,3 +94,25 @@ printf 'boot_id=darwin_%s uptime_seconds=%s systemd_state=darwin_ssh startup_ena "$boot_epoch" "$uptime_seconds" "$startup_enabled" "$startup_active" printf 'perf_schema=agent99_perf_readback_v1 os=darwin cores=%s load1=%s mem_available_percent=%s disk_used_percent=%s\n' \ "$logical_cores" "$load1" "$mem_available_percent" "$disk_used_percent" +printf 'darwin_pressure_schema=agent99_darwin_pressure_v1 cpu_host_percent=%s swap_used_percent=%s\n' \ + "$cpu_host_percent" "$swap_used_percent" + +ps -Ao pid=,ppid=,%cpu=,%mem=,rss=,comm= -r 2>/dev/null | awk ' + NR <= 8 { + pid = $1 + ppid = $2 + cpu = $3 + memory = $4 + rss = $5 + name = "" + for (i = 6; i <= NF; i += 1) { + name = name (name == "" ? "" : " ") $i + } + count = split(name, path, "/") + name = path[count] + gsub(/[^A-Za-z0-9._-]/, "_", name) + if (name == "") { name = "unknown" } + printf "darwin_process_sample_v1 pid=%s ppid=%s cpu_percent=%s memory_percent=%s rss_kib=%s name=%s\n", \ + pid, ppid, cpu, memory, rss, name + } +' diff --git a/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py index 747518b37..8484e9229 100644 --- a/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_node_exporter_performance_contract.py @@ -38,6 +38,8 @@ def test_node_exporter_routes_are_explicit_and_identical_in_policy_configs() -> assert performance["nodeExporterUrls"] == expected assert performance["nodeExporterTimeoutSeconds"] == 8 assert performance["topCpuDiagnosticCooldownMinutes"] == 15 + assert performance["processCpuHostPercentWarning"] == 25 + assert performance["processCpuHostPercentCritical"] == 50 assert performance["kernelPercpuMemoryPercentWarning"] == 10 assert performance["kernelPercpuMemoryPercentCritical"] == 20 assert performance["kernelSUnreclaimMemoryPercentCritical"] == 25 @@ -117,6 +119,11 @@ def test_perf_uses_metrics_first_and_ssh_only_as_fallback_or_cooldown_diagnostic assert 'metricSource = if ($nodeExporterReadback.ok) { "node_exporter" }' in perf assert 'diagnosticStatus = "cooldown_suppressed"' in perf assert "topCpuDiagnosticCooldownMinutes" in perf + assert 'if ($metricOs -eq "darwin" -and $null -ne $cpuHostPercent)' in perf + assert '"cpu_host_percent_critical"' in perf + assert '"cpu_host_percent_warning"' in perf + assert 'diagnosticStatus = "captured_inline"' in perf + assert "darwin_process_sample_v1" in perf assert "$diagnosticCooldownMinutes = [math]::Min(120, [math]::Max(5" in perf assert "memory_available_low" in perf assert "kernel_percpu_memory_warning" in perf @@ -145,6 +152,8 @@ def test_deploy_migrates_and_post_verifies_canonical_metric_routes() -> None: "kernelPercpuMemoryPercentWarning", "kernelPercpuMemoryPercentCritical", "kernelSUnreclaimMemoryPercentCritical", + "processCpuHostPercentWarning", + "processCpuHostPercentCritical", ): assert f'Add-DefaultProperty $config.performance "{policy_name}"' in deploy assert policy_name in deploy[deploy.index("$nodeExporterPolicyReady") :] diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index 644f4c4f1..0f264ff98 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -152,6 +152,10 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "boot_id=darwin_%s" in macos_readback assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_readback assert "memory_pressure -Q" in macos_readback + assert "darwin_pressure_schema=agent99_darwin_pressure_v1" in macos_readback + assert "darwin_process_sample_v1" in macos_readback + assert "cpu_host_percent" in macos_readback + assert "swap_used_percent" in macos_readback assert "df -Pk /" in macos_readback assert "127.0.0.1:11434/api/tags" in macos_readback assert "restrict,command=" in macos_installer @@ -159,6 +163,8 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "macos111_readback_schema_incomplete" in macos_installer assert "macos111_control_readback_schema_incomplete" in macos_installer assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_installer + assert "darwin_pressure_schema=agent99_darwin_pressure_v1" in macos_installer + assert "darwin_process_sample_v1" in macos_installer assert "controlled_identity_required" in macos_installer assert "incomplete_control_identity" in macos_installer assert "controlled_run_already_exists" in macos_installer diff --git a/scripts/reboot-recovery/tests/test_windows99_host112_kernel_memory_soft_reboot.py b/scripts/reboot-recovery/tests/test_windows99_host112_kernel_memory_soft_reboot.py new file mode 100644 index 000000000..018dfe87b --- /dev/null +++ b/scripts/reboot-recovery/tests/test_windows99_host112_kernel_memory_soft_reboot.py @@ -0,0 +1,80 @@ +from pathlib import Path + + +ROOT = Path(__file__).resolve().parents[3] +SCRIPT = ( + ROOT + / "scripts" + / "reboot-recovery" + / "windows99-host112-kernel-memory-soft-reboot.ps1" +) + + +def source() -> str: + return SCRIPT.read_text(encoding="utf-8") + + +def test_host112_soft_reboot_is_exact_target_and_same_run_bounded() -> None: + text = source() + + assert '[ValidateSet("Check", "Apply", "Verify")]' in text + assert "S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx" in text + assert 'ExpectedSshTarget = "kali@192.168.0.112"' in text + assert "Assert-ControlIdentity" in text + assert 'gate.traceId -eq $TraceId' in text + assert 'gate.runId -eq $RunId' in text + assert 'gate.workItemId -eq $WorkItemId' in text + assert 'schemaVersion = "windows99_host112_kernel_memory_soft_reboot_v1"' in text + + +def test_check_gate_requires_pressure_services_and_quiet_broker_source() -> None: + text = source() + + assert "$vmxRuntime.exactSystemOwner" in text + assert "$guest.servicesReady" in text + assert "$guest.memAvailablePercent -le 5" in text + assert "$guest.percpuPercent -ge 20" in text + assert "$guest.sunreclaimPercent -ge 25" in text + assert "$guest.source121Sessions15m -eq 0" in text + assert '"check_failed_no_write"' in text + + +def test_apply_allows_only_vmware_soft_reset_without_destructive_fallback() -> None: + text = source() + + assert "& $VmrunPath -T ws reset $VmxPath soft" in text + assert "& $VmrunPath -T ws list" in text + assert '"SYSTEM vmrun does not own the exact Host112 VMX"' in text + assert "exactTargetVisible = $vmrunSeesTarget" in text + assert 'resetMode"] = "soft"' in text + assert '"soft_reset_failed_no_hard_fallback"' in text + assert 'hardResetPerformed = $false' in text + assert 'vmStopPerformed = $false' in text + assert "reset $VmxPath hard" not in text + assert "stop $VmxPath hard" not in text + assert "stop $VmxPath soft" not in text + + +def test_verify_requires_new_boot_services_memory_and_vmx_owner() -> None: + text = source() + + assert '$guest.bootId -ne [string]$apply.beforeBootId' in text + assert "$guest.memAvailablePercent -ge 25" in text + assert "$guest.percpuPercent -lt 5" in text + assert "$guest.sunreclaimPercent -lt 10" in text + assert "$guest.source121Sessions15m -eq 0" in text + assert "$vmxRuntime.exactSystemOwner" in text + assert '"verified_recovered"' in text + assert '"post_verifier_failed_no_hard_fallback"' in text + + +def test_windows_transport_uses_agent99_identity_and_process_timeout() -> None: + text = source() + + assert '"-o", "IdentitiesOnly=yes"' in text + assert '"-o", "BatchMode=yes"' in text + assert "Start-Process" in text + assert "WaitForExit(35000)" in text + assert "taskkill.exe /PID $process.Id /T /F" in text + assert 'rawOutputStored = $false' in text + assert 'secretReadPerformed = $false' in text diff --git a/scripts/reboot-recovery/windows99-host112-kernel-memory-soft-reboot.ps1 b/scripts/reboot-recovery/windows99-host112-kernel-memory-soft-reboot.ps1 new file mode 100644 index 000000000..f136a5bd4 --- /dev/null +++ b/scripts/reboot-recovery/windows99-host112-kernel-memory-soft-reboot.ps1 @@ -0,0 +1,464 @@ +param( + [ValidateSet("Check", "Apply", "Verify")] + [string]$Mode = "Check", + [Parameter(Mandatory = $true)] + [string]$TraceId, + [Parameter(Mandatory = $true)] + [string]$RunId, + [Parameter(Mandatory = $true)] + [string]$WorkItemId, + [string]$GateReceiptPath = "", + [string]$ApplyReceiptPath = "", + [string]$EvidenceDir = "C:\Wooo\Agent99\evidence", + [string]$VmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx", + [string]$VmrunPath = "C:\Program Files (x86)\VMware\VMware Workstation\vmrun.exe", + [string]$SshTarget = "kali@192.168.0.112", + [string]$SshIdentityFile = "C:\Wooo\Agent99\keys\agent99_ed25519", + [int]$VerifyTimeoutSeconds = 420, + [int]$VerifyIntervalSeconds = 10, + [int]$GateMaxAgeMinutes = 10 +) + +$ErrorActionPreference = "Stop" +$IdentityPattern = "^[A-Za-z0-9][A-Za-z0-9._:@+-]{0,127}$" +$ExpectedVmxPath = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" +$ExpectedSshTarget = "kali@192.168.0.112" + +function Assert-ControlIdentity { + foreach ($value in @($TraceId, $RunId, $WorkItemId)) { + if ($value -notmatch $IdentityPattern) { + throw "controlled identity is invalid" + } + } +} + +function Get-SafeId { + param([string]$Value) + return ($Value -replace "[^A-Za-z0-9._-]", "_") +} + +function Write-ControlReceipt { + param( + [string]$Phase, + [System.Collections.IDictionary]$Receipt + ) + + New-Item -ItemType Directory -Force -Path $EvidenceDir | Out-Null + $safeRunId = Get-SafeId -Value $RunId + $path = Join-Path $EvidenceDir ( + "windows99-host112-kernel-memory-{0}-{1}.json" -f ( + $Phase.ToLowerInvariant() + ), $safeRunId + ) + $Receipt["evidencePath"] = $path + $Receipt | ConvertTo-Json -Depth 12 | Set-Content ` + -LiteralPath $path ` + -Encoding UTF8 + return $path +} + +function Get-VmxRuntime { + $matches = @(Get-CimInstance Win32_Process -ErrorAction Stop | + Where-Object { + $_.Name -eq "vmware-vmx.exe" -and + $_.CommandLine -and + $_.CommandLine.IndexOf( + $VmxPath, + [StringComparison]::OrdinalIgnoreCase + ) -ge 0 + }) + $rows = @($matches | ForEach-Object { + $owner = Invoke-CimMethod -InputObject $_ -MethodName GetOwner + [pscustomobject]@{ + processId = [int]$_.ProcessId + sessionId = [int]$_.SessionId + owner = [string]$owner.User + } + }) + return [pscustomobject]@{ + count = $rows.Count + rows = $rows + exactSystemOwner = [bool]( + $rows.Count -eq 1 -and + $rows[0].sessionId -eq 0 -and + $rows[0].owner -eq "SYSTEM" + ) + } +} + +function Convert-KeyValueOutput { + param([string[]]$Lines) + + $values = @{} + foreach ($line in $Lines) { + if ($line -match "^([A-Za-z0-9_]+)=(.*)$") { + $values[$Matches[1]] = $Matches[2].Trim() + } + } + return $values +} + +function Convert-ToInt64 { + param( + [object]$Value, + [long]$Default = -1 + ) + + $parsed = 0L + if ([long]::TryParse([string]$Value, [ref]$parsed)) { + return $parsed + } + return $Default +} + +function Invoke-Host112Readback { + $remoteScript = @' +set -uo pipefail +mem_value() { + awk -v key="$1" '$1 == key ":" {print $2; exit}' /proc/meminfo +} +unit_state() { + systemctl is-active "$1" 2>/dev/null || true +} +source_121_count="$( + timeout 15 journalctl -b -u ssh --since '15 minutes ago' \ + --grep='Accepted publickey' --no-pager -o cat 2>/dev/null | + awk '{for (i=1; i<=NF; i++) if ($i == "from" && $(i+1) == "192.168.0.121") count++} END {print count+0}' +)" +printf 'boot_id=%s\n' "$(cat /proc/sys/kernel/random/boot_id)" +printf 'uptime_seconds=%s\n' "$(awk '{print int($1)}' /proc/uptime)" +printf 'systemd_state=%s\n' "$(systemctl is-system-running 2>/dev/null || true)" +printf 'mem_total_kb=%s\n' "$(mem_value MemTotal)" +printf 'mem_available_kb=%s\n' "$(mem_value MemAvailable)" +printf 'percpu_kb=%s\n' "$(mem_value Percpu)" +printf 'sunreclaim_kb=%s\n' "$(mem_value SUnreclaim)" +printf 'source_121_sessions_15m=%s\n' "$source_121_count" +printf 'ssh=%s\n' "$(unit_state ssh)" +printf 'lightdm=%s\n' "$(unit_state lightdm)" +printf 'open_vm_tools=%s\n' "$(unit_state open-vm-tools)" +printf 'wazuh_indexer=%s\n' "$(unit_state wazuh-indexer)" +printf 'wazuh_manager=%s\n' "$(unit_state wazuh-manager)" +printf 'wazuh_dashboard=%s\n' "$(unit_state wazuh-dashboard)" +printf 'filebeat=%s\n' "$(unit_state filebeat)" +printf 'recovery_timer=%s\n' "$(unit_state awoooi-host112-guest-recovery.timer)" +'@ + $encoded = [Convert]::ToBase64String( + [Text.Encoding]::UTF8.GetBytes($remoteScript) + ) + $remoteCommand = ( + "printf '%s' '{0}' | base64 -d | timeout 25 bash" -f $encoded + ) + if (-not (Test-Path -LiteralPath $SshIdentityFile)) { + throw "Agent99 SSH identity path is unavailable" + } + $arguments = @( + "-i", $SshIdentityFile, + "-o", "IdentitiesOnly=yes", + "-o", "BatchMode=yes", + "-o", "StrictHostKeyChecking=accept-new", + "-o", "NumberOfPasswordPrompts=0", + "-o", "ConnectTimeout=8", + "-o", "ConnectionAttempts=1", + "-o", "ServerAliveInterval=5", + "-o", "ServerAliveCountMax=1", + $SshTarget, + $remoteCommand + ) + $id = [guid]::NewGuid().ToString("N") + $stdoutPath = Join-Path $env:TEMP "host112-kmem-$id.out" + $stderrPath = Join-Path $env:TEMP "host112-kmem-$id.err" + $process = Start-Process ` + -FilePath "ssh.exe" ` + -ArgumentList $arguments ` + -NoNewWindow ` + -RedirectStandardOutput $stdoutPath ` + -RedirectStandardError $stderrPath ` + -PassThru + $null = $process.Handle + if (-not $process.WaitForExit(35000)) { + try { + & taskkill.exe /PID $process.Id /T /F 2>&1 | Out-Null + } catch {} + $process.WaitForExit(2000) | Out-Null + Remove-Item $stdoutPath, $stderrPath ` + -Force ` + -ErrorAction SilentlyContinue + throw "Host112 SSH readback timed out after 35 seconds" + } + $process.WaitForExit() | Out-Null + $process.Refresh() + $stdout = if (Test-Path -LiteralPath $stdoutPath) { + Get-Content -LiteralPath $stdoutPath -Raw + } else { "" } + $stderr = if (Test-Path -LiteralPath $stderrPath) { + Get-Content -LiteralPath $stderrPath -Raw + } else { "" } + Remove-Item $stdoutPath, $stderrPath ` + -Force ` + -ErrorAction SilentlyContinue + $output = @($stdout -split "`r?`n" | Where-Object { $_ }) + $exitCode = [int]$process.ExitCode + if ($stderr) { + $output += @($stderr -split "`r?`n" | Where-Object { $_ }) + } + $values = Convert-KeyValueOutput -Lines $output + $memTotalKb = Convert-ToInt64 $values["mem_total_kb"] + $memAvailableKb = Convert-ToInt64 $values["mem_available_kb"] + $percpuKb = Convert-ToInt64 $values["percpu_kb"] + $sunreclaimKb = Convert-ToInt64 $values["sunreclaim_kb"] + $memAvailablePercent = if ($memTotalKb -gt 0 -and $memAvailableKb -ge 0) { + [math]::Round(($memAvailableKb / $memTotalKb) * 100, 2) + } else { $null } + $percpuPercent = if ($memTotalKb -gt 0 -and $percpuKb -ge 0) { + [math]::Round(($percpuKb / $memTotalKb) * 100, 2) + } else { $null } + $sunreclaimPercent = if ($memTotalKb -gt 0 -and $sunreclaimKb -ge 0) { + [math]::Round(($sunreclaimKb / $memTotalKb) * 100, 2) + } else { $null } + $requiredUnits = @( + "ssh", + "lightdm", + "open_vm_tools", + "wazuh_indexer", + "wazuh_manager", + "wazuh_dashboard", + "filebeat", + "recovery_timer" + ) + $failedUnits = @($requiredUnits | Where-Object { + [string]$values[$_] -ne "active" + }) + return [pscustomobject]@{ + ok = [bool]( + $exitCode -eq 0 -and + $values["boot_id"] -match "^[0-9a-f-]{36}$" -and + $values["systemd_state"] -eq "running" + ) + exitCode = $exitCode + bootId = [string]$values["boot_id"] + uptimeSeconds = Convert-ToInt64 $values["uptime_seconds"] + systemdState = [string]$values["systemd_state"] + memTotalKb = $memTotalKb + memAvailableKb = $memAvailableKb + memAvailablePercent = $memAvailablePercent + percpuKb = $percpuKb + percpuPercent = $percpuPercent + sunreclaimKb = $sunreclaimKb + sunreclaimPercent = $sunreclaimPercent + source121Sessions15m = Convert-ToInt64 ` + $values["source_121_sessions_15m"] + failedUnits = $failedUnits + servicesReady = [bool]($failedUnits.Count -eq 0) + rawOutputStored = $false + } +} + +function New-BaseReceipt { + param([string]$Phase) + + return [ordered]@{ + schemaVersion = "windows99_host112_kernel_memory_soft_reboot_v1" + phase = $Phase + traceId = $TraceId + runId = $RunId + workItemId = $WorkItemId + generatedAt = Get-Date -Format o + targetHost = "192.168.0.112" + targetVmx = $ExpectedVmxPath + runtimeWritePerformed = $false + hardResetPerformed = $false + vmStopPerformed = $false + secretReadPerformed = $false + terminal = "started" + } +} + +Assert-ControlIdentity +if ($VmxPath -ne $ExpectedVmxPath -or $SshTarget -ne $ExpectedSshTarget) { + throw "target contract mismatch" +} + +if ($Mode -eq "Check") { + $receipt = New-BaseReceipt -Phase "check" + $vmxRuntime = Get-VmxRuntime + $guest = Invoke-Host112Readback + $kernelPressureCritical = [bool]( + ($null -ne $guest.percpuPercent -and $guest.percpuPercent -ge 20) -or + ($null -ne $guest.sunreclaimPercent -and $guest.sunreclaimPercent -ge 25) + ) + $gatePassed = [bool]( + (Test-Path -LiteralPath $VmxPath) -and + (Test-Path -LiteralPath $VmrunPath) -and + $vmxRuntime.exactSystemOwner -and + $guest.ok -and + $guest.servicesReady -and + $guest.memAvailablePercent -le 5 -and + $kernelPressureCritical -and + $guest.source121Sessions15m -eq 0 + ) + $receipt["vmxRuntime"] = $vmxRuntime + $receipt["guest"] = $guest + $receipt["kernelPressureCritical"] = $kernelPressureCritical + $receipt["gatePassed"] = $gatePassed + $receipt["terminal"] = if ($gatePassed) { + "check_passed_apply_allowed" + } else { + "check_failed_no_write" + } + $evidencePath = Write-ControlReceipt -Phase "check" -Receipt $receipt + $receipt | ConvertTo-Json -Depth 12 + if (-not $gatePassed) { exit 2 } + exit 0 +} + +if ($Mode -eq "Apply") { + $receipt = New-BaseReceipt -Phase "apply" + if (-not $GateReceiptPath -or -not (Test-Path -LiteralPath $GateReceiptPath)) { + throw "fresh check receipt is required" + } + $gate = Get-Content -LiteralPath $GateReceiptPath -Raw | + ConvertFrom-Json + $gateAgeMinutes = ((Get-Date) - [datetime]$gate.generatedAt).TotalMinutes + $identityMatches = [bool]( + $gate.traceId -eq $TraceId -and + $gate.runId -eq $RunId -and + $gate.workItemId -eq $WorkItemId + ) + if ( + -not $identityMatches -or + -not $gate.gatePassed -or + $gate.runtimeWritePerformed -or + $gateAgeMinutes -lt 0 -or + $gateAgeMinutes -gt $GateMaxAgeMinutes + ) { + throw "check receipt is stale or invalid" + } + $vmxRuntime = Get-VmxRuntime + if (-not $vmxRuntime.exactSystemOwner) { + throw "host112 VM is not held by the expected SYSTEM runtime" + } + $vmrunListOutput = @(& $VmrunPath -T ws list 2>&1 | + ForEach-Object { "$_" }) + $vmrunListExitCode = $LASTEXITCODE + $vmrunSeesTarget = [bool](@($vmrunListOutput | Where-Object { + $_ -and $_.IndexOf( + $VmxPath, + [StringComparison]::OrdinalIgnoreCase + ) -ge 0 + }).Count -eq 1) + if ($vmrunListExitCode -ne 0 -or -not $vmrunSeesTarget) { + throw "SYSTEM vmrun does not own the exact Host112 VMX" + } + $receipt["gateReceiptPath"] = $GateReceiptPath + $receipt["gateAgeMinutes"] = [math]::Round($gateAgeMinutes, 2) + $receipt["preApplyVmrun"] = [ordered]@{ + exitCode = $vmrunListExitCode + exactTargetVisible = $vmrunSeesTarget + rawOutputStored = $false + } + $receipt["beforeBootId"] = [string]$gate.guest.bootId + $receipt["beforeMemory"] = [ordered]@{ + memAvailablePercent = $gate.guest.memAvailablePercent + percpuPercent = $gate.guest.percpuPercent + sunreclaimPercent = $gate.guest.sunreclaimPercent + } + $output = @(& $VmrunPath -T ws reset $VmxPath soft 2>&1 | + ForEach-Object { "$_" }) + $exitCode = $LASTEXITCODE + $receipt["vmrunExitCode"] = $exitCode + $receipt["vmrunOutputStored"] = $false + $receipt["runtimeWritePerformed"] = [bool]($exitCode -eq 0) + $receipt["resetMode"] = "soft" + $receipt["postVerifierPending"] = [bool]($exitCode -eq 0) + $receipt["terminal"] = if ($exitCode -eq 0) { + "soft_reset_requested_verifier_pending" + } else { + "soft_reset_failed_no_hard_fallback" + } + $evidencePath = Write-ControlReceipt -Phase "apply" -Receipt $receipt + $receipt | ConvertTo-Json -Depth 12 + if ($exitCode -ne 0) { exit 2 } + exit 0 +} + +$receipt = New-BaseReceipt -Phase "verify" +if (-not $ApplyReceiptPath -or -not (Test-Path -LiteralPath $ApplyReceiptPath)) { + throw "apply receipt is required" +} +$apply = Get-Content -LiteralPath $ApplyReceiptPath -Raw | ConvertFrom-Json +$identityMatches = [bool]( + $apply.traceId -eq $TraceId -and + $apply.runId -eq $RunId -and + $apply.workItemId -eq $WorkItemId +) +if ( + -not $identityMatches -or + -not $apply.runtimeWritePerformed -or + $apply.resetMode -ne "soft" -or + $apply.vmrunExitCode -ne 0 +) { + throw "apply receipt is invalid" +} + +$deadline = (Get-Date).AddSeconds($VerifyTimeoutSeconds) +$attempts = 0 +$guest = $null +do { + $attempts += 1 + try { + $guest = Invoke-Host112Readback + } catch { + $guest = $null + } + $bootChanged = [bool]( + $guest -and + $guest.bootId -and + $guest.bootId -ne [string]$apply.beforeBootId + ) + $memoryRecovered = [bool]( + $guest -and + $guest.memAvailablePercent -ge 25 -and + $guest.percpuPercent -lt 5 -and + $guest.sunreclaimPercent -lt 10 + ) + if ( + $guest -and + $guest.ok -and + $guest.servicesReady -and + $bootChanged -and + $memoryRecovered + ) { + break + } + if ((Get-Date) -lt $deadline) { + Start-Sleep -Seconds $VerifyIntervalSeconds + } +} while ((Get-Date) -lt $deadline) + +$vmxRuntime = Get-VmxRuntime +$verified = [bool]( + $guest -and + $guest.ok -and + $guest.servicesReady -and + $guest.bootId -ne [string]$apply.beforeBootId -and + $guest.memAvailablePercent -ge 25 -and + $guest.percpuPercent -lt 5 -and + $guest.sunreclaimPercent -lt 10 -and + $guest.source121Sessions15m -eq 0 -and + $vmxRuntime.exactSystemOwner +) +$receipt["applyReceiptPath"] = $ApplyReceiptPath +$receipt["beforeBootId"] = [string]$apply.beforeBootId +$receipt["after"] = $guest +$receipt["vmxRuntime"] = $vmxRuntime +$receipt["attempts"] = $attempts +$receipt["verified"] = $verified +$receipt["terminal"] = if ($verified) { + "verified_recovered" +} else { + "post_verifier_failed_no_hard_fallback" +} +$evidencePath = Write-ControlReceipt -Phase "verify" -Receipt $receipt +$receipt | ConvertTo-Json -Depth 12 +if (-not $verified) { exit 2 } +exit 0