From 4ea132d992a545389cd0d8565f3509d3692b5883 Mon Sep 17 00:00:00 2001 From: ogt Date: Tue, 14 Jul 2026 18:29:48 +0800 Subject: [PATCH] fix(agent99): close canonical host readbacks --- agent99-bootstrap.ps1 | 13 ++-- agent99-contract-check.ps1 | 9 ++- agent99-control-plane.ps1 | 61 +++++++++++-------- agent99-deploy.ps1 | 25 ++++---- agent99.config.99.example.json | 1 + agent99.config.example.json | 1 + ...nt99_transport_recovery_deploy_contract.py | 26 +++++++- docs/runbooks/FULL-STACK-COLD-START-SOP.md | 12 +++- docs/runbooks/REBOOT-RECOVERY-SOP.md | 12 +++- .../deploy-agent99-via-windows99-ssh.sh | 53 +++++++++++++++- .../host112-guest-readiness.sh | 30 ++++++++- .../install-macos111-host-boot-readback.sh | 21 ++++++- .../macos-host-boot-readback.sh | 30 +++++++++ ...t_agent99_recovery_coordinator_contract.py | 6 ++ ...remote_atomic_deploy_transport_contract.py | 11 +++- .../test_host112_manager_recovery_contract.py | 8 +-- .../test_reboot_p0_operational_contract.py | 8 +++ 17 files changed, 259 insertions(+), 68 deletions(-) diff --git a/agent99-bootstrap.ps1 b/agent99-bootstrap.ps1 index 0b9d205ab..2f3674a3c 100644 --- a/agent99-bootstrap.ps1 +++ b/agent99-bootstrap.ps1 @@ -95,6 +95,7 @@ function Ensure-AgentHost112CanonicalSshConfig { if (-not $config.PSObject.Properties["sshUsers"] -or $null -eq $config.sshUsers) { Set-AgentBootstrapProperty $config "sshUsers" ([pscustomobject]@{}) } + Set-AgentBootstrapProperty $config.sshUsers "192.168.0.111" "ooo" Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali" if (-not $config.PSObject.Properties["k3s"] -or $null -eq $config.k3s) { Set-AgentBootstrapProperty $config "k3s" ([pscustomobject]@{}) @@ -117,14 +118,16 @@ function Ensure-AgentHost112CanonicalSshConfig { $config | ConvertTo-Json -Depth 20 | Set-Content -Encoding UTF8 $Path $persisted = Get-Content $Path -Raw | ConvertFrom-Json - $persistedUser = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } + $persistedHost111User = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.111"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.111"].Value } else { "" } + $persistedHost112User = if ($persisted.sshUsers -and $persisted.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persisted.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } $persistedDirectHosts = if ($persisted.k3s -and $persisted.k3s.PSObject.Properties["directHosts"]) { @($persisted.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } $persistedHost112Vm = @($persisted.vms | Where-Object { [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" }) $persistedHost112Vmx = if ($persistedHost112Vm.Count -eq 1) { [string]$persistedHost112Vm[0].vmx } else { "" } if ( - $persistedUser -ne "kali" -or + $persistedHost111User -ne "ooo" -or + $persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts -or $persistedHost112Vm.Count -ne 1 -or [string]$persistedHost112Vm[0].name -ne [string]$canonicalHost112Vm.name -or @@ -132,7 +135,7 @@ function Ensure-AgentHost112CanonicalSshConfig { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Host112 canonical SSH and VMX config bootstrap verifier failed." + throw "Host111/Host112 canonical SSH and Host112 VMX config bootstrap verifier failed." } } @@ -218,10 +221,10 @@ if (-not (Test-Path $configPath)) { "agentRoot": "$($AgentRoot.Replace('\', '\\'))", "sshUser": "wooo", "sshIdentityFile": "$($AgentRoot.Replace('\', '\\'))\\keys\\agent99_ed25519", - "sshUsers": { "192.168.0.112": "kali" }, + "sshUsers": { "192.168.0.111": "ooo", "192.168.0.112": "kali" }, "sshTransport": { "serialized": true, "lockTimeoutSeconds": 90, "lockPollMilliseconds": 200 }, "vmrunPath": "", - "hosts": ["192.168.0.110", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"], + "hosts": ["192.168.0.110", "192.168.0.111", "192.168.0.112", "192.168.0.120", "192.168.0.121", "192.168.0.188"], "vms": [], "k3s": { "jumpHost": "192.168.0.110", diff --git a/agent99-contract-check.ps1 b/agent99-contract-check.ps1 index 5586e2f74..79c481480 100644 --- a/agent99-contract-check.ps1 +++ b/agent99-contract-check.ps1 @@ -59,6 +59,11 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js $hasRoutes = @($config.publicUrls).Count -ge 5 $hasSlo = [bool]($config.recoverySlo -and [int]$config.recoverySlo.targetMinutes -eq 10) $hasIdentity = [bool]($config.sshIdentityFile -and $config.sshIdentityFile -match "agent99_ed25519$") + $hasCanonicalHost111User = [bool]( + $config.sshUsers -and + $config.sshUsers.PSObject.Properties["192.168.0.111"] -and + [string]$config.sshUsers.PSObject.Properties["192.168.0.111"].Value -eq "ooo" + ) $hasJump = [bool]($config.k3s -and $config.k3s.jumpHost -and $config.k3s.preferJumpHost -eq $true) $hasAutoRecovery = [bool]($config.autoRecovery -and $config.autoRecovery.enabled -eq $true) $hasRecoveryCoordinator = [bool]( @@ -80,8 +85,8 @@ foreach ($name in @("agent99.config.example.json", "agent99.config.99.example.js [string]$_.vmx -eq "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" }).Count -eq 1) } else { $true } - $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx - Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" + $ok = $hasHosts -and $hasExternalHostRecovery -and $hasRoutes -and $hasSlo -and $hasIdentity -and $hasCanonicalHost111User -and $hasJump -and $hasAutoRecovery -and $hasRecoveryCoordinator -and $hasCompletionCallback -and $hasCanonicalHost112Vmx + Add-Check "config:$name" $ok "hosts=$(@($config.hosts).Count) externalHostRecovery=$hasExternalHostRecovery routes=$(@($config.publicUrls).Count) slo10m=$hasSlo identity=$hasIdentity canonicalHost111User=$hasCanonicalHost111User jump=$hasJump autoRecovery=$hasAutoRecovery recoveryCoordinator=$hasRecoveryCoordinator completionCallback=$hasCompletionCallback canonicalHost112Vmx=$hasCanonicalHost112Vmx" } catch { Add-Check "config:$name" $false "json_parse_failed: $($_.Exception.Message)" } diff --git a/agent99-control-plane.ps1 b/agent99-control-plane.ps1 index b7176a386..7fc0ab43f 100644 --- a/agent99-control-plane.ps1 +++ b/agent99-control-plane.ps1 @@ -2961,36 +2961,47 @@ df -P / 2>/dev/null } $text = $readback.output $cores = 1 - $coresMatch = [regex]::Match($text, "CORES=(\d+)") - if ($coresMatch.Success) { $cores = [int]$coresMatch.Groups[1].Value } - $load1 = $null - $loadMatch = [regex]::Match($text, "load averages?:\s*([0-9]+(?:[\.,][0-9]+)?)") - if (-not $loadMatch.Success) { - $loadMatch = [regex]::Match($text, "load average:\s*([0-9]+(?:[\.,][0-9]+)?)") - } - if ($loadMatch.Success) { - $load1 = Convert-AgentDouble $loadMatch.Groups[1].Value - } - $memAvailablePercent = $null - $memLine = (($text -split "`n") | Where-Object { $_ -match "^\s*Mem:" } | Select-Object -First 1) - if ($memLine) { - $parts = @($memLine -split "\s+" | Where-Object { $_ }) - if ($parts.Count -ge 7) { - $memTotal = Convert-AgentDouble $parts[1] - $memAvailable = Convert-AgentDouble $parts[6] - if ($memTotal -and $memTotal -gt 0 -and $null -ne $memAvailable) { - $memAvailablePercent = [math]::Round(($memAvailable / $memTotal) * 100, 2) + $diskUsedPercent = $null + $canonicalPerfMatch = [regex]::Match( + $text, + "(?m)^perf_schema=agent99_perf_readback_v1\s+os=(?\S+)\s+cores=(?\d+)\s+load1=(?[0-9]+(?:[\.,][0-9]+)?)\s+mem_available_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s+disk_used_percent=(?[0-9]+(?:[\.,][0-9]+)?)\s*$" + ) + if ($canonicalPerfMatch.Success) { + $cores = [int]$canonicalPerfMatch.Groups["cores"].Value + $load1 = Convert-AgentDouble $canonicalPerfMatch.Groups["load"].Value + $memAvailablePercent = Convert-AgentDouble $canonicalPerfMatch.Groups["memory"].Value + $diskUsedPercent = Convert-AgentDouble $canonicalPerfMatch.Groups["disk"].Value + } else { + $coresMatch = [regex]::Match($text, "CORES=(\d+)") + if ($coresMatch.Success) { $cores = [int]$coresMatch.Groups[1].Value } + + $loadMatch = [regex]::Match($text, "load averages?:\s*([0-9]+(?:[\.,][0-9]+)?)") + if (-not $loadMatch.Success) { + $loadMatch = [regex]::Match($text, "load average:\s*([0-9]+(?:[\.,][0-9]+)?)") + } + if ($loadMatch.Success) { + $load1 = Convert-AgentDouble $loadMatch.Groups[1].Value + } + + $memLine = (($text -split "`n") | Where-Object { $_ -match "^\s*Mem:" } | Select-Object -First 1) + if ($memLine) { + $parts = @($memLine -split "\s+" | Where-Object { $_ }) + if ($parts.Count -ge 7) { + $memTotal = Convert-AgentDouble $parts[1] + $memAvailable = Convert-AgentDouble $parts[6] + if ($memTotal -and $memTotal -gt 0 -and $null -ne $memAvailable) { + $memAvailablePercent = [math]::Round(($memAvailable / $memTotal) * 100, 2) + } } } - } - $diskUsedPercent = $null - $diskLine = (($text -split "`n") | Where-Object { $_ -match "\s+\d+%\s+/$" } | Select-Object -First 1) - if ($diskLine) { - $diskMatch = [regex]::Match($diskLine, "\s(\d+)%\s+/$") - if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } + $diskLine = (($text -split "`n") | Where-Object { $_ -match "\s+\d+%\s+/$" } | Select-Object -First 1) + if ($diskLine) { + $diskMatch = [regex]::Match($diskLine, "\s(\d+)%\s+/$") + if ($diskMatch.Success) { $diskUsedPercent = [double]$diskMatch.Groups[1].Value } + } } $topCpu = @() diff --git a/agent99-deploy.ps1 b/agent99-deploy.ps1 index a25ab60f9..99a5a0d7b 100644 --- a/agent99-deploy.ps1 +++ b/agent99-deploy.ps1 @@ -228,6 +228,15 @@ try { $config = Get-Content $ConfigPath -Raw | ConvertFrom-Json Add-DefaultProperty $config "sshIdentityFile" (Join-Path $AgentRoot "keys\agent99_ed25519") Add-DefaultProperty $config "sshUsers" ([pscustomobject]@{}) + $previousHost111User = if ($config.sshUsers.PSObject.Properties["192.168.0.111"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.111"].Value } else { "" } + if ($previousHost111User -ne "ooo") { + Set-AgentProperty $config.sshUsers "192.168.0.111" "ooo" + $script:ConfigMigrations += [pscustomobject]@{ + name = "host111_canonical_ssh_user" + from = if ($previousHost111User) { $previousHost111User } else { "missing" } + to = "ooo" + } + } $previousHost112User = if ($config.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$config.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } if ($previousHost112User -ne "kali") { Set-AgentProperty $config.sshUsers "192.168.0.112" "kali" @@ -292,18 +301,6 @@ try { Add-DefaultProperty $host111External "enabled" $true Add-DefaultProperty $host111External "required" $true } - Add-DefaultProperty $config "vms" @() - $host112Vm = @($config.vms | Where-Object { [string]$_.host -eq "192.168.0.112" -or [string]$_.name -eq "host112-kali" } | Select-Object -First 1) - $host112CanonicalVmx = "S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" - if ($host112Vm.Count -gt 0 -and [string]$host112Vm[0].vmx -ne $host112CanonicalVmx) { - $previousHost112Vmx = [string]$host112Vm[0].vmx - Set-AgentProperty $host112Vm[0] "vmx" $host112CanonicalVmx - $script:ConfigMigrations += [pscustomobject]@{ - name = "host112_ssd_vmx_source" - from = if ($previousHost112Vmx) { $previousHost112Vmx } else { "missing" } - to = $host112CanonicalVmx - } - } Add-DefaultProperty $config "k3s" ([pscustomobject]@{}) Add-DefaultProperty $config.k3s "jumpHost" "192.168.0.110" Add-DefaultProperty $config.k3s "preferJumpHost" $true @@ -410,6 +407,7 @@ try { } $config | ConvertTo-Json -Depth 20 | Set-Content -Path $ConfigPath -Encoding UTF8 $persistedConfig = Get-Content $ConfigPath -Raw | ConvertFrom-Json + $persistedHost111User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.111"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.111"].Value } else { "" } $persistedHost112User = if ($persistedConfig.sshUsers -and $persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"]) { [string]$persistedConfig.sshUsers.PSObject.Properties["192.168.0.112"].Value } else { "" } $persistedDirectHosts = if ($persistedConfig.k3s -and $persistedConfig.k3s.PSObject.Properties["directHosts"]) { @($persistedConfig.k3s.directHosts | ForEach-Object { [string]$_ }) } else { @() } $persistedHost112Vm = @($persistedConfig.vms | Where-Object { @@ -428,6 +426,7 @@ try { 22 -in @($persistedHost111External[0].verifyTcpPorts | ForEach-Object { [int]$_ }) ) if ( + $persistedHost111User -ne "ooo" -or $persistedHost112User -ne "kali" -or "192.168.0.112" -notin $persistedDirectHosts -or "192.168.0.111" -notin $persistedHosts -or @@ -439,7 +438,7 @@ try { $persistedHost112Vmx -ne $canonicalHost112Vmx -or -not (Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf) ) { - throw "Canonical Host111 recovery and Host112 direct/SSD config post-verifier failed." + throw "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed." } $manifestRows = @() diff --git a/agent99.config.99.example.json b/agent99.config.99.example.json index 77e567ba3..48b481e49 100644 --- a/agent99.config.99.example.json +++ b/agent99.config.99.example.json @@ -14,6 +14,7 @@ "hardStaleMinutes": 60 }, "sshUsers": { + "192.168.0.111": "ooo", "192.168.0.112": "kali", "192.168.0.188": "ollama" }, diff --git a/agent99.config.example.json b/agent99.config.example.json index 617f4f31b..d970d69fe 100644 --- a/agent99.config.example.json +++ b/agent99.config.example.json @@ -14,6 +14,7 @@ "hardStaleMinutes": 60 }, "sshUsers": { + "192.168.0.111": "ooo", "192.168.0.112": "kali" }, "vmrunPath": "", diff --git a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py index 9ec00e760..edea82da1 100644 --- a/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py +++ b/apps/api/tests/test_agent99_transport_recovery_deploy_contract.py @@ -189,6 +189,7 @@ def test_agent99_99_config_has_bounded_transport_and_recovery_defaults() -> None assert config["recoverySlo"]["targetMinutes"] == 10 assert "192.168.0.111" in config["hosts"] assert "192.168.0.111" in config["selfHealth"]["requiredHosts"] + assert config["sshUsers"]["192.168.0.111"] == "ooo" host111 = next(item for item in config["externalHosts"] if item["host"] == "192.168.0.111") assert host111["recoveryAction"] == "wake_on_lan" assert host111["required"] is True @@ -227,6 +228,21 @@ def test_agent99_host111_recovery_is_allowlisted_and_independently_verified() -> assert 'New-AgentOutcomeCheck "external_hosts_recovered"' in source +def test_agent99_host111_restricted_perf_readback_has_a_canonical_parser() -> None: + source = CONTROL.read_text(encoding="utf-8") + readback = (ROOT / "scripts/reboot-recovery/macos-host-boot-readback.sh").read_text( + encoding="utf-8" + ) + + assert "perf_schema=agent99_perf_readback_v1 os=darwin" in readback + assert "memory_pressure -Q" in readback + assert "sysctl -n hw.logicalcpu" in readback + assert "df -Pk /" in readback + assert "perf_schema=agent99_perf_readback_v1" in source + for group in ('Groups["cores"]', 'Groups["load"]', 'Groups["memory"]', 'Groups["disk"]'): + assert group in source + + def test_agent99_startup_recovery_uses_full_outcome_callback_and_learning_chain() -> None: tasks = (ROOT / "agent99-register-tasks.ps1").read_text(encoding="utf-8") @@ -243,15 +259,19 @@ def test_agent99_deployer_migrates_and_verifies_host111_and_host112_sources() -> deploy = DEPLOY.read_text(encoding="utf-8") for migration in ( + "host111_canonical_ssh_user", "host111_canonical_recovery_target", "host111_wol_executor", "host111_self_health_coverage", - "host112_ssd_vmx_source", + "host112_canonical_vmx_path", ): assert migration in deploy - assert '"S:\\VMs\\host112\\kali-linux-2025.4-vmware-amd64.vmx"' in deploy + assert "Get-AgentHost112CanonicalVm $policyConfigPath" in deploy + assert r"S:\VMs\host112\kali-linux-2025.4-vmware-amd64.vmx" not in deploy + assert "$persistedHost111User" in deploy assert "$host111ExternalReady" in deploy - assert "$host112VmxReady" in deploy + assert "$persistedHost112Vmx -ne $canonicalHost112Vmx" in deploy + assert "Test-Path -LiteralPath $persistedHost112Vmx -PathType Leaf" in deploy def test_agent99_deployer_verifies_promoted_runtime_and_writes_revision() -> None: diff --git a/docs/runbooks/FULL-STACK-COLD-START-SOP.md b/docs/runbooks/FULL-STACK-COLD-START-SOP.md index 496573d83..4ac798562 100644 --- a/docs/runbooks/FULL-STACK-COLD-START-SOP.md +++ b/docs/runbooks/FULL-STACK-COLD-START-SOP.md @@ -1,7 +1,7 @@ # AWOOOI 全棧冷啟動與主機重啟 SOP -> Version: v1.133 -> Last updated: 2026-07-11 Asia/Taipei +> Version: v1.134 +> Last updated: 2026-07-14 Asia/Taipei > Scope: 99 / 110 / 111 / 112 / 120 / 121 / 188 全棧重啟恢復。112 仍是 Kali / VM guest 訊號，但 2026-06-30 全主機重啟後已納入 10 分鐘 SLO 的必要 boot / power signal；此納入不代表授權任何破壞性 runtime apply。 --- @@ -10,6 +10,14 @@ 本節是每次接手、開機、關機、重啟後的第一個判定錨點。若日期不是今天，必須先重跑 live check，再更新本節與 `docs/workplans/2026-06-04-reboot-cold-start-backup-recovery-workplan.md`。 +### v1.134 2026-07-14 Agent99 canonical host readback closure + +本輪釐清 111 的 `perf_readback_failed` 不是 CPU 壓力，而是 Agent99 沿用預設 `wooo`，但 110 到 111 的 canonical restricted identity 是 `ooo`。部署合約固定 `sshUsers["192.168.0.111"]="ooo"`，舊 live config 必須由 `host111_canonical_ssh_user` migration 寫入後重新讀回；帳號、transport 或四項指標任一缺失時，只能標示 readback degraded，不得產生 CPU/記憶體/磁碟故障結論或執行降載。 + +- 111 仍使用 `restrict,command=` public key，禁止 shell、PTY 與 forwarding。`/Users/ooo/.local/bin/awoooi-host-boot-readback` 固定額外輸出 `perf_schema=agent99_perf_readback_v1 os=darwin`、logical cores、load1、memory available percent 與 root disk used percent；Agent99 解析此 schema，不能用 Linux `free` 輸出假設 Mac。 +- 112 的 `systemctl is-system-running=degraded` 只能在 `systemctl list-units --failed` 查詢成功、且排除 `awoooi-host112-guest-recovery.service` 自身後 `nonself_failed_count=0` 時正規化為 ready。readback 必須保留 `systemd_state_raw`、`failed_units_query_ok`、`nonself_failed_count` 與 units；查詢失敗或任何其他 failed unit 都 fail closed。 +- 固定驗收順序是 source/test -> Gitea CD/deploy marker -> exact Agent99 runtime manifest -> 111 restricted readback -> Agent99 Perf outcome/callback -> AWOOOI recurrence/work-item -> Telegram/KM。`runtimeMatched=true`、單次 route 200 或 Telegram 已送都不能單獨關閉事件。 + ### v1.133 2026-07-11 Cold-start runtime convergence 2026-07-11 14:58 最新完成 artifact 為 `/home/wooo/reboot-recovery/reboot-auto-recovery-slo-20260711-145616`。`OVERALL_DECLARATION=GREEN_WITH_EVIDENCE_WARNINGS`、`SERVICE_GREEN=1`、`PRODUCT_DATA_GREEN=1`、`BACKUP_CORE_GREEN=1`、`POST_START_BLOCKED=0`、`HOST_188_HYGIENE_BLOCKED=0`；scorecard readiness `87%`。active blockers 只剩 `all_required_hosts_not_in_10_minute_reboot_window`、`fresh_all_host_reboot_event_missing`、`host_boot_observation_older_than_target_window`，三者都只能由下一次真實全主機 reboot drill 在同一 10 分鐘 window 驗收；不得靠重跑 verifier 或為了清分數擅自 reboot。 diff --git a/docs/runbooks/REBOOT-RECOVERY-SOP.md b/docs/runbooks/REBOOT-RECOVERY-SOP.md index fb0b8f44e..6d6065ac6 100644 --- a/docs/runbooks/REBOOT-RECOVERY-SOP.md +++ b/docs/runbooks/REBOOT-RECOVERY-SOP.md @@ -1,9 +1,9 @@ # AWOOOI 重開機恢復 SOP -> **版本**: v5.31 -> **最後更新**: 2026-07-11 (台北時間) +> **版本**: v5.32 +> **最後更新**: 2026-07-14 (台北時間) > **更新者**: Codex -> **觸發事件**: 2026-07-11 真實全主機 reboot drill、late recovery 與 Agent99/VMware/runtime 閉環 +> **觸發事件**: 2026-07-14 Agent99 host111/host112 readback false-alert closure --- @@ -21,6 +21,12 @@ ## 架構概覽與依賴圖 +### 2026-07-14 Agent99 host readback false-alert closure + +111 是獨立 Darwin 主機，不是 Windows99 VM。Agent99 必須以 `ooo@192.168.0.111` 經 110 的 restricted forced-command route 讀取狀態；使用預設 `wooo` 會得到 SSH failure，不能包裝成 CPU 壓力。forced command 固定輸出 boot/Ollama 加 `agent99_perf_readback_v1` 四項唯讀指標，仍禁止 shell、PTY、forwarding、secret read 與任意命令。Windows99 deploy/bootstrap 必須自動遷移並 post-verify `sshUsers["192.168.0.111"]="ooo"`；Perf 只有在 cores/load/memory/disk 全部可解析時才可通過。 + +112 guest 的 systemd raw 狀態不得被直接洗綠。只有 failed-unit 查詢成功、唯一失敗是 recovery oneshot 自身、且 console/SSH/Wazuh/timer 各自 verifier 全綠時，effective systemd state 才可視為 running；其他 failed unit 或 query failure 都保持 degraded。這些欄位必須進同一 readback、Agent99 outcome、callback、AWOOOI recurrence 與 KM，避免每次重啟重新猜測原因。 + ### 2026-07-11 真實 reboot drill：SLA breach 與 late recovery 真相 本輪不是模擬。重啟前 Agent99 runtime `7cf83f7f59a5d72ce6c303f59c144bd5bf8566df` 已 `14/14` matched、核心 task `9/9`，production-principal `Recover` baseline 已通過 full-SOP coordinator，且正確標示 `scope=service_recovery`、`rebootSloClaimed=false`。七主機與六核心路由在 observer 開始時均可達。 diff --git a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh index d04145f74..a8255ba77 100755 --- a/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh +++ b/scripts/reboot-recovery/deploy-agent99-via-windows99-ssh.sh @@ -39,7 +39,7 @@ WORK_ITEM_ID="" IDENTITY_FILE="" KNOWN_HOSTS_FILE="${HOME:-}/.ssh/known_hosts" CONNECT_TIMEOUT_SECONDS="8" -REMOTE_EXECUTION_TIMEOUT_SECONDS="45" +REMOTE_EXECUTION_TIMEOUT_SECONDS="180" usage() { printf '%s\n' \ @@ -206,6 +206,7 @@ trap cleanup EXIT HUP INT TERM ENVELOPE_PATH="${WORK_DIR}/envelope.json" BOOTSTRAP_PATH="${WORK_DIR}/remote-bootstrap.ps1" +BOOTSTRAP_PAYLOAD_PATH="${WORK_DIR}/remote-bootstrap.ps1.gz.b64" python3 - \ "${SOURCE_ROOT}" \ @@ -300,6 +301,28 @@ bootstrap = ( output_path.write_text(bootstrap, encoding="utf-8") PY +python3 - \ + "${BOOTSTRAP_PATH}" \ + "${BOOTSTRAP_PAYLOAD_PATH}" <<'PY' +from __future__ import annotations + +import base64 +import gzip +import sys +from pathlib import Path + +bootstrap_path, output_path = map(Path, sys.argv[1:3]) +compressed = gzip.compress( + bootstrap_path.read_bytes(), + compresslevel=9, + mtime=0, +) +output_path.write_text( + base64.b64encode(compressed).decode("ascii"), + encoding="ascii", +) +PY + SSH_OPTIONS=( -T -o BatchMode=yes @@ -321,10 +344,34 @@ if [[ -n "${IDENTITY_FILE}" ]]; then SSH_OPTIONS+=( -i "${IDENTITY_FILE}" ) fi -readonly REMOTE_COMMAND='powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass -Command "$inputText=[Console]::In.ReadToEnd(); & ([ScriptBlock]::Create($inputText))"' +readonly REMOTE_COMMAND="$(python3 - <<'PY' +from __future__ import annotations + +import base64 + +decoder = r'''$payload = [Console]::In.ReadToEnd() +$compressed = [Convert]::FromBase64String(($payload -replace '\s', '')) +$inputStream = New-Object IO.MemoryStream(,$compressed) +$gzip = New-Object IO.Compression.GzipStream($inputStream, [IO.Compression.CompressionMode]::Decompress) +$reader = New-Object IO.StreamReader($gzip, [Text.Encoding]::UTF8) +try { + $inputText = $reader.ReadToEnd() +} finally { + $reader.Dispose() + $gzip.Dispose() + $inputStream.Dispose() +} +& ([ScriptBlock]::Create($inputText))''' +encoded = base64.b64encode(decoder.encode("utf-16le")).decode("ascii") +print( + "powershell.exe -NoProfile -NonInteractive -ExecutionPolicy Bypass " + f"-EncodedCommand {encoded}" +) +PY +)" set +e timeout --signal=TERM --kill-after=10 "${REMOTE_EXECUTION_TIMEOUT_SECONDS}s" \ - ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PATH}" + ssh "${SSH_OPTIONS[@]}" "${TARGET}" "${REMOTE_COMMAND}" <"${BOOTSTRAP_PAYLOAD_PATH}" status=$? set -e exit "${status}" diff --git a/scripts/reboot-recovery/host112-guest-readiness.sh b/scripts/reboot-recovery/host112-guest-readiness.sh index 435005664..de6a9648f 100755 --- a/scripts/reboot-recovery/host112-guest-readiness.sh +++ b/scripts/reboot-recovery/host112-guest-readiness.sh @@ -1388,8 +1388,32 @@ elif [ "$MODE" = "apply" ] || [ "$MODE" = "timer_apply" ]; then capture_manager_state fi -systemd_state="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)" -systemd_state="${systemd_state:-unknown}" +systemd_state_raw="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl is-system-running 2>/dev/null || true)" +systemd_state_raw="${systemd_state_raw:-unknown}" +failed_units_query_ok=1 +if ! failed_units_output="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl list-units --failed --no-legend --no-pager --plain 2>/dev/null)"; then + failed_units_query_ok=0 + failed_units_output="" +fi +nonself_failed_units=() +while IFS= read -r failed_unit; do + [ -n "$failed_unit" ] || continue + if [[ "$failed_unit" =~ ^awoooi-host112-guest-recovery\.service$ ]]; then + continue + fi + nonself_failed_units+=("$failed_unit") +done < <(printf '%s\n' "$failed_units_output" | awk '{ if ($1 == "●") print $2; else print $1 }') +nonself_failed_count="${#nonself_failed_units[@]}" +nonself_failed_text="none" +if [ "$nonself_failed_count" -gt 0 ]; then + nonself_failed_text="$(IFS=,; printf '%s' "${nonself_failed_units[*]}")" +fi +systemd_state="$systemd_state_raw" +if [ "$systemd_state_raw" = "degraded" ] \ + && [ "$failed_units_query_ok" -eq 1 ] \ + && [ "$nonself_failed_count" -eq 0 ]; then + systemd_state="running" +fi default_target="$(bounded_timeout "$READ_TIMEOUT_SECONDS" systemctl get-default 2>/dev/null || true)" default_target="${default_target:-unknown}" graphical_target="$(unit_active graphical.target)" @@ -1524,7 +1548,7 @@ run_output="${RUN_ID:-none}" work_item_output="${WORK_ITEM_ID:-none}" readback="schema_version=host112_guest_recovery_v2 mode=$MODE trace_id=$trace_output run_id=$run_output work_item_id=$work_item_output boot_id=$boot_id uptime_seconds=$uptime_seconds systemd_state=$systemd_state startup_enabled=$startup_enabled startup_active=$startup_active default_target=$default_target graphical_target=$graphical_target display_manager=$display_manager lightdm=$lightdm vmware_tools=$vmware_tools xorg=$xorg wazuh_indexer=$wazuh_indexer wazuh_manager=$wazuh_manager wazuh_manager_result=$manager_result manager_failed_state_observed=$manager_failed_state_observed wazuh_analysisd_process_count=$analysisd_count wazuh_dashboard=$wazuh_dashboard filebeat=$filebeat ssh_service=$ssh_service ssh_enabled=$ssh_enabled ssh_port_listening=$ssh_port_listening ssh_ready=$ssh_ready console_ready=$console_ready services_ready=$services_ready recovery_timer_ready=$recovery_timer_ready guest_ready=$guest_ready manager_preflight_status=$manager_preflight_status manager_terminal=$manager_terminal run_terminal=$run_terminal terminal=$run_terminal action_failure_observed=$action_failure_observed manager_orphan_analysisd=$manager_orphan_analysisd manager_memory_available_kb=$manager_memory_available_kb manager_disk_available_kb=$manager_disk_available_kb manager_disk_used_percent=$manager_disk_used_percent manager_resource_ready=$manager_resource_ready wazuh_agent_event_port_1514_tcp=$event_1514_tcp wazuh_agent_event_port_1514_udp=$event_1514_udp wazuh_agent_event_port_1514_ready=$manager_event_port_ready wazuh_agent_enrollment_port_1515_tcp=$enrollment_1515_tcp wazuh_manager_api_port_55000_tcp=$manager_api_55000_tcp manager_independent_verifier_ready=$manager_verifier_ready manager_cooldown_remaining_seconds=$manager_cooldown_remaining_seconds manager_attempt_count=$manager_attempt_count manager_start_exit=$manager_start_exit apply_phase_timeout_budget_seconds=$APPLY_PHASE_TIMEOUT_BUDGET_SECONDS executor_deadline_seconds=$EXECUTOR_DEADLINE_SECONDS runtime_write_performed=$runtime_write_performed state_write_performed=$state_write_performed manager_marker_write_performed=$manager_marker_write_performed manager_runtime_write_performed=$manager_runtime_write_performed artifact_transaction_status=$artifact_transaction_status artifact_transaction_phase=$artifact_transaction_phase artifact_transaction_path=$ARTIFACT_TRANSACTION_PATH artifact_transaction_started=$artifact_transaction_started artifact_transaction_blocked=$artifact_transaction_blocked artifact_transaction_write_performed=$artifact_transaction_write_performed artifact_transaction_pair_verified=$artifact_transaction_pair_verified artifact_transaction_prior_pair_verified=$artifact_transaction_prior_pair_verified receipt_write_performed=$receipt_write_performed receipt_status=$receipt_status receipt_ref=$receipt_ref receipt_path=$receipt_path receipt_sha256=$receipt_sha256 receipt_reused=$receipt_reused durable_receipt_present=$durable_receipt_present durable_receipt_identity_match=$durable_receipt_identity_match durable_receipt_boot_match=$durable_receipt_boot_match durable_receipt_terminal=$durable_receipt_terminal durable_readback_present=$durable_readback_present durable_readback_identity_match=$durable_readback_identity_match durable_readback_boot_match=$durable_readback_boot_match durable_readback_receipt_link_match=$durable_readback_receipt_link_match durable_readback_terminal_match=$durable_readback_terminal_match rollback_attempted=$rollback_attempted rollback_verified=$rollback_verified rollback_terminal=$rollback_terminal timer_artifact_policy=$timer_artifact_policy timer_ledger_capacity=$timer_ledger_capacity timer_ledger_slot=$timer_ledger_slot timer_observation_status=$timer_observation_status timer_observation_path=$timer_observation_path timer_observation_sha256=$timer_observation_sha256 timer_observation_repeat_count=$timer_observation_repeat_count timer_observation_write_performed=$timer_observation_write_performed signal_readback_status=$signal_readback_status signal_readback_path=$signal_readback_path signal_readback_sha256=$signal_readback_sha256 action_count=${#actions[@]} actions=$action_text prohibited_actions=host_reboot,vm_power_change,vm_reset,firewall_change,secret_read,database_write" -readback="$readback systemd_state_raw=$systemd_state_raw nonself_failed_count=$nonself_failed_count nonself_failed_units=$nonself_failed_text" +readback="$readback systemd_state_raw=$systemd_state_raw failed_units_query_ok=$failed_units_query_ok nonself_failed_count=$nonself_failed_count nonself_failed_units=$nonself_failed_text" canonical_readback_base() { printf '%s\n' "$readback" | awk ' diff --git a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh index f0b27a539..dd125e9c4 100755 --- a/scripts/reboot-recovery/install-macos111-host-boot-readback.sh +++ b/scripts/reboot-recovery/install-macos111-host-boot-readback.sh @@ -25,8 +25,9 @@ Usage: install-macos111-host-boot-readback.sh --check install-macos111-host-boot-readback.sh --apply -Installs a no-secret Darwin boot readback and authorizes host 110's existing -public key with a forced command. The key cannot open a shell or forwarding. +Installs a no-secret Darwin boot and performance readback and authorizes host +110's existing public key with a forced command. The key cannot open a shell, +request a PTY, or open forwarding. USAGE } @@ -61,6 +62,12 @@ control_path_check() { "ssh -i /home/wooo/.ssh/id_ed25519 -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=/home/wooo/.ssh/known_hosts -o ConnectTimeout=6 -o ConnectionAttempts=1 -o NumberOfPasswordPrompts=0 '${TARGET_USER}@${TARGET_ADDRESS}' ignored-by-forced-command" } +validate_readback() { + local output="$1" + grep -Eq '^boot_id=darwin_[0-9]+ uptime_seconds=[0-9]+ systemd_state=darwin_ssh startup_enabled=(enabled|unknown) startup_active=(active|inactive)$' <<<"$output" \ + && grep -Eq '^perf_schema=agent99_perf_readback_v1 os=darwin cores=[1-9][0-9]* load1=[0-9]+([.][0-9]+)? mem_available_percent=[0-9]+([.][0-9]+)? disk_used_percent=[0-9]+([.][0-9]+)?$' <<<"$output" +} + check() { local direct_output control_output echo "AWOOOI_MACOS111_BOOT_READBACK=1" @@ -71,6 +78,11 @@ check() { return 75 fi echo "$direct_output" + if ! validate_readback "$direct_output"; then + echo "MACOS111_READBACK_SCRIPT_READY=0" + echo "BLOCKER macos111_readback_schema_incomplete" + return 75 + fi echo "MACOS111_READBACK_SCRIPT_READY=1" if ! control_output="$(control_path_check)"; then @@ -79,6 +91,11 @@ check() { return 75 fi echo "$control_output" + if ! validate_readback "$control_output"; then + echo "MACOS111_CONTROL_PATH_READY=0" + echo "BLOCKER macos111_control_readback_schema_incomplete" + return 75 + fi echo "MACOS111_CONTROL_PATH_READY=1" } diff --git a/scripts/reboot-recovery/macos-host-boot-readback.sh b/scripts/reboot-recovery/macos-host-boot-readback.sh index d8590eccc..149e09fae 100755 --- a/scripts/reboot-recovery/macos-host-boot-readback.sh +++ b/scripts/reboot-recovery/macos-host-boot-readback.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash set -euo pipefail +export LC_ALL=C boot_raw="$(sysctl -n kern.boottime)" boot_epoch="$(printf '%s\n' "$boot_raw" | sed -E 's/^\{ sec = ([0-9]+),.*/\1/')" @@ -29,5 +30,34 @@ else startup_active="inactive" fi +logical_cores="$(sysctl -n hw.logicalcpu 2>/dev/null || true)" +load1="$(sysctl -n vm.loadavg 2>/dev/null | awk '{ print $2; exit }')" +mem_available_percent="$(memory_pressure -Q 2>/dev/null | awk -F': ' ' + /System-wide memory free percentage/ { + value = $2 + gsub(/%/, "", value) + print value + exit + } +')" +disk_used_percent="$(df -Pk / 2>/dev/null | awk ' + NR == 2 { + value = $5 + gsub(/%/, "", value) + print value + exit + } +')" + +if [[ ! "$logical_cores" =~ ^[0-9]+$ ]] || (( logical_cores < 1 )) || + [[ ! "$load1" =~ ^[0-9]+([.][0-9]+)?$ ]] || + [[ ! "$mem_available_percent" =~ ^[0-9]+([.][0-9]+)?$ ]] || + [[ ! "$disk_used_percent" =~ ^[0-9]+([.][0-9]+)?$ ]]; then + echo "BLOCKER darwin_perf_readback_unavailable" + exit 65 +fi + printf 'boot_id=darwin_%s uptime_seconds=%s systemd_state=darwin_ssh startup_enabled=%s startup_active=%s\n' \ "$boot_epoch" "$uptime_seconds" "$startup_enabled" "$startup_active" +printf 'perf_schema=agent99_perf_readback_v1 os=darwin cores=%s load1=%s mem_available_percent=%s disk_used_percent=%s\n' \ + "$logical_cores" "$load1" "$mem_available_percent" "$disk_used_percent" diff --git a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py index d7f1c692b..a0a02ca71 100644 --- a/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_recovery_coordinator_contract.py @@ -25,14 +25,20 @@ def test_recover_requires_full_sop_coordinator_before_verified_resolution() -> N def test_host111_is_in_recovery_executor_and_all_host_readiness_scope() -> None: control = read("agent99-control-plane.ps1") config = read("agent99.config.99.example.json") + generic_config = read("agent99.config.example.json") deploy = read("agent99-deploy.ps1") + bootstrap = read("agent99-bootstrap.ps1") tasks = read("agent99-register-tasks.ps1") assert '"host": "192.168.0.111"' in config + assert '"192.168.0.111": "ooo"' in config + assert '"192.168.0.111": "ooo"' in generic_config assert '"recoveryAction": "wake_on_lan"' in config assert "Invoke-AgentExternalHostRecovery" in control assert 'verifier = "icmp_and_required_tcp_ports"' in control assert "host111_wol_executor" in deploy + assert "host111_canonical_ssh_user" in deploy + assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.111" "ooo"' in bootstrap assert '-Mode Recover -ControlledApply -RunNow' in tasks assert '-Source agent99-startup-recovery' in tasks diff --git a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py index 0eb6e16cd..b1ceea467 100644 --- a/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py +++ b/scripts/reboot-recovery/tests/test_agent99_remote_atomic_deploy_transport_contract.py @@ -58,7 +58,7 @@ def test_sender_is_valid_shell_with_check_as_the_default() -> None: assert "apply_run_id_missing_or_invalid" in source assert "apply_work_item_id_missing_or_invalid" in source assert 'MODE_SELECTION="default"' in source - assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="45"' in source + assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="180"' in source assert 'REMOTE_EXECUTION_TIMEOUT_SECONDS="2400"' in source assert "timeout --signal=TERM --kill-after=10" in source @@ -144,8 +144,12 @@ def test_sender_builds_sha256_envelope_and_streams_it_without_remote_cli_data() assert "hashlib.sha256(content).hexdigest()" in source assert '"manifestSha256": manifest_sha256' in source assert '"contentBase64": base64.b64encode(content).decode("ascii")' in source + assert "gzip.compress(" in source + assert "IO.Compression.GzipStream" in source + assert 'decoder.encode("utf-16le")' in source + assert "-EncodedCommand" in source assert "[Console]::In.ReadToEnd()" in source - assert '<"${BOOTSTRAP_PATH}"' in source + assert '<"${BOOTSTRAP_PAYLOAD_PATH}"' in source assert "rawEnvelope" not in source assert "sshpass" not in source @@ -635,4 +639,5 @@ def test_check_mode_builds_and_streams_bundle_over_bounded_fake_transport( assert "PasswordAuthentication=no" in args assert "Administrator@192.168.0.99" in args assert str(identity) in args - assert int(bytes_path.read_text(encoding="utf-8")) > 500_000 + payload_bytes = int(bytes_path.read_text(encoding="utf-8")) + assert 50_000 < payload_bytes < 500_000 diff --git a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py index d15f0bb86..a17ba8670 100644 --- a/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py +++ b/scripts/reboot-recovery/tests/test_host112_manager_recovery_contract.py @@ -637,10 +637,10 @@ def test_deploy_and_bootstrap_canonicalize_host112_direct_transport() -> None: assert 'Set-AgentProperty $config.sshUsers "192.168.0.112" "kali"' in deploy assert 'name = "host112_canonical_direct_route"' in deploy - assert 'Host112 canonical SSH and VMX config post-verifier failed.' in deploy + assert 'Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed.' in deploy assert "Ensure-AgentHost112CanonicalSshConfig" in bootstrap assert 'Set-AgentBootstrapProperty $config.sshUsers "192.168.0.112" "kali"' in bootstrap - assert 'Host112 canonical SSH and VMX config bootstrap verifier failed.' in bootstrap + assert 'Host111/Host112 canonical SSH and Host112 VMX config bootstrap verifier failed.' in bootstrap for config in (generic_config, host_config): assert '"192.168.0.112": "kali"' in config assert '"directHosts"' in config @@ -682,8 +682,8 @@ def test_deploy_and_bootstrap_canonicalize_host112_ssd_vmx_identity() -> None: assert bootstrap.index(bootstrap_preflight) < bootstrap.index(bootstrap_first_write) assert 'name = "host112_canonical_vmx_path"' in deploy assert "Test-Path -LiteralPath $canonicalHost112Vmx -PathType Leaf" in deploy - assert "Host112 canonical SSH and VMX config post-verifier failed." in deploy - assert "Host112 canonical SSH and VMX config bootstrap verifier failed." in bootstrap + assert "Canonical Host111 SSH/recovery and Host112 direct/SSD config post-verifier failed." in deploy + assert "Host111/Host112 canonical SSH and Host112 VMX config bootstrap verifier failed." in bootstrap assert '$persistedHost112Vm.Count -ne 1' in deploy assert '$persistedHost112Vm.Count -ne 1' in bootstrap assert 'Set-AgentProperty $config "vms" @(@($nonHost112Vms) + @($canonicalHost112Vm))' in deploy diff --git a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py index ecfe0c830..7a57ed211 100644 --- a/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py +++ b/scripts/reboot-recovery/tests/test_reboot_p0_operational_contract.py @@ -127,6 +127,7 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "console_ready=$console_ready" in host112_readiness assert "guest_ready=$guest_ready" in host112_readiness assert "systemd_state_raw=$systemd_state_raw" in host112_readiness + assert "failed_units_query_ok=$failed_units_query_ok" in host112_readiness assert "nonself_failed_count=$nonself_failed_count" in host112_readiness assert "^awoooi-host112-guest-recovery\\.service$" in host112_readiness assert "restrict,command=" in host112_installer @@ -142,8 +143,15 @@ def test_reboot_p0_contract_covers_all_required_hosts_and_vmware_autostart() -> assert "sysctl -n kern.boottime" in macos_readback assert "s/^\\{ sec = ([0-9]+),.*/\\1/" in macos_readback assert "boot_id=darwin_%s" in macos_readback + assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_readback + assert "memory_pressure -Q" in macos_readback + assert "df -Pk /" in macos_readback assert "127.0.0.1:11434/api/tags" in macos_readback assert "restrict,command=" in macos_installer + assert "validate_readback" in macos_installer + assert "macos111_readback_schema_incomplete" in macos_installer + assert "macos111_control_readback_schema_incomplete" in macos_installer + assert "perf_schema=agent99_perf_readback_v1 os=darwin" in macos_installer assert "IdentitiesOnly=yes" in macos_installer assert "CONTROL_PUBLIC_KEY_PATH" in macos_installer assert "authorized_keys" in macos_installer